From 3d534fcde6ff36761bb54722b00af16a561586dc Mon Sep 17 00:00:00 2001 From: psainics Date: Thu, 24 Apr 2025 13:05:22 +0530 Subject: [PATCH] Changed getConnectionString to incorporate SRV fomrat --- docs/MongoDB-batchsink.md | 3 ++ docs/MongoDB-batchsource.md | 3 ++ .../java/io/cdap/plugin/MongoDBConfig.java | 45 ++++++++++++++----- .../java/io/cdap/plugin/MongoDBConstants.java | 5 +++ .../plugin/batch/sink/MongoDBBatchSink.java | 6 ++- .../batch/source/MongoDBBatchSource.java | 6 ++- .../plugin/batch/MongoDBConfigBuilder.java | 9 +++- .../cdap/plugin/batch/MongoDBConfigTest.java | 23 ++++++++++ .../batch/sink/MongoDBSinkConfigBuilder.java | 2 +- .../source/MongoDBSourceConfigBuilder.java | 4 +- widgets/MongoDB-batchsink.json | 16 +++++++ widgets/MongoDB-batchsource.json | 16 +++++++ 12 files changed, 119 insertions(+), 19 deletions(-) diff --git a/docs/MongoDB-batchsink.md b/docs/MongoDB-batchsink.md index 71e9e01..58f6253 100644 --- a/docs/MongoDB-batchsink.md +++ b/docs/MongoDB-batchsink.md @@ -26,6 +26,9 @@ automatically generated. **Password:** Password to use to connect to the specified database. +**Connect Using SRV String:** Toggle to determine whether to use an SRV connection string for MongoDB. It can be +enabled if the MongoDB deployment supports SRV DNS records for connection resolution. + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. See [Connection String Options] for a full description of these arguments. diff --git a/docs/MongoDB-batchsource.md b/docs/MongoDB-batchsource.md index 8596147..5557d00 100644 --- a/docs/MongoDB-batchsource.md +++ b/docs/MongoDB-batchsource.md @@ -28,6 +28,9 @@ and use the [MongoDB extended JSON format] to represent non-native JSON data typ **Password:** Password to use to connect to the specified database. +**Connect Using SRV String:** Toggle to determine whether to use an SRV connection string for MongoDB. It can be +enabled if the MongoDB deployment supports SRV DNS records for connection resolution. + **Authentication Connection String:** Optional MongoDB connection string to connect to the 'config' database of a sharded cluster. It can be omitted if username and password do not differ from the previously provided ones or if 'config' database does not require authentication. diff --git a/src/main/java/io/cdap/plugin/MongoDBConfig.java b/src/main/java/io/cdap/plugin/MongoDBConfig.java index 2ffb9db..606f7de 100644 --- a/src/main/java/io/cdap/plugin/MongoDBConfig.java +++ b/src/main/java/io/cdap/plugin/MongoDBConfig.java @@ -50,7 +50,8 @@ public class MongoDBConfig extends PluginConfig { @Name(MongoDBConstants.PORT) @Description("Port that MongoDB is listening to.") @Macro - private int port; + @Nullable + private Integer port; @Name(MongoDBConstants.DATABASE) @Description("MongoDB database name.") @@ -76,6 +77,13 @@ public class MongoDBConfig extends PluginConfig { @Nullable private String password; + @Name(MongoDBConstants.CONNECT_USING_SRV_STRING) + @Description("Toggle to determine whether to use an SRV connection string for MongoDB. It can be " + + "enabled if the MongoDB deployment supports SRV DNS records for connection resolution.") + @Macro + @Nullable + private boolean connectUsingSRVString; + @Name(MongoDBConstants.CONNECTION_ARGUMENTS) @Description("A list of arbitrary string key/value pairs as connection arguments.") @Macro @@ -83,7 +91,7 @@ public class MongoDBConfig extends PluginConfig { private String connectionArguments; public MongoDBConfig(String referenceName, String host, int port, String database, String collection, String user, - String password, String connectionArguments) { + String password, boolean connectUsingSRVString, String connectionArguments) { this.referenceName = referenceName; this.host = host; this.port = port; @@ -91,6 +99,7 @@ public MongoDBConfig(String referenceName, String host, int port, String databas this.collection = collection; this.user = user; this.password = password; + this.connectUsingSRVString = connectUsingSRVString; this.connectionArguments = connectionArguments; } @@ -102,7 +111,8 @@ public String getHost() { return host; } - public int getPort() { + @Nullable + public Integer getPort() { return port; } @@ -124,6 +134,10 @@ public String getPassword() { return password; } + public boolean connectUsingSRVString() { + return connectUsingSRVString; + } + @Nullable public String getConnectionArguments() { return connectionArguments; @@ -146,7 +160,8 @@ public void validate() { if (!containsMacro(MongoDBConstants.HOST) && Strings.isNullOrEmpty(host)) { throw new InvalidConfigPropertyException("Host must be specified", MongoDBConstants.HOST); } - if (!containsMacro(MongoDBConstants.PORT)) { + if ((!containsMacro(MongoDBConstants.CONNECT_USING_SRV_STRING) && !connectUsingSRVString) && + !containsMacro(MongoDBConstants.PORT)) { if (port < 1) { throw new InvalidConfigPropertyException("Port number must be greater than 0", MongoDBConstants.PORT); } @@ -161,24 +176,32 @@ public void validate() { /** * Constructs a connection string such as: "mongodb://admin:password@localhost:27017/admin.analytics?key=value;" - * using host, port, username, password, database, collection and optional connection properties. In the case when - * username or password is not provided the connection string will not contain credentials: + * using host, port, username, password, database, collection and optional connection properties. + * If SRV is enabled, the connection string will use the "mongodb+srv://" protocol instead of "mongodb://". + * In the case when username or password is not provided, the connection string will not contain credentials: * "mongodb://localhost:27017/admin.analytics?key=value;" + * When SRV is not used, the port will be included in the connection string. * * @return connection string. */ public String getConnectionString() { - StringBuilder connectionStringBuilder = new StringBuilder("mongodb://"); + StringBuilder connectionStringBuilder = new StringBuilder(); + if (connectUsingSRVString()) { + connectionStringBuilder.append("mongodb+srv://"); + } else { + connectionStringBuilder.append("mongodb://"); + } if (!Strings.isNullOrEmpty(user) || !Strings.isNullOrEmpty(password)) { connectionStringBuilder.append(user).append(":").append(password).append("@"); } - connectionStringBuilder.append(host).append(":").append(port).append("/") - .append(database).append(".").append(collection); - + connectionStringBuilder.append(host); + if (!connectUsingSRVString()) { + connectionStringBuilder.append(":").append(port); + } + connectionStringBuilder.append("/").append(database).append(".").append(collection); if (!Strings.isNullOrEmpty(connectionArguments)) { connectionStringBuilder.append("?").append(connectionArguments); } - return connectionStringBuilder.toString(); } diff --git a/src/main/java/io/cdap/plugin/MongoDBConstants.java b/src/main/java/io/cdap/plugin/MongoDBConstants.java index 3a43797..13fdcf7 100644 --- a/src/main/java/io/cdap/plugin/MongoDBConstants.java +++ b/src/main/java/io/cdap/plugin/MongoDBConstants.java @@ -79,6 +79,11 @@ private MongoDBConstants() { */ public static final String PASSWORD = "password"; + /** + * Configuration property name used to specify whether to use an SRV Connection string for MongoDB. + */ + public static final String CONNECT_USING_SRV_STRING = "connectUsingSRVString"; + /** * Configuration property name used to specify auxiliary MongoDB connection string to authenticate against when * constructing splits. diff --git a/src/main/java/io/cdap/plugin/batch/sink/MongoDBBatchSink.java b/src/main/java/io/cdap/plugin/batch/sink/MongoDBBatchSink.java index ebe54c2..ec79f6f 100644 --- a/src/main/java/io/cdap/plugin/batch/sink/MongoDBBatchSink.java +++ b/src/main/java/io/cdap/plugin/batch/sink/MongoDBBatchSink.java @@ -171,8 +171,10 @@ public static class MongoDBSinkConfig extends MongoDBConfig { private String idField; public MongoDBSinkConfig(String referenceName, String host, int port, String database, String collection, - String user, String password, String connectionArguments, String idField) { - super(referenceName, host, port, database, collection, user, password, connectionArguments); + String user, String password, boolean connectUsingSRVString, + String connectionArguments, String idField) { + super(referenceName, host, port, database, collection, user, password, + connectUsingSRVString, connectionArguments); this.idField = idField; } diff --git a/src/main/java/io/cdap/plugin/batch/source/MongoDBBatchSource.java b/src/main/java/io/cdap/plugin/batch/source/MongoDBBatchSource.java index e85ccfe..fdaa837 100644 --- a/src/main/java/io/cdap/plugin/batch/source/MongoDBBatchSource.java +++ b/src/main/java/io/cdap/plugin/batch/source/MongoDBBatchSource.java @@ -190,9 +190,11 @@ public static class MongoDBSourceConfig extends MongoDBConfig { private String authConnectionString; public MongoDBSourceConfig(String referenceName, String host, int port, String database, String collection, - String user, String password, String connectionArguments, String schema, + String user, String password, boolean connectUsingSRVString, + String connectionArguments, String schema, String inputQuery, String onError, String authConnectionString) { - super(referenceName, host, port, database, collection, user, password, connectionArguments); + super(referenceName, host, port, database, collection, user, password, + connectUsingSRVString, connectionArguments); this.schema = schema; this.inputQuery = inputQuery; this.onError = onError; diff --git a/src/test/java/io/cdap/plugin/batch/MongoDBConfigBuilder.java b/src/test/java/io/cdap/plugin/batch/MongoDBConfigBuilder.java index 16cdc3a..49f4b33 100644 --- a/src/test/java/io/cdap/plugin/batch/MongoDBConfigBuilder.java +++ b/src/test/java/io/cdap/plugin/batch/MongoDBConfigBuilder.java @@ -29,6 +29,7 @@ public class MongoDBConfigBuilder { protected String collection; protected String user; protected String password; + protected boolean connectUsingSRVString; protected String connectionArguments; public static MongoDBConfigBuilder builder() { @@ -82,12 +83,18 @@ public T setPassword(String password) { return (T) this; } + public T setConnectUsingSRVString(boolean connectUsingSRVString) { + this.connectUsingSRVString = connectUsingSRVString; + return (T) this; + } + public T setConnectionArguments(String connectionArguments) { this.connectionArguments = connectionArguments; return (T) this; } public MongoDBConfig build() { - return new MongoDBConfig(referenceName, host, port, database, collection, user, password, connectionArguments); + return new MongoDBConfig(referenceName, host, port, database, collection, user, password, + connectUsingSRVString, connectionArguments); } } diff --git a/src/test/java/io/cdap/plugin/batch/MongoDBConfigTest.java b/src/test/java/io/cdap/plugin/batch/MongoDBConfigTest.java index daf3812..821ca28 100644 --- a/src/test/java/io/cdap/plugin/batch/MongoDBConfigTest.java +++ b/src/test/java/io/cdap/plugin/batch/MongoDBConfigTest.java @@ -36,6 +36,7 @@ public class MongoDBConfigTest { .setCollection("analytics") .setUser("admin") .setPassword("password") + .setConnectUsingSRVString(false) .setConnectionArguments("key=value;") .build(); @@ -45,6 +46,16 @@ public void testConfigConnectionString() { VALID_CONFIG.getConnectionString()); } + @Test + public void testConfigConnectionStringWithSRV() { + String connectionString = MongoDBConfigBuilder.builder(VALID_CONFIG) + .setConnectUsingSRVString(true) + .build() + .getConnectionString(); + + Assert.assertEquals("mongodb+srv://admin:password@localhost/admin.analytics?key=value;", connectionString); + } + @Test public void testConfigConnectionStringNoCreds() { String connectionString = MongoDBConfigBuilder.builder(VALID_CONFIG) @@ -56,6 +67,18 @@ public void testConfigConnectionStringNoCreds() { Assert.assertEquals("mongodb://localhost:27017/admin.analytics?key=value;", connectionString); } + @Test + public void testConfigConnectionStringWithSRVNoCreds() { + String connectionString = MongoDBConfigBuilder.builder(VALID_CONFIG) + .setConnectUsingSRVString(true) + .setUser(null) + .setPassword(null) + .build() + .getConnectionString(); + + Assert.assertEquals("mongodb+srv://localhost/admin.analytics?key=value;", connectionString); + } + @Test public void testValidateValid() { VALID_CONFIG.validate(); diff --git a/src/test/java/io/cdap/plugin/batch/sink/MongoDBSinkConfigBuilder.java b/src/test/java/io/cdap/plugin/batch/sink/MongoDBSinkConfigBuilder.java index a226477..910ded1 100644 --- a/src/test/java/io/cdap/plugin/batch/sink/MongoDBSinkConfigBuilder.java +++ b/src/test/java/io/cdap/plugin/batch/sink/MongoDBSinkConfigBuilder.java @@ -49,6 +49,6 @@ public MongoDBSinkConfigBuilder setIdField(String idField) { public MongoDBBatchSink.MongoDBSinkConfig build() { return new MongoDBBatchSink.MongoDBSinkConfig(referenceName, host, port, database, collection, user, password, - connectionArguments, idField); + connectUsingSRVString, connectionArguments, idField); } } diff --git a/src/test/java/io/cdap/plugin/batch/source/MongoDBSourceConfigBuilder.java b/src/test/java/io/cdap/plugin/batch/source/MongoDBSourceConfigBuilder.java index 5a87fb7..466e379 100644 --- a/src/test/java/io/cdap/plugin/batch/source/MongoDBSourceConfigBuilder.java +++ b/src/test/java/io/cdap/plugin/batch/source/MongoDBSourceConfigBuilder.java @@ -70,7 +70,7 @@ public MongoDBSourceConfigBuilder setAuthConnectionString(String authConnectionS public MongoDBBatchSource.MongoDBSourceConfig build() { return new MongoDBBatchSource.MongoDBSourceConfig(referenceName, host, port, database, collection, user, password, - connectionArguments, schema, inputQuery, onError, - authConnectionString); + connectUsingSRVString, connectionArguments, schema, inputQuery, + onError, authConnectionString); } } diff --git a/widgets/MongoDB-batchsink.json b/widgets/MongoDB-batchsink.json index da4278d..1cc0ef3 100644 --- a/widgets/MongoDB-batchsink.json +++ b/widgets/MongoDB-batchsink.json @@ -57,6 +57,22 @@ "widget-type": "password", "label": "Password", "name": "password" + }, + { + "widget-type": "toggle", + "label": "Connect Using SRV String", + "name": "connectUsingSRVString", + "widget-attributes": { + "on": { + "value": "true", + "label": "True" + }, + "off": { + "value": "false", + "label": "False" + }, + "default": "false" + } } ] }, diff --git a/widgets/MongoDB-batchsource.json b/widgets/MongoDB-batchsource.json index 6f9d852..788e045 100644 --- a/widgets/MongoDB-batchsource.json +++ b/widgets/MongoDB-batchsource.json @@ -61,6 +61,22 @@ "label": "Password", "name": "password" }, + { + "widget-type": "toggle", + "label": "Connect Using SRV String", + "name": "connectUsingSRVString", + "widget-attributes": { + "on": { + "value": "true", + "label": "True" + }, + "off": { + "value": "false", + "label": "False" + }, + "default": "false" + } + }, { "widget-type": "textbox", "label": "Authentication Connection String",