🎉 Destination Postgres: Add jdbc_url_params input (#12195)

* Add jdbc_url_params input for Destination Postgres
airbytehq · Apr 25, 2022 · d46058e · d46058e
1 parent c42b00e
commit d46058e
Show file tree

Hide file tree

Showing 5 changed files with 78 additions and 22 deletions.
diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.3.18
+LABEL io.airbyte.version=0.3.19
 LABEL io.airbyte.name=airbyte/destination-postgres
diff --git a/...tgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/...tgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java
@@ -25,6 +25,13 @@ public class PostgresDestination extends AbstractJdbcDestination implements Dest
   public static final String DRIVER_CLASS = "org.postgresql.Driver";
   public static final List<String> HOST_KEY = List.of("host");
   public static final List<String> PORT_KEY = List.of("port");
+  public static final String DATABASE_KEY = "database";
+  public static final String JDBC_URL_KEY = "jdbc_url";
+  public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params";
+  public static final String PASSWORD_KEY = "password";
+  public static final String SSL_KEY = "ssl";
+  public static final String USERNAME_KEY = "username";
+  public static final String SCHEMA_KEY = "schema";
 
   static final Map<String, String> SSL_JDBC_PARAMETERS = ImmutableMap.of(
       "ssl", "true",
@@ -55,21 +62,26 @@ public JsonNode toJdbcConfig(final JsonNode config) {
     final String jdbcUrl = String.format("jdbc:postgresql://%s:%s/%s?",
         config.get("host").asText(),
         config.get("port").asText(),
-        config.get("database").asText());
+        config.get(DATABASE_KEY).asText());
 
     final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
-        .put("username", config.get("username").asText())
-        .put("jdbc_url", jdbcUrl)
-        .put("schema", schema);
+        .put(USERNAME_KEY, config.get(USERNAME_KEY).asText())
+        .put(JDBC_URL_KEY, jdbcUrl)
+        .put(SCHEMA_KEY, schema);
 
-    if (config.has("password")) {
-      configBuilder.put("password", config.get("password").asText());
+    if (config.has(PASSWORD_KEY)) {
+      configBuilder.put(PASSWORD_KEY, config.get(PASSWORD_KEY).asText());
     }
+
+    if (config.has(JDBC_URL_PARAMS_KEY)) {
+      configBuilder.put(JDBC_URL_PARAMS_KEY, config.get(JDBC_URL_PARAMS_KEY).asText());
+    }
+
     return Jsons.jsonNode(configBuilder.build());
   }
 
   private boolean useSsl(final JsonNode config) {
-    return !config.has("ssl") || config.get("ssl").asBoolean();
+    return !config.has(SSL_KEY) || config.get(SSL_KEY).asBoolean();
   }
 
   public static void main(final String[] args) throws Exception {

diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json
@@ -60,6 +60,12 @@
         "type": "boolean",
         "default": false,
         "order": 6
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 7
       }
     }
   }

diff --git a/...s/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/...s/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java
@@ -9,7 +9,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.map.MoreMaps;
 import io.airbyte.db.jdbc.JdbcDatabase;
 import io.airbyte.db.jdbc.JdbcUtils;
 import io.airbyte.integrations.base.AirbyteMessageConsumer;
@@ -50,16 +49,33 @@ public class PostgresDestinationTest {
 
   private JsonNode config;
 
-  private static final Map<String, String> CONFIG_WITH_SSL = ImmutableMap.of(
-      "host", "localhost",
-      "port", "1337",
-      "username", "user",
-      "database", "db");
+  private static final String EXPECTED_JDBC_URL = "jdbc:postgresql://localhost:1337/db?";
 
-  private static final Map<String, String> CONFIG_NO_SSL = MoreMaps.merge(
-      CONFIG_WITH_SSL,
-      ImmutableMap.of(
-          "ssl", "false"));
+  private JsonNode buildConfigNoJdbcParameters() {
+    return Jsons.jsonNode(ImmutableMap.of(
+        "host", "localhost",
+        "port", 1337,
+        "username", "user",
+        "database", "db"));
+  }
+
+  private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) {
+    return Jsons.jsonNode(ImmutableMap.of(
+        "host", "localhost",
+        "port", 1337,
+        "username", "user",
+        "database", "db",
+        "jdbc_url_params", extraParam));
+  }
+
+  private JsonNode buildConfigNoExtraJdbcParametersWithoutSsl() {
+    return Jsons.jsonNode(ImmutableMap.of(
+        "host", "localhost",
+        "port", 1337,
+        "username", "user",
+        "database", "db",
+        "ssl", false));
+  }
 
   @BeforeAll
   static void init() {
@@ -77,17 +93,36 @@ static void cleanUp() {
     PSQL_DB.close();
   }
 
+  @Test
+  void testJdbcUrlAndConfigNoExtraParams() {
+    final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigNoJdbcParameters());
+    assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
+  }
+
+  @Test
+  void testJdbcUrlEmptyExtraParams() {
+    final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(""));
+    assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
+  }
+
+  @Test
+  void testJdbcUrlExtraParams() {
+    final String extraParam = "key1=value1&key2=value2&key3=value3";
+    final JsonNode jdbcConfig = new PostgresDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam));
+    assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get("jdbc_url").asText());
+  }
+
   @Test
   void testDefaultParamsNoSSL() {
     final Map<String, String> defaultProperties = new PostgresDestination().getDefaultConnectionProperties(
-        Jsons.jsonNode(CONFIG_NO_SSL));
+        buildConfigNoExtraJdbcParametersWithoutSsl());
     assertEquals(new HashMap<>(), defaultProperties);
   }
 
   @Test
   void testDefaultParamsWithSSL() {
     final Map<String, String> defaultProperties = new PostgresDestination().getDefaultConnectionProperties(
-        Jsons.jsonNode(CONFIG_WITH_SSL));
+        buildConfigNoJdbcParameters());
     assertEquals(PostgresDestination.SSL_JDBC_PARAMETERS, defaultProperties);
   }
 

diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md
@@ -63,6 +63,8 @@ You should now have all the requirements needed to configure Postgres as a desti
 * **Password**
 * **Default Schema Name**
 * **Database**
+* **JDBC URL Params** (optional)
+
 
 ## Naming Conventions
 
@@ -84,8 +86,9 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the
 
 | Version | Date       | Pull Request | Subject                                                                                             |
 |:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------|
-| 0.3.18  | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0                                                                   |
-| 0.3.17  | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name                                                                   |
+| 0.3.19  | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input                                                    |
+| 0.3.18  | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0                                                                  |
+| 0.3.17  | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name                                                                |
 | 0.3.15  | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                   |
 | 0.3.14  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                          |
 | 0.3.13  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key                                                            |