Merge remote-tracking branch 'origin/master' into bazarnov/5712-dat-i…

…so8601-compliant-date-strings

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.29.13-alpha
+current_version = 0.29.15-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -1,4 +1,4 @@
-VERSION=0.29.13-alpha
+VERSION=0.29.15-alpha
 
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker

.gitbook.yaml

@@ -72,3 +72,4 @@ redirects:
   contributing-to-airbyte/building-new-connector/tutorials/building-a-python-source: ./connector-development/tutorials/building-a-python-source.md
   contributing-to-airbyte/building-new-connector/tutorials/building-a-python-destination: ./connector-development/tutorials/building-a-python-destination.md
   contributing-to-airbyte/building-new-connector/tutorials/building-a-java-destination: ./connector-development/tutorials/building-a-java-destination.md
+  project-overview/code-of-conduct: ./project-overview/slack-code-of-conduct

.github/pull_request_template.md

@@ -23,6 +23,7 @@ Expand the relevant checklist and delete the others.
 - [ ] Code reviews completed
 - [ ] Documentation updated 
     - [ ] Connector's `README.md`
+    - [ ] Connector's `bootstrap.md`. See [description and examples](https://docs.google.com/document/d/1ypdgmwmEHWv-TrO4_YOQ7pAJGVrMp5BOkEVh831N260/edit?usp=sharing)
     - [ ] `docs/SUMMARY.md`
     - [ ] `docs/integrations/<source or destination>/<name>.md` including changelog. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog)
     - [ ] `docs/integrations/README.md`
@@ -55,6 +56,7 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 - [ ] Code reviews completed
 - [ ] Documentation updated 
     - [ ] Connector's `README.md`
+    - [ ] Connector's `bootstrap.md`. See [description and examples](https://docs.google.com/document/d/1ypdgmwmEHWv-TrO4_YOQ7pAJGVrMp5BOkEVh831N260/edit?usp=sharing)
     - [ ] Changelog updated in `docs/integrations/<source or destination>/<name>.md` including changelog. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog)
 - [ ] PR name follows [PR naming conventions](https://docs.airbyte.io/contributing-to-airbyte/updating-documentation#issues-and-pull-requests)
 - [ ] Connector version bumped like described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)

.github/workflows/publish-command.yml

@@ -120,6 +120,8 @@ jobs:
           MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }}
           MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }}
           PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
+          POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
+          POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
           POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
           PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
           RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
@@ -141,6 +143,7 @@ jobs:
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
           SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }}
+          SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }}
           SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }}
           SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
           STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}

.github/workflows/release-airbyte-os.yml

@@ -15,18 +15,25 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '14'
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.7'
       - name: Save Old Version
         id: old_version
         run: |
           echo ::set-output name=OLD_VERSION::$(grep VERSION .env | cut -d"=" -f2)
       - name: Release Airbyte
         id: release_airbyte
         env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
           PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
           CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }}
           CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }}
         run: |
-          docker login -u airbytebot -p ${{ secrets.DOCKER_PASSWORD }}
           ./tools/bin/release_version.sh
       - name: Save New Version
         id: new_version

.github/workflows/test-command.yml

@@ -121,6 +121,8 @@ jobs:
           MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }}
           MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }}
           PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
+          POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
+          POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
           POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
           PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
           RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
@@ -141,6 +143,7 @@ jobs:
           SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG: ${{ secrets.SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG }}
           SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
           SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }}
+          SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }}
           SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }}
           SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
           STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}

airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java

@@ -31,16 +31,20 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
 import io.airbyte.commons.jackson.MoreMappers;
 import io.airbyte.commons.stream.MoreStreams;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
 public class Jsons {
@@ -49,42 +53,42 @@ public class Jsons {
   private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper();
   private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writer(new JsonPrettyPrinter());
 
-  public static <T> String serialize(T object) {
+  public static <T> String serialize(final T object) {
     try {
       return OBJECT_MAPPER.writeValueAsString(object);
-    } catch (JsonProcessingException e) {
+    } catch (final JsonProcessingException e) {
       throw new RuntimeException(e);
     }
   }
 
   public static <T> T deserialize(final String jsonString, final Class<T> klass) {
     try {
       return OBJECT_MAPPER.readValue(jsonString, klass);
-    } catch (IOException e) {
+    } catch (final IOException e) {
       throw new RuntimeException(e);
     }
   }
 
   public static JsonNode deserialize(final String jsonString) {
     try {
       return OBJECT_MAPPER.readTree(jsonString);
-    } catch (IOException e) {
+    } catch (final IOException e) {
       throw new RuntimeException(e);
     }
   }
 
   public static <T> Optional<T> tryDeserialize(final String jsonString, final Class<T> klass) {
     try {
       return Optional.of(OBJECT_MAPPER.readValue(jsonString, klass));
-    } catch (IOException e) {
+    } catch (final IOException e) {
       return Optional.empty();
     }
   }
 
   public static Optional<JsonNode> tryDeserialize(final String jsonString) {
     try {
       return Optional.of(OBJECT_MAPPER.readTree(jsonString));
-    } catch (IOException e) {
+    } catch (final IOException e) {
       return Optional.empty();
     }
   }
@@ -108,15 +112,15 @@ public static <T> T object(final JsonNode jsonNode, final TypeReference<T> typeR
   public static <T> Optional<T> tryObject(final JsonNode jsonNode, final Class<T> klass) {
     try {
       return Optional.of(OBJECT_MAPPER.convertValue(jsonNode, klass));
-    } catch (Exception e) {
+    } catch (final Exception e) {
       return Optional.empty();
     }
   }
 
   public static <T> Optional<T> tryObject(final JsonNode jsonNode, final TypeReference<T> typeReference) {
     try {
       return Optional.of(OBJECT_MAPPER.convertValue(jsonNode, typeReference));
-    } catch (Exception e) {
+    } catch (final Exception e) {
       return Optional.empty();
     }
   }
@@ -126,30 +130,85 @@ public static <T> T clone(final T object) {
     return (T) deserialize(serialize(object), object.getClass());
   }
 
-  public static byte[] toBytes(JsonNode jsonNode) {
+  public static byte[] toBytes(final JsonNode jsonNode) {
     return serialize(jsonNode).getBytes(Charsets.UTF_8);
   }
 
-  public static Set<String> keys(JsonNode jsonNode) {
+  public static Set<String> keys(final JsonNode jsonNode) {
     if (jsonNode.isObject()) {
       return Jsons.object(jsonNode, new TypeReference<Map<String, Object>>() {}).keySet();
     } else {
       return new HashSet<>();
     }
   }
 
-  public static List<JsonNode> children(JsonNode jsonNode) {
+  public static List<JsonNode> children(final JsonNode jsonNode) {
     return MoreStreams.toStream(jsonNode.elements()).collect(Collectors.toList());
   }
 
-  public static String toPrettyString(JsonNode jsonNode) {
+  public static String toPrettyString(final JsonNode jsonNode) {
     try {
       return OBJECT_WRITER.writeValueAsString(jsonNode) + "\n";
-    } catch (JsonProcessingException e) {
+    } catch (final JsonProcessingException e) {
       throw new RuntimeException(e);
     }
   }
 
+  public static JsonNode navigateTo(JsonNode node, final List<String> keys) {
+    for (final String key : keys) {
+      node = node.get(key);
+    }
+    return node;
+  }
+
+  public static void replaceNestedString(final JsonNode json, final List<String> keys, final String replacement) {
+    replaceNested(json, keys, (node, finalKey) -> node.put(finalKey, replacement));
+  }
+
+  public static void replaceNestedInt(final JsonNode json, final List<String> keys, final int replacement) {
+    replaceNested(json, keys, (node, finalKey) -> node.put(finalKey, replacement));
+  }
+
+  private static void replaceNested(final JsonNode json, final List<String> keys, final BiConsumer<ObjectNode, String> typedReplacement) {
+    Preconditions.checkArgument(keys.size() > 0, "Must pass at least one key");
+    final JsonNode nodeContainingFinalKey = navigateTo(json, keys.subList(0, keys.size() - 1));
+    typedReplacement.accept((ObjectNode) nodeContainingFinalKey, keys.get(keys.size() - 1));
+  }
+
+  public static Optional<JsonNode> getOptional(final JsonNode json, final String... keys) {
+    return getOptional(json, Arrays.asList(keys));
+  }
+
+  public static Optional<JsonNode> getOptional(JsonNode json, final List<String> keys) {
+    for (final String key : keys) {
+      if (json == null) {
+        return Optional.empty();
+      }
+
+      json = json.get(key);
+    }
+
+    return Optional.ofNullable(json);
+  }
+
+  public static String getStringOrNull(final JsonNode json, final String... keys) {
+    return getStringOrNull(json, Arrays.asList(keys));
+  }
+
+  public static String getStringOrNull(final JsonNode json, final List<String> keys) {
+    final Optional<JsonNode> optional = getOptional(json, keys);
+    return optional.map(JsonNode::asText).orElse(null);
+  }
+
+  public static int getIntOrZero(final JsonNode json, final String... keys) {
+    return getIntOrZero(json, Arrays.asList(keys));
+  }
+
+  public static int getIntOrZero(final JsonNode json, final List<String> keys) {
+    final Optional<JsonNode> optional = getOptional(json, keys);
+    return optional.map(JsonNode::asInt).orElse(0);
+  }
+
   /**
    * By the Jackson DefaultPrettyPrinter prints objects with an extra space as follows: {"name" :
    * "airbyte"}. We prefer {"name": "airbyte"}.
@@ -165,7 +224,7 @@ public DefaultPrettyPrinter createInstance() {
 
     // override the method that inserts the extra space.
     @Override
-    public DefaultPrettyPrinter withSeparators(Separators separators) {
+    public DefaultPrettyPrinter withSeparators(final Separators separators) {
       _separators = separators;
       _objectFieldValueSeparatorWithSpaces = separators.getObjectFieldValueSeparator() + " ";
       return this;

airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java

@@ -30,14 +30,18 @@
 
 public class Strings {
 
-  public static String join(Iterable<?> iterable, CharSequence separator) {
+  public static String join(final Iterable<?> iterable, final CharSequence separator) {
     return Streams.stream(iterable)
         .map(Object::toString)
         .collect(Collectors.joining(separator));
   }
 
-  public static String addRandomSuffix(String base, String separator, int suffixLength) {
+  public static String addRandomSuffix(final String base, final String separator, final int suffixLength) {
     return base + separator + RandomStringUtils.randomAlphabetic(suffixLength).toLowerCase();
   }
 
+  public static String safeTrim(final String string) {
+    return string == null ? null : string.trim();
+  }
+
 }

airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java

@@ -27,6 +27,7 @@
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -241,6 +242,32 @@ void testToPrettyString() {
     assertEquals(expectedOutput, Jsons.toPrettyString(jsonNode));
   }
 
+  @Test
+  void testGetOptional() {
+    final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": {}, \"mno\": \"pqr\", \"stu\": null }");
+
+    assertEquals(Optional.of(Jsons.jsonNode("ghi")), Jsons.getOptional(json, "abc", "def"));
+    assertEquals(Optional.of(Jsons.emptyObject()), Jsons.getOptional(json, "jkl"));
+    assertEquals(Optional.of(Jsons.jsonNode("pqr")), Jsons.getOptional(json, "mno"));
+    assertEquals(Optional.of(Jsons.jsonNode(null)), Jsons.getOptional(json, "stu"));
+    assertEquals(Optional.empty(), Jsons.getOptional(json, "xyz"));
+    assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "xyz"));
+    assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "def", "xyz"));
+    assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "jkl", "xyz"));
+    assertEquals(Optional.empty(), Jsons.getOptional(json, "stu", "xyz"));
+  }
+
+  @Test
+  void testGetStringOrNull() {
+    final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": \"mno\", \"pqr\": 1 }");
+
+    assertEquals("ghi", Jsons.getStringOrNull(json, "abc", "def"));
+    assertEquals("mno", Jsons.getStringOrNull(json, "jkl"));
+    assertEquals("1", Jsons.getStringOrNull(json, "pqr"));
+    assertNull(Jsons.getStringOrNull(json, "abc", "def", "xyz"));
+    assertNull(Jsons.getStringOrNull(json, "xyz"));
+  }
+
   private static class ToClass {
 
     @JsonProperty("str")
@@ -254,21 +281,21 @@ private static class ToClass {
 
     public ToClass() {}
 
-    public ToClass(String str, Integer num, long numLong) {
+    public ToClass(final String str, final Integer num, final long numLong) {
       this.str = str;
       this.num = num;
       this.numLong = numLong;
     }
 
     @Override
-    public boolean equals(Object o) {
+    public boolean equals(final Object o) {
       if (this == o) {
         return true;
       }
       if (o == null || getClass() != o.getClass()) {
         return false;
       }
-      ToClass toClass = (ToClass) o;
+      final ToClass toClass = (ToClass) o;
       return numLong == toClass.numLong
           && Objects.equals(str, toClass.str)
           && Objects.equals(num, toClass.num);

airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json

@@ -2,6 +2,6 @@
   "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba",
   "name": "Snowflake",
   "dockerRepository": "airbyte/destination-snowflake",
-  "dockerImageTag": "0.3.12",
+  "dockerImageTag": "0.3.13",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake"
 }

airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8ccd8909-4e99-4141-b48d-4984b70b2d89.json

@@ -0,0 +1,7 @@
+{
+  "destinationDefinitionId": "8ccd8909-4e99-4141-b48d-4984b70b2d89",
+  "name": "DynamoDB",
+  "dockerRepository": "airbyte/destination-dynamodb",
+  "dockerImageTag": "0.1.0",
+  "documentationUrl": "https://docs.airbyte.io/integrations/destinations/dynamodb"
+}

airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json

@@ -2,7 +2,7 @@
   "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc",
   "name": "Redshift",
   "dockerRepository": "airbyte/destination-redshift",
-  "dockerImageTag": "0.3.12",
+  "dockerImageTag": "0.3.13",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift",
   "icon": "redshift.svg"
 }