Create a secure-only MongoDb destination

airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/8b746512-8c2e-6ac1-4adc-b59faafd473c.json

@@ -2,6 +2,6 @@
   "destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c",
   "name": "MongoDB",
   "dockerRepository": "airbyte/destination-mongodb",
-  "dockerImageTag": "0.1.1",
+  "dockerImageTag": "0.1.2",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/mongodb"
 }

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -93,5 +93,5 @@
 - destinationDefinitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c
   name: MongoDB
   dockerRepository: airbyte/destination-mongodb
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/destinations/mongodb

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile

@@ -0,0 +1,11 @@
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+ENV APPLICATION destination-mongodb-strict-encrypt
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md

@@ -0,0 +1,27 @@
+# MongoDB Test Configuration
+
+In order to test the MongoDB secure only destination, you need a service account key file.
+
+## Community Contributor
+
+As a community contributor, you will need access to a MongoDB to run tests.
+
+1. Create a new account or log into an already created account for mongodb
+2. Go to the `Database Access` page and add new database user with read and write permissions
+3. Add new database with default collection
+4. Add host, port or cluster_url, database name, username and password to `secrets/credentials.json` file
+     ```
+      {
+         "database": "database_name",
+         "user": "user",
+         "password": "password",
+         "cluster_url": "cluster_url",
+         "host": "host",
+         "port": "port"
+       }
+      ```
+
+## Airbyte Employee
+
+1. Access the `MONGODB_TEST_CREDS` secret on the LastPass
+1. Create a file with the contents at `secrets/credentials.json`

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle

@@ -0,0 +1,27 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestinationStrictEncrypt'
+    applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
+}
+
+dependencies {
+    implementation project(':airbyte-db:lib')
+    implementation project(':airbyte-config:models')
+    implementation project(':airbyte-integrations:bases:base-java')
+    implementation project(':airbyte-protocol:models')
+
+    implementation project(':airbyte-integrations:connectors:destination-mongodb')
+    implementation 'org.mongodb:mongodb-driver-sync:4.3.0'
+
+    testImplementation 'org.testcontainers:mongodb:1.15.3'
+
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb-strict-encrypt')
+    integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test')
+
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+}

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.mongodb;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.integrations.base.Destination;
+import io.airbyte.integrations.base.IntegrationRunner;
+import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MongodbDestinationStrictEncrypt extends SpecModifyingDestination implements Destination {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MongodbDestinationStrictEncrypt.class);
+
+  public MongodbDestinationStrictEncrypt() {
+    super(new MongodbDestination());
+  }
+
+  @Override
+  public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception {
+    final ConnectorSpecification spec = Jsons.clone(originalSpec);
+    // removing tls property for a standalone instance to disable possibility to switch off a tls connection
+    ((ObjectNode) spec.getConnectionSpecification().get("properties").get("instance_type").get("oneOf").get(0).get("properties")).remove("tls");
+    return spec;
+  }
+
+  public static void main(String[] args) throws Exception {
+    final Destination destination = new MongodbDestinationStrictEncrypt();
+    LOGGER.info("starting destination: {}", MongodbDestinationStrictEncrypt.class);
+    new IntegrationRunner(destination).run(args);
+    LOGGER.info("completed destination: {}", MongodbDestinationStrictEncrypt.class);
+  }
+
+}

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.mongodb;
+
+import static com.mongodb.client.model.Projections.excludeId;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import com.mongodb.client.MongoCursor;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.db.mongodb.MongoDatabase;
+import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType;
+import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.bson.Document;
+import org.junit.jupiter.api.BeforeAll;
+
+public class MongodbDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest {
+
+  private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json");
+
+  private static final String DATABASE = "database";
+  private static final String AUTH_TYPE = "auth_type";
+  private static final String INSTANCE_TYPE = "instance_type";
+  private static final String AIRBYTE_DATA = "_airbyte_data";
+
+  private static JsonNode config;
+  private static JsonNode failCheckConfig;
+
+  private MongoDatabase mongoDatabase;
+  private MongodbNameTransformer namingResolver = new MongodbNameTransformer();
+
+  @BeforeAll
+  static void setupConfig() throws IOException {
+    if (!Files.exists(CREDENTIALS_PATH)) {
+      throw new IllegalStateException(
+          "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH
+              + ". Override by setting setting path with the CREDENTIALS_PATH constant.");
+    }
+    final String credentialsJsonString = new String(Files.readAllBytes(CREDENTIALS_PATH));
+    final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString);
+
+    final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder()
+        .put("instance", MongoInstanceType.STANDALONE.getType())
+        .put("host", credentialsJson.get("host").asText())
+        .put("port", credentialsJson.get("port").asInt())
+        .build());
+
+    final JsonNode authConfig = Jsons.jsonNode(ImmutableMap.builder()
+        .put("authorization", "login/password")
+        .put("username", credentialsJson.get("user").asText())
+        .put("password", credentialsJson.get("password").asText())
+        .build());
+
+    config = Jsons.jsonNode(ImmutableMap.builder()
+        .put(DATABASE, credentialsJson.get(DATABASE).asText())
+        .put(AUTH_TYPE, authConfig)
+        .put(INSTANCE_TYPE, instanceConfig)
+        .build());
+
+    failCheckConfig = Jsons.jsonNode(ImmutableMap.builder()
+        .put(DATABASE, credentialsJson.get(DATABASE).asText())
+        .put(AUTH_TYPE, Jsons.jsonNode(ImmutableMap.builder()
+            .put("authorization", "none")
+            .build()))
+        .put(INSTANCE_TYPE, instanceConfig)
+        .build());
+  }
+
+  @Override
+  protected String getImageName() {
+    return "airbyte/destination-mongodb-strict-encrypt:dev";
+  }
+
+  @Override
+  protected JsonNode getConfig() {
+    return Jsons.clone(config);
+  }
+
+  @Override
+  protected JsonNode getFailCheckConfig() {
+    return Jsons.clone(failCheckConfig);
+  }
+
+  @Override
+  protected List<JsonNode> retrieveRecords(TestDestinationEnv testEnv, String streamName, String namespace, JsonNode streamSchema) {
+    var collection = mongoDatabase.getOrCreateNewCollection(namingResolver.getRawTableName(streamName));
+    List<JsonNode> result = new ArrayList<>();
+    try (MongoCursor<Document> cursor = collection.find().projection(excludeId()).iterator()) {
+      while (cursor.hasNext()) {
+        result.add(Jsons.jsonNode(cursor.next().get(AIRBYTE_DATA)));
+      }
+    }
+    return result;
+  }
+
+  @Override
+  protected void setup(TestDestinationEnv testEnv) {
+    String connectionString = String.format("mongodb://%s:%s@%s:%s/%s?authSource=admin&ssl=true",
+        config.get(AUTH_TYPE).get("username").asText(),
+        config.get(AUTH_TYPE).get("password").asText(),
+        config.get(INSTANCE_TYPE).get("host").asText(),
+        config.get(INSTANCE_TYPE).get("port").asText(),
+        config.get(DATABASE).asText());
+
+    mongoDatabase = new MongoDatabase(connectionString, config.get(DATABASE).asText());
+  }
+
+  @Override
+  protected void tearDown(TestDestinationEnv testEnv) throws Exception {
+    for (String collectionName : mongoDatabase.getCollectionNames()) {
+      mongoDatabase.getDatabase().getCollection(collectionName).drop();
+    }
+    mongoDatabase.close();
+  }
+
+}

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptTest.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.destination.mongodb;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.resources.MoreResources;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import org.junit.jupiter.api.Test;
+
+public class MongodbDestinationStrictEncryptTest {
+
+  @Test
+  void testSpec() throws Exception {
+    final ConnectorSpecification actual = new MongodbDestinationStrictEncrypt().spec();
+    final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class);
+
+    assertEquals(expected, actual);
+  }
+
+}