🎉 S3 source: add support for non-AWS S3 Storage (#6398)

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/69589781-7828-43c5-9f63-8925b1c1ccc2.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2",
   "name": "S3",
   "dockerRepository": "airbyte/source-s3",
-  "dockerImageTag": "0.1.4",
+  "dockerImageTag": "0.1.5",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/s3"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -85,7 +85,7 @@
 - sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
   name: S3
   dockerRepository: airbyte/source-s3
-  dockerImageTag: 0.1.4
+  dockerImageTag: 0.1.5
   documentationUrl: https://docs.airbyte.io/integrations/sources/s3
   sourceType: file
 - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py

@@ -100,6 +100,9 @@ def pytest_collection_modifyitems(config, items):
         i += len(inner_items)
 
     for items in packed_items:
+        if not hasattr(items[0].cls, "config_key"):
+            # Skip user defined test classes from integration_tests/ directory.
+            continue
         test_configs = getattr(config.tests, items[0].cls.config_key())
         for test_config, item in zip(test_configs, items):
             default_timeout = item.get_closest_marker("default_timeout")

airbyte-integrations/connectors/source-s3/Dockerfile

@@ -17,7 +17,7 @@ COPY source_s3 ./source_s3
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
 LABEL io.airbyte.name=airbyte/source-s3
 
 

airbyte-integrations/connectors/source-s3/acceptance-test-config.yml

@@ -11,13 +11,18 @@ tests:
     # for Parquet format
     - config_path: "secrets/parquet_config.json"
       status: "succeed"
+    # for custom server
+    - config_path: "integration_tests/config_minio.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
-   # for CSV format
+    # for CSV format
     - config_path: "secrets/config.json"
     # for Parquet format
     - config_path: "secrets/parquet_config.json"
+    # for custom server
+    - config_path: "integration_tests/config_minio.json"
   basic_read:
     # for CSV format
     - config_path: "secrets/config.json"
@@ -29,17 +34,32 @@ tests:
       configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
       expect_records:
         path: "integration_tests/parquet_expected_records.txt"
-  incremental: 
+    # for custom server
+    - config_path: "integration_tests/config_minio.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      # expected records contains _ab_source_file_last_modified property which
+      # is modified all the time s3 file changed and for custom server it is
+      # file creating date and it always new. Uncomment this line when SAT
+      # would have ability to ignore specific fields from expected records.
+      # expect_records:
+      #  path: "integration_tests/expected_records_custom_server.txt.txt"
+  incremental:
     # for CSV format
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      cursor_paths: 
+      cursor_paths:
         test: ["_ab_source_file_last_modified"]
       future_state_path: "integration_tests/abnormal_state.json"
     # for Parquet format
     - config_path: "secrets/parquet_config.json"
       configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
-      cursor_paths: 
+      cursor_paths:
+        test: ["_ab_source_file_last_modified"]
+      future_state_path: "integration_tests/abnormal_state.json"
+    # for custom server
+    - config_path: "integration_tests/config_minio.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      cursor_paths:
         test: ["_ab_source_file_last_modified"]
       future_state_path: "integration_tests/abnormal_state.json"
 
@@ -50,3 +70,6 @@ tests:
     # for Parquet format
     - config_path: "secrets/parquet_config.json"
       configured_catalog_path: "integration_tests/parquet_configured_catalog.json"
+    # for custom server
+    - config_path: "integration_tests/config_minio.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py

@@ -23,6 +23,11 @@
 #
 
 
+import shutil
+import tempfile
+from zipfile import ZipFile
+
+import docker
 import pytest
 
 pytest_plugins = ("source_acceptance_test.plugin",)
@@ -32,3 +37,22 @@
 def connector_setup():
     """ This fixture is a placeholder for external resources that acceptance test might require."""
     yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def minio_setup():
+    client = docker.from_env()
+    tmp_dir = tempfile.mkdtemp()
+    with ZipFile("./integration_tests/minio_data.zip") as archive:
+        archive.extractall(tmp_dir)
+
+    container = client.containers.run(
+        "minio/minio",
+        f"server {tmp_dir}/minio_data",
+        network_mode="host",
+        volumes=["/tmp:/tmp", "/var/run/docker.sock:/var/run/docker.sock"],
+        detach=True,
+    )
+    yield
+    shutil.rmtree(tmp_dir)
+    container.stop()

airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json

@@ -0,0 +1,16 @@
+{
+  "dataset": "test",
+  "provider": {
+    "storage": "S3",
+    "bucket": "test-bucket",
+    "aws_access_key_id": "123456",
+    "aws_secret_access_key": "123456key",
+    "path_prefix": "",
+    "endpoint": "http://localhost:9000"
+  },
+  "format": {
+    "filetype": "csv"
+  },
+  "path_pattern": "*.csv",
+  "schema": "{}"
+}

airbyte-integrations/connectors/source-s3/integration_tests/expected_records_custom_server.txt

@@ -0,0 +1,55 @@
+{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}
+{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_additional_properties": {}, "_ab_source_file_last_modified": "2021-09-23T11:48:44+0000", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000}

airbyte-integrations/connectors/source-s3/integration_tests/spec.json

@@ -116,13 +116,11 @@
                 "title": "Columns",
                 "description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here. Leave it empty to sync all columns.",
                 "type": "array",
-                "items": {
-                  "type": "string"
-                }
+                "items": { "type": "string" }
               },
               "batch_size": {
                 "title": "Batch Size",
-                "description": "Maximum number of records per batch. Batches may be smaller if there aren’t enough rows in the file. This option can help to optimize a work with memory if your data is particularly wide or failing during detection of OOM errors.",
+                "description": "Maximum number of records per batch. Batches may be smaller if there aren\u2019t enough rows in the file. This option can help to optimize a work with memory if your data is particularly wide or failing during detection of OOM errors.",
                 "default": 65536,
                 "type": "integer"
               }
@@ -156,6 +154,22 @@
             "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimise finding these in S3. This is optional but recommended if your bucket contains many folders/files.",
             "default": "",
             "type": "string"
+          },
+          "endpoint": {
+            "title": "Endpoint",
+            "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.",
+            "default": "",
+            "type": "string"
+          },
+          "use_ssl": {
+            "title": "Use Ssl",
+            "description": "Is remote server using secure SSL/TLS connection",
+            "type": "boolean"
+          },
+          "verify_ssl_cert": {
+            "title": "Verify Ssl Cert",
+            "description": "Allow self signed certificates",
+            "type": "boolean"
           }
         },
         "required": ["bucket"]