🎉 Source File: separate secure fork (#6768)

* create a fork for the source-file connector

* update docs and tests

* update docs and tests

* add secrets

Co-authored-by: George Claireaux <[email protected]>

* switching among auth methods

* refactoring after reviews

* correction of doc

* update spec file



Co-authored-by: Maksym Pavlenok <[email protected]>
Co-authored-by: George Claireaux <[email protected]>

.github/workflows/test-command.yml

@@ -92,6 +92,7 @@ jobs:
           EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
           FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
           FACEBOOK_PAGES_INTEGRATION_TEST_CREDS: ${{ secrets.FACEBOOK_PAGES_INTEGRATION_TEST_CREDS }}
+          FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }}
           FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
           GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }}
           GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }}

airbyte-integrations/connectors/source-file-secure/.dockerignore

@@ -0,0 +1,2 @@
+build
+

airbyte-integrations/connectors/source-file-secure/Dockerfile

@@ -0,0 +1,14 @@
+FROM airbyte/source-file:0.2.6 
+
+WORKDIR /airbyte/integration_code
+
+ENV CODE_PATH="source_file_secure"
+ENV AIRBYTE_IMPL_MODULE="source_file_secure"
+ENV AIRBYTE_IMPL_PATH="SourceFileSecure"
+
+COPY $CODE_PATH ./$CODE_PATH
+RUN sed -i 's/source_file/source_file_secure/g' setup.py
+RUN pip install .
+
+LABEL io.airbyte.name=airbyte/source-file-secure
+LABEL io.airbyte.version=0.1.0

airbyte-integrations/connectors/source-file-secure/README.md

@@ -0,0 +1,95 @@
+# File Source Secure
+This is the repository for the File source connector, written in Python. 
+This is modificaion of another connector Source File. This version has only one difference with the origin version is this one doesn't support local file storages and is orientated for cloud and cluster platforms.
+More details about dependencies and requirement are available [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-file/README.md)
+
+#### Build & Activate Virtual Environment and install dependencies
+From this connector directory, create a virtual environment:
+```
+python -m venv .venv
+```
+
+This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
+development environment of choice. To activate it from the terminal, run:
+```
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
+
+Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
+used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
+If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
+should work as you expect.
+
+#### Building via Gradle
+From the Airbyte repository root, run:
+```
+./gradlew :airbyte-integrations:connectors:source-file-secure:build
+```
+
+#### Create credentials
+Details are explained [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-file/README.md#create-credentials)
+
+Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information.
+See `sample_files/sample_config.json` for a sample config file.
+
+
+
+**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source file test creds`
+and place them into `secrets/config.json`.
+
+
+### Locally running the connector
+```
+python main_dev.py spec
+python main_dev.py check --config secrets/config.json
+python main_dev.py discover --config secrets/config.json
+python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
+```
+
+### Unit Tests
+To run unit tests locally, from the connector directory run:
+```
+python -m pytest unit_tests
+```
+
+### Locally running the connector docker image
+
+#### Build
+First, make sure you build the latest Docker image:
+```
+docker build . -t airbyte/source-file-secure:dev
+```
+
+You can also build the connector image via Gradle:
+```
+./gradlew :airbyte-integrations:connectors:source-file-secure:airbyteDocker
+```
+When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
+the Dockerfile.
+
+#### Run
+Then run any of the connector commands as follows:
+```
+docker run --rm airbyte/source-file-secure:dev spec
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file-secure:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file-secure:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-file-secure:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json
+```
+
+### Integration Tests
+1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-file-secure:integrationTest` to run the standard integration test suite.
+1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s ../source-file/integration_tests`.
+   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+
+## Dependency Management
+All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
+
+### Publishing a new version of the connector
+You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
+1. Make sure your changes are passing unit and integration tests
+1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
+1. Create a Pull Request
+1. Pat yourself on the back for being an awesome contributor
+1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master

airbyte-integrations/connectors/source-file-secure/acceptance-test-config.yml

@@ -0,0 +1,29 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+
+# Here we tries to test a basic tests only. 
+# The main part of tests should be executed for the source-file connector
+connector_image: airbyte/source-file-secure:dev
+tests:
+  spec:
+    - spec_path: "integration_tests/spec.json"
+  connection:
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+    # for https
+    - config_path: "integration_tests/https_config.json"
+      status: "succeed"   
+    # for local should be failed
+    - config_path: "integration_tests/local_config.json"
+      status: "exception"  
+
+  discovery:
+    # for https
+    - config_path: "integration_tests/https_config.json"
+
+  basic_read:
+    # for https
+    - config_path: "integration_tests/https_config.json"
+      configured_catalog_path: "integration_tests/configured_https_catalog.json"
+
+

airbyte-integrations/connectors/source-file-secure/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+source_image=$(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
+echo "try to build the source image: ${source_image}"
+docker build  -t ${source_image} .
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input

airbyte-integrations/connectors/source-file-secure/build.gradle

@@ -0,0 +1,15 @@
+
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+    id 'airbyte-source-acceptance-test'
+}
+
+airbytePython {
+    moduleDirectory 'source_file_secure'
+}
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+}

airbyte-integrations/connectors/source-file-secure/integration_tests/acceptance.py

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """ This fixture is a placeholder for external resources that acceptance test might require."""
+    yield

airbyte-integrations/connectors/source-file-secure/integration_tests/configured_https_catalog.json

@@ -0,0 +1,13 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "test",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-file-secure/integration_tests/https_config.json

@@ -0,0 +1,10 @@
+{
+  "dataset_name": "test",
+  "format": "csv",
+  "reader_options": "{\"sep\": \",\", \"nrows\": 20}",
+  "url": "https://www.stats.govt.nz/assets/Uploads/Business-price-indexes/Business-price-indexes-September-2020-quarter/Download-data/business-price-indexes-september-2020-quarter-corrections-to-previously-published-statistics.csv",
+  "provider": {
+    "storage": "HTTPS",
+    "reader_impl": "gcsfs"
+  }
+}

airbyte-integrations/connectors/source-file-secure/integration_tests/invalid_config.json

@@ -0,0 +1,10 @@
+{
+  "dataset_name": "fake_name",
+  "format": "csv",
+  "reader_options": "{\"bla\": \",\", \"nrows\": 20}",
+  "url": "https://fake-fake.com",
+  "provider": {
+    "storage": "fake",
+    "reader_impl": "fake"
+  }
+}

airbyte-integrations/connectors/source-file-secure/integration_tests/local_config.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "test",
+  "format": "csv",
+  "reader_options": "{\"bla\": \",\", \"nrows\": 20}",
+  "url": "file:///tmp/fake_file.csv",
+  "provider": {
+    "storage": "local"
+  }
+}