Merge pull request #16 from invitae/sync-upstream

invitae · Dec 22, 2020 · 778a698 · 778a698
2 parents 3f080c6 + 85a6b80
commit 778a698
Show file tree

Hide file tree

Showing 77 changed files with 3,066 additions and 671 deletions.
diff --git a/.github/scripts/decrypt_secret.sh b/.github/scripts/decrypt_secret.sh
@@ -3,5 +3,12 @@
 # Decrypt the file
 # mkdir $HOME/secrets
 # --batch to prevent interactive command --yes to assume "yes" for questions
-gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
---output profile.json .github/scripts/profile.json.gpg
+snowflake_deployment=$1
+
+if [ $snowflake_deployment = 'aws' ]; then
+  gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
+  --output profile.json .github/scripts/profile.json.gpg
+else
+  gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_PROFILE_SECRET" \
+  --output profile.json .github/scripts/profile_azure.json.gpg
+fi
diff --git a/.github/scripts/profile_azure.json.gpg b/.github/scripts/profile_azure.json.gpg
diff --git a/.github/workflows/End2EndFullTest.yml b/.github/workflows/End2EndFullTest.yml
@@ -1,4 +1,4 @@
-name: Kafka Connector End2End Full Version Test
+name: Kafka Connector End2End Full Version Test AWS
 
 on:
   push:
@@ -24,21 +24,14 @@ jobs:
         python-version: '3.6'
         architecture: 'x64'
     - name: Decrypt profile.json
-      run: ./.github/scripts/decrypt_secret.sh
+      run: ./.github/scripts/decrypt_secret.sh 'aws'
       env:
         SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
-    - name: Restore Maven Cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-maven-
     - name: Install Dependency
       run: |
         pip3 install --upgrade setuptools
-        pip3 install "confluent-kafka[avro]"
-        pip3 install avro-python3
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+        pip3 install avro-python3 kafka-python
         pip3 install protobuf
         pip3 install --upgrade snowflake-connector-python
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
@@ -54,20 +47,6 @@ jobs:
         cd test
         ./build_apache.sh ../../snowflake-kafka-connector
 
-    - name: End to End Test of Confluent Platform 5.0.0
-      env:
-        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
-      run: |
-        cd test
-        ./run_test_confluent.sh 5.0.0 ./apache_properties
-
-    - name: End to End Test of Apache Plarform 2.0.0
-      env:
-        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
-      run: |
-        cd test
-        ./run_test_apache.sh 2.0.0 ./apache_properties
-
     - name: End to End Test of Confluent Platform 5.2.0
       env:
         SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"

diff --git a/.github/workflows/End2EndTest.yml → .github/workflows/End2EndFullTestAzure.yml b/.github/workflows/End2EndTest.yml → .github/workflows/End2EndFullTestAzure.yml
@@ -1,14 +1,12 @@
-name: Kafka Connector End2End Test
+name: Kafka Connector End2End Full Version Test Azure
 
 on:
   push:
     branches-ignore:
       - '**'
 #on:
-#  push:
-#    branches: [ master ]
-#  pull_request:
-#    branches: '**'
+#  schedule:
+#    - cron:  '0 8 * * *'
 
 jobs:
   build:
@@ -26,37 +24,20 @@ jobs:
         python-version: '3.6'
         architecture: 'x64'
     - name: Decrypt profile.json
-      run: ./.github/scripts/decrypt_secret.sh
+      run: ./.github/scripts/decrypt_secret.sh 'azure'
       env:
         SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
-    - name: Restore Maven Cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-maven-
     - name: Install Dependency
       run: |
         pip3 install --upgrade setuptools
-        pip3 install "confluent-kafka[avro]"
-        pip3 install avro-python3
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+        pip3 install avro-python3 kafka-python
         pip3 install protobuf
         pip3 install --upgrade snowflake-connector-python
         curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
         sudo apt-get -y install jq
         sudo apt-get -y install protobuf-compiler
 
-#    - name: Start Kubernetes Cluster
-#      env:
-#        CHANGE_MINIKUBE_NONE_USER: true
-#      run: |
-#        sudo apt-get -y install conntrack
-#        curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.9.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-#        sudo minikube start --vm-driver=none
-#        sudo chown -R $USER $HOME/.kube $HOME/.minikube || true
-#        minikube update-context
-
     - name: Build with Unit and Integration Test
       env:
         JACOCO_COVERAGE: true
@@ -66,14 +47,42 @@ jobs:
         cd test
         ./build_apache.sh ../../snowflake-kafka-connector
 
-    - name: End to End Test of Confluent Platform 5.5
+    - name: End to End Test of Confluent Platform 5.2.0
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_confluent.sh 5.2.0 ./apache_properties
+
+    - name: End to End Test of Apache Plarform 2.2.0
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_apache.sh 2.2.0 ./apache_properties
+
+    - name: End to End Test of Confluent Platform 5.4.0
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_confluent.sh 5.4.0 ./apache_properties
+
+    - name: End to End Test of Apache Plarform 2.4.0
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_apache.sh 2.4.0 ./apache_properties
+
+    - name: End to End Test of Confluent Platform 5.5.0
       env:
         SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
       run: |
         cd test
         ./run_test_confluent.sh 5.5.0 ./apache_properties
 
-    - name: End to End Test of Apache Plarform 2.5
+    - name: End to End Test of Apache Plarform 2.5.0
       env:
         SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
       run: |
@@ -82,6 +91,3 @@ jobs:
 
     - name: Code Coverage
       uses: codecov/codecov-action@v1
-
-
-
diff --git a/.github/workflows/End2EndTestApacheAws.yml b/.github/workflows/End2EndTestApacheAws.yml
@@ -0,0 +1,57 @@
+name: Kafka Connector Apache End2End Test AWS
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+#on:
+#  push:
+#    branches: [ master ]
+#  pull_request:
+#    branches: '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: "Install Java 8"
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Install Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+        architecture: 'x64'
+    - name: Decrypt profile.json
+      run: ./.github/scripts/decrypt_secret.sh 'aws'
+      env:
+        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+    - name: Install Dependency
+      run: |
+        pip3 install --upgrade setuptools
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+        pip3 install avro-python3 kafka-python
+        pip3 install protobuf
+        pip3 install --upgrade snowflake-connector-python
+        curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+        sudo apt-get -y install jq
+        sudo apt-get -y install protobuf-compiler
+    
+    - name: Build with Unit Test
+      env:
+        JACOCO_COVERAGE: true
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        SHELL: "/bin/bash"
+      run: |
+        cd test
+        ./build_apache.sh ../../snowflake-kafka-connector package
+
+    - name: End to End Test of Apache Plarform 2.5
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_apache.sh 2.5.0 ./apache_properties
diff --git a/.github/workflows/End2EndTestApacheAzure.yml b/.github/workflows/End2EndTestApacheAzure.yml
@@ -0,0 +1,57 @@
+name: Kafka Connector Apache End2End Test Azure
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+#on:
+#  push:
+#    branches: [ master ]
+#  pull_request:
+#    branches: '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: "Install Java 8"
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Install Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+        architecture: 'x64'
+    - name: Decrypt profile.json
+      run: ./.github/scripts/decrypt_secret.sh 'azure'
+      env:
+        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+    - name: Install Dependency
+      run: |
+        pip3 install --upgrade setuptools
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+        pip3 install avro-python3 kafka-python
+        pip3 install protobuf
+        pip3 install --upgrade snowflake-connector-python
+        curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+        sudo apt-get -y install jq
+        sudo apt-get -y install protobuf-compiler
+    
+    - name: Build with Unit Test
+      env:
+        JACOCO_COVERAGE: true
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        SHELL: "/bin/bash"
+      run: |
+        cd test
+        ./build_apache.sh ../../snowflake-kafka-connector package
+
+    - name: End to End Test of Apache Plarform 2.5
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_apache.sh 2.5.0 ./apache_properties
diff --git a/.github/workflows/End2EndTestConfluentAws.yml b/.github/workflows/End2EndTestConfluentAws.yml
@@ -0,0 +1,57 @@
+name: Kafka Connector Confluent End2End Test Aws
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+#on:
+#  push:
+#    branches: [ master ]
+#  pull_request:
+#    branches: '**'
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: "Install Java 8"
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Install Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+        architecture: 'x64'
+    - name: Decrypt profile.json
+      run: ./.github/scripts/decrypt_secret.sh 'aws'
+      env:
+        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+    - name: Install Dependency
+      run: |
+        pip3 install --upgrade setuptools
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]>=1.5.0"
+        pip3 install avro-python3 kafka-python
+        pip3 install protobuf
+        pip3 install --upgrade snowflake-connector-python
+        curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+        sudo apt-get -y install jq
+        sudo apt-get -y install protobuf-compiler
+
+    - name: Build with Unit Test
+      env:
+        JACOCO_COVERAGE: true
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        SHELL: "/bin/bash"
+      run: |
+        cd test
+        ./build_apache.sh ../../snowflake-kafka-connector package
+
+    - name: End to End Test of Confluent Platform 5.5
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test
+        ./run_test_confluent.sh 5.5.0 ./apache_properties