Merge branch 'master' into patch-2

LeoTheGriff · Nov 20, 2023 · 14760a7 · 14760a7
2 parents cd4e985 + 9dae0d3
commit 14760a7
Show file tree

Hide file tree

Showing 575 changed files with 15,321 additions and 8,389 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -38,6 +38,9 @@
 /airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources
 
 # Java-based destination connectors
+airbyte-cdk/java/airbyte-cdk/db-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/s3-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
 /airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
 /airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
 /airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations

diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml
@@ -69,6 +69,11 @@ inputs:
   s3_build_cache_secret_key:
     description: "Gradle S3 Build Cache AWS secret key"
     required: false
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+
 runs:
   using: "composite"
   steps:
@@ -89,22 +94,18 @@ runs:
       id: get-start-timestamp
       shell: bash
       run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
-    - name: Install Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-        token: ${{ inputs.github_token }}
-    - name: Install ci-connector-ops package
+    - name: Install airbyte-ci binary
+      id: install-airbyte-ci
       shell: bash
       run: |
-        pip install pipx
-        pipx ensurepath
-        pipx install airbyte-ci/connectors/pipelines/
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
     - name: Run airbyte-ci
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"

diff --git a/.github/workflows/airbyte-ci-release-experiment.yml b/.github/workflows/airbyte-ci-release-experiment.yml
diff --git a/.github/workflows/airbyte-ci-release.yml b/.github/workflows/airbyte-ci-release.yml
@@ -0,0 +1,128 @@
+name: Connector Ops CI - Airbyte CI Release
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths:
+      - "airbyte-ci/connectors/pipelines/**"
+  workflow_dispatch:
+
+env:
+  DEV_GCS_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service
+  PROD_GCS_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service
+  BINARY_FILE_NAME: airbyte-ci
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu", "macos"]
+
+    steps:
+      - name: Checkout Airbyte
+        id: checkout_airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.sha }} # This is required to make sure that the same commit is checked out on all runners
+
+      - name: Get short SHA
+        id: get_short_sha
+        uses: benjlevesque/[email protected]
+
+      - name: Install Python
+        id: install_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        id: install_poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Dependencies
+        id: install_dependencies
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry install --with dev
+
+      - name: Build release
+        id: build_release
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }}
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+
+      - name: Authenticate to Google Cloud Dev
+        id: auth_dev
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}"
+
+      - name: Upload pre-release to GCS dev bucket
+        id: upload_pre_release_to_gcs
+        if: github.ref != 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print pre-release public url
+        id: print_pre_release_public_url
+        run: |
+          echo "https://storage.googleapis.com/${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}/${{ env.BINARY_FILE_NAME }}"
+
+      # if master, upload per version and latest to prod bucket
+
+      - name: Set version from poetry version --short
+        id: set_version
+        if: github.ref == 'refs/heads/master'
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: |
+          echo "::set-output name=version::$(poetry version --short)"
+
+      - name: Authenticate to Google Cloud Prod
+        id: auth_prod
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}"
+
+      - name: Upload version release to GCS prod bucket
+        id: upload_version_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print release version public url
+        id: print_version_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}/${{ env.BINARY_FILE_NAME }}"
+
+      - name: Upload latest release to GCS prod bucket
+        id: upload_latest_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print latest release public url
+        id: print_latest_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest/${{ env.BINARY_FILE_NAME }}"
diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml
@@ -13,8 +13,9 @@ on:
       - synchronize
 jobs:
   run-airbyte-ci-tests:
+    # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow
     name: Run Airbyte CI tests
-    runs-on: "conn-prod-xlarge-runner"
+    runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml
@@ -0,0 +1,50 @@
+name: Approve and Merge Demo Command Dispatch
+
+# Note: We have a two stage dispatch so that we can wait for the formatters to run before approving and merging.
+on:
+  repository_dispatch:
+    types: [approve-and-merge-demo-command]
+
+jobs:
+  checkFormat:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for formatters to succeed
+        id: wait-for-formatters
+        uses: lewagon/[email protected]
+        with:
+          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+          check-name: "Apply All Formatting Rules"
+          repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          wait-interval: 30
+      - name: Comment if formatters failed
+        if: failure()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
+          body: |
+            > Error: Formatters failed. Ensure formatting is passing before using approve-and-merge.
+
+  approveAndMergeDispatch:
+    runs-on: ubuntu-latest
+    needs: [checkFormat]
+    steps:
+      - name: Auto Approve Slash Command Dispatch
+        uses: peter-evans/slash-command-dispatch@v3
+        id: scd
+        with:
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          permission: write
+          issue-type: pull-request
+          repository: airbytehq/airbyte-cloud
+          dispatch-type: repository
+          commands: |
+            approve-and-merge
+
+      - name: Edit comment with error message
+        if: steps.scd.outputs.error-message
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
+          body: |
+            > Error: ${{ steps.scd.outputs.error-message }}
diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   run-cat-unit-tests:
     name: Run CAT unit tests
-    runs-on: "conn-prod-xlarge-runner"
+    runs-on: "ci-runner-connector-test-large-dagger-0-6-4"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml
@@ -137,7 +137,7 @@ jobs:
         id: regex
         uses: AsasInnab/regex-action@v1
         with:
-          regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$"
+          regex_pattern: "^(connectors/)?[a-zA-Z0-9-_]+$"
           regex_flags: "i" # required to be set for this plugin
           search_string: ${{ inputs.connector }}
       - name: Validate input workflow format
@@ -179,6 +179,8 @@ jobs:
         run: |
           export PATH="$PATH:/root/.local/bin"
           ci_credentials connectors-performance/$HARNESS_TYPE write-to-storage
+          connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,')
+          ci_credentials connectors-performance/$connector_name write-to-storage
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }}
@@ -195,7 +197,7 @@ jobs:
         run: |
           echo "Building... ${{inputs.connector}}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line
-          connector_name=$(echo ${{ inputs.connector }} | cut -d / -f 2)
+          connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,')
           echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check"
           ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check
         env:

diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml
@@ -8,19 +8,19 @@ on:
     inputs:
       runs-on:
         type: string
-        default: conn-nightly-xlarge-runner
+        default: ci-runner-connector-nightly-xlarge-dagger-0-6-4
         required: true
       test-connectors-options:
         default: --concurrency=5 --support-level=certified
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
     timeout-minutes: 720 # 12 hours
-    runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml
@@ -19,7 +19,7 @@ on:
         default: "--modified"
       runner:
         description: "The runner to use for this job"
-        default: "conn-prod-xlarge-runner"
+        default: "ci-runner-connector-test-large-dagger-0-6-4"
   pull_request:
     types:
       - opened
@@ -29,7 +29,7 @@ jobs:
   connectors_ci:
     name: Connectors CI
     timeout-minutes: 1440 # 24 hours
-    runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}}
+    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-6-4'}}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml
@@ -8,19 +8,19 @@ on:
     inputs:
       runs-on:
         type: string
-        default: conn-nightly-xlarge-runner
+        default: ci-runner-connector-nightly-xlarge-dagger-0-6-4
         required: true
       test-connectors-options:
         default: --concurrency=3 --support-level=community
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}"
     timeout-minutes: 8640 # 6 days
-    runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}
+    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3