From a3100ef1beb618305b4957118e2c51a8f6350940 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:50:39 +0100 Subject: [PATCH 01/38] start --- .../connectors/destination-aws-datalake/Dockerfile | 2 +- docs/integrations/destinations/aws-datalake.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index ebe2844b7244..a036661f7da7 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index 0fa59745a562..a788b8e2b95c 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -99,5 +99,6 @@ following information to configure the destination: ## Changelog +| 0.1.2 | 2022-06-16 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | | 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | \ No newline at end of file From 7a3958f7d18f9710610a1c7a19c4452e4171a896 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:54:25 +0100 Subject: [PATCH 02/38] revert --- .../connectors/destination-aws-datalake/Dockerfile | 2 +- docs/integrations/destinations/aws-datalake.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index a036661f7da7..ebe2844b7244 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index a788b8e2b95c..0fa59745a562 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -99,6 +99,5 @@ following information to configure the destination: ## Changelog -| 0.1.2 | 2022-06-16 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | | 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | \ No newline at end of file From 4575f36b0baeb665d0a236ba3bc726ad4003b32b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:55:52 +0100 Subject: [PATCH 03/38] azblob --- .../connectors/destination-azure-blob-storage/Dockerfile | 2 +- docs/integrations/destinations/azureblobstorage.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile b/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile index f0c88d716a67..95c65d89053d 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-azure-blob-storage COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-azure-blob-storage diff --git a/docs/integrations/destinations/azureblobstorage.md b/docs/integrations/destinations/azureblobstorage.md index 45e083acb083..3a6753bd9856 100644 --- a/docs/integrations/destinations/azureblobstorage.md +++ b/docs/integrations/destinations/azureblobstorage.md @@ -137,6 +137,7 @@ They will be like this in the output file: | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.4 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | 2022-01-20 | [\#9682](https://github.com/airbytehq/airbyte/pull/9682) | Each data synchronization for each stream is written to a new blob to the folder with stream name. | From 9028a585f35ef32acbaf4a5fb0f25fda0cb96b88 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:57:08 +0100 Subject: [PATCH 04/38] bq --- airbyte-integrations/connectors/destination-bigquery/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index db02ad580dba..9d02050874a2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.8 +LABEL io.airbyte.version=1.1.9 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index b655d3a6b439..401232de4079 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | | 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | From 892037b565eb59de9c47354c489db58b437c259f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:58:32 +0100 Subject: [PATCH 05/38] bq denorm --- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index ec8026955d27..832a44505bff 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.8 +LABEL io.airbyte.version=1.1.9 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 401232de4079..0ce737895f55 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -172,6 +172,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | | 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | From 1ef7488b7cc5bb6666195ea2f39a0f513487bd56 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:00:43 +0100 Subject: [PATCH 06/38] megapublish baaaabyyyy --- .github/workflows/publish-command.yml | 34 ++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 60bf4453586a..6109fc8e93a5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -65,9 +65,27 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + preprocess-matrix: + needs: start-publish-image-runner + runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + outputs: + connectorjson: ${{ steps.preprocess.outputs.connectorjson }} + steps: + - id: preprocess + run: | + start="[\"" + replace="\",\"" + end="\"]" + stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')" + middle=${stripped_connector//,/$replace} + full="$start$middle$end" + echo "::set-output name=connectorjson::$full" publish-image: timeout-minutes: 240 - needs: start-publish-image-runner + needs: preprocess-matrix + strategy: + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} runs-on: ${{ needs.start-publish-image-runner.outputs.label }} environment: more-secrets steps: @@ -89,7 +107,7 @@ jobs: with: regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ matrix.connector }} - name: Validate input workflow format if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 @@ -126,18 +144,18 @@ jobs: source venv/bin/activate tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | source venv/bin/activate - ci_credentials ${{ github.event.inputs.connector }} + ci_credentials ${{ matrix.connector }} env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars if: startsWith(github.event.inputs.connector, 'connectors') run: | source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry if: startsWith(github.event.inputs.connector, 'connectors') @@ -151,10 +169,10 @@ jobs: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ github.event.inputs.connector }} + - name: Publish ${{ matrix.connector }} run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache id: publish env: DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} From c3cf966ac19fb1df9167a4a14ce68122c82e3931 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:15:18 +0100 Subject: [PATCH 07/38] fix needs --- .github/workflows/publish-command.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 6109fc8e93a5..780ca315fc11 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -82,7 +82,9 @@ jobs: echo "::set-output name=connectorjson::$full" publish-image: timeout-minutes: 240 - needs: preprocess-matrix + needs: + - start-publish-image-runner + - preprocess-matrix strategy: matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} From 2c05ac5008d6622d657d3b89584e9aea72d91e0b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:23:22 +0100 Subject: [PATCH 08/38] matrix connectors --- .github/workflows/publish-command.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 780ca315fc11..05e7aa30e99e 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -97,7 +97,7 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -111,7 +111,7 @@ jobs: regex_flags: "i" # required to be set for this plugin search_string: ${{ matrix.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != matrix.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v2 @@ -153,18 +153,18 @@ jobs: env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | source tools/lib/lib.sh DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | curl -sL https://sentry.io/get-cli/ | bash - name: Create Sentry Release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing env: @@ -182,7 +182,7 @@ jobs: # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - name: Finalize Sentry release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" env: @@ -195,14 +195,14 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :rocket: Successfully published ${{github.event.inputs.connector}} + > :rocket: Successfully published ${{matrix.connector}} - name: Add Published Failure Comment if: github.event.inputs.comment-id && !success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Failed to publish ${{github.event.inputs.connector}} + > :x: Failed to publish ${{matrix.connector}} - name: Check if connector in definitions yaml if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -246,21 +246,21 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :rocket: Auto-bumped version for ${{github.event.inputs.connector}} + > :rocket: Auto-bumped version for ${{matrix.connector}} - name: Add Version Bump Failure Comment if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}} + > :x: Couldn't auto-bump version for ${{matrix.connector}} - name: Add Final Success Comment if: github.event.inputs.comment-id && success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner: name: Stop Build EC2 Runner From 981cb6ac2a25605a4340ec07304eef631dd8cc37 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 16 Jun 2022 22:39:10 +0000 Subject: [PATCH 09/38] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 19231af7cb1a..ea6afc10a8b6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -1,7 +1,7 @@ - name: Azure Blob Storage destinationDefinitionId: b4c5d105-31fd-4817-96b6-cb923bfc04cb dockerRepository: airbyte/destination-azure-blob-storage - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/azureblobstorage icon: azureblobstorage.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index ec97b237f846..81d82333f41e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1,7 +1,7 @@ # This file is generated by io.airbyte.config.specs.SeedConnectorSpecGenerator. # Do NOT edit this file directly. See generator class for more details. --- -- dockerImage: "airbyte/destination-azure-blob-storage:0.1.4" +- dockerImage: "airbyte/destination-azure-blob-storage:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/azureblobstorage" connectionSpecification: From 5181fa050941881ed1329e56891ff63b15dd4884 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:47:19 +0100 Subject: [PATCH 10/38] dont failfast and max parallel 5 --- .github/workflows/publish-command.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 05e7aa30e99e..d882486cec3f 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -86,6 +86,8 @@ jobs: - start-publish-image-runner - preprocess-matrix strategy: + max-parallel: 5 + fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} runs-on: ${{ needs.start-publish-image-runner.outputs.label }} From 4e3575bd989553d980987f006e39f62ef6c6d10b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:12:54 +0100 Subject: [PATCH 11/38] multi runno --- .github/workflows/publish-command.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index d882486cec3f..2d9306fbb834 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -90,7 +90,9 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + runs-on: + - ${{ needs.start-publish-image-runner.outputs.label }} + - ubuntu-latest environment: more-secrets steps: - name: Link comment to workflow run From ba45af5f1237cc3824906bcb91ff323242341316 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:40:16 +0100 Subject: [PATCH 12/38] minor --- .github/workflows/publish-command.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 2d9306fbb834..e38652863358 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -24,6 +24,10 @@ on: description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" required: true default: "true" + parallel: + description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" + required: true + default: "false" jobs: find_valid_pat: @@ -90,9 +94,7 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: - - ${{ needs.start-publish-image-runner.outputs.label }} - - ubuntu-latest + runs-on: ${{ needs.start-publish-image-runner.outputs.label }} environment: more-secrets steps: - name: Link comment to workflow run From b9a993944d3b59130614888d47098832385367e0 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:46:26 +0100 Subject: [PATCH 13/38] testing matrix agents --- .github/workflows/TEMP-testing-command.yml | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/TEMP-testing-command.yml diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml new file mode 100644 index 000000000000..b0440673044b --- /dev/null +++ b/.github/workflows/TEMP-testing-command.yml @@ -0,0 +1,114 @@ +name: Publish Connector Image +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + run-tests: + description: "Should run tests when publishing" + required: true + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + auto-bump-version: + description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" + required: true + default: "true" + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-publish-image-runner: + name: Start Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWCRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + ec2-instance-id: ${{ matrix.ec2-instance-id }} + + stop-publish-image-runner: + name: Stop Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] + needs: + - start-publish-image-runner # required to get output from the start-runner job + # - publish-image # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner.outputs.label }} + ec2-instance-id: ${{ matrix.ec2-instance-id }} From 93a769547badcb63bf8d7a038082b13c869e0431 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:47:24 +0100 Subject: [PATCH 14/38] name --- .github/workflows/TEMP-testing-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml index b0440673044b..d931ef501ee8 100644 --- a/.github/workflows/TEMP-testing-command.yml +++ b/.github/workflows/TEMP-testing-command.yml @@ -1,4 +1,4 @@ -name: Publish Connector Image +name: TEMP testing command on: workflow_dispatch: inputs: From 4169452c08a1c2304717893931ec35cda9fb4165 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:51:40 +0100 Subject: [PATCH 15/38] testing multi agents --- .github/workflows/publish-command.yml | 394 ++++++++++++++------------ 1 file changed, 209 insertions(+), 185 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index e38652863358..fba4c7c1d174 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -51,11 +51,22 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-publish-image-runner: name: Start Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] runs-on: ubuntu-latest needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -69,6 +80,8 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + ec2-instance-id: ${{ matrix.ec2-instance-id }} preprocess-matrix: needs: start-publish-image-runner runs-on: ${{ needs.start-publish-image-runner.outputs.label }} @@ -84,189 +97,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - publish-image: - timeout-minutes: 240 - needs: - - start-publish-image-runner - - preprocess-matrix - strategy: - max-parallel: 5 - fail-fast: false - matrix: - connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} - environment: more-secrets - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ matrix.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != matrix.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - token: ${{ secrets.OCTAVIA_PAT }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: "17" - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Install Pyenv and Tox - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - pip install --quiet tox==3.24.4 - - name: Install yq - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo apt update - sudo apt install yq -y - - name: Test and install CI scripts - # all CI python packages have the prefix "ci_" - run: | - source venv/bin/activate - tox -r -c ./tools/tox_ci.ini - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ matrix.connector }} - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Set Name and Version Environment Vars - if: startsWith(matrix.connector, 'connectors') - run: | - source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - - name: Prepare Sentry - if: startsWith(matrix.connector, 'connectors') - run: | - curl -sL https://sentry.io/get-cli/ | bash - - name: Create Sentry Release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ matrix.connector }} - run: | - echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - - name: Finalize Sentry release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - - name: Check if connector in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - then exit 1 - fi - - name: Bump version in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - if [ -z "$sourcename" ] - then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - fi - - name: Run gradle process changes - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - ./gradlew :airbyte-config:init:processResources - - name: git config - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git config user.name 'Octavia Squidington III' - git config user.email 'octavia-squidington-iii@users.noreply.github.com' - - name: git commit and push - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git add -u - git commit -m "auto-bump connector version" - git pull origin ${{ github.event.inputs.gitref }} - git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # publish-image: + # timeout-minutes: 240 + # needs: + # - start-publish-image-runner + # - preprocess-matrix + # strategy: + # max-parallel: 5 + # fail-fast: false + # matrix: + # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + # runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + # environment: more-secrets + # steps: + # - name: Link comment to workflow run + # if: github.event.inputs.comment-id + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set up Cloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + # export_default_credentials: true + # - name: Search for valid connector name format + # id: regex + # uses: AsasInnab/regex-action@v1 + # with: + # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + # regex_flags: "i" # required to be set for this plugin + # search_string: ${{ matrix.connector }} + # - name: Validate input workflow format + # if: steps.regex.outputs.first_match != matrix.connector + # run: echo "The connector provided has an invalid format!" && exit 1 + # - name: Checkout Airbyte + # uses: actions/checkout@v2 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # token: ${{ secrets.OCTAVIA_PAT }} + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v2 + # with: + # python-version: "3.9" + # - name: Install Pyenv and Tox + # run: | + # python3 -m pip install --quiet virtualenv==16.7.9 --user + # python3 -m virtualenv venv + # source venv/bin/activate + # pip install --quiet tox==3.24.4 + # - name: Install yq + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + # sudo add-apt-repository ppa:rmescandon/yq + # sudo apt update + # sudo apt install yq -y + # - name: Test and install CI scripts + # # all CI python packages have the prefix "ci_" + # run: | + # source venv/bin/activate + # tox -r -c ./tools/tox_ci.ini + # pip install --quiet -e ./tools/ci_* + # - name: Write Integration Test Credentials for ${{ matrix.connector }} + # run: | + # source venv/bin/activate + # ci_credentials ${{ matrix.connector }} + # env: + # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # - name: Set Name and Version Environment Vars + # if: startsWith(matrix.connector, 'connectors') + # run: | + # source tools/lib/lib.sh + # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + # - name: Prepare Sentry + # if: startsWith(matrix.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Publish ${{ matrix.connector }} + # run: | + # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # - name: Finalize Sentry release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Add Published Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully published ${{matrix.connector}} + # - name: Add Published Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to publish ${{matrix.connector}} + # - name: Check if connector in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + # then exit 1 + # fi + # - name: Bump version in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + # if [ -z "$sourcename" ] + # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + # fi + # - name: Run gradle process changes + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # ./gradlew :airbyte-config:init:processResources + # - name: git config + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git config user.name 'Octavia Squidington III' + # git config user.email 'octavia-squidington-iii@users.noreply.github.com' + # - name: git commit and push + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git add -u + # git commit -m "auto-bump connector version" + # git pull origin ${{ github.event.inputs.gitref }} + # git push origin ${{ github.event.inputs.gitref }} + # - name: Add Version Bump Success Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Auto-bumped version for ${{matrix.connector}} + # - name: Add Version Bump Failure Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Couldn't auto-bump version for ${{matrix.connector}} + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner: name: Stop Build EC2 Runner @@ -274,6 +287,17 @@ jobs: - start-publish-image-runner # required to get output from the start-runner job - publish-image # required to wait when the main job is done - find_valid_pat + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -289,4 +313,4 @@ jobs: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }} + ec2-instance-id: ${{ matrix.ec2-instance-id }} From ec764903a5bf19f8eaab03f43c4fba094d8e03d2 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:53:56 +0100 Subject: [PATCH 16/38] tmp fix --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index fba4c7c1d174..3a80e5618089 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -285,7 +285,7 @@ jobs: name: Stop Build EC2 Runner needs: - start-publish-image-runner # required to get output from the start-runner job - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From 2da540a3e7945a58095a244110732dc75f4e77a7 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:11:56 +0100 Subject: [PATCH 17/38] new multi agents --- .github/workflows/publish-command.yml | 116 +++++++++++++++++++++----- 1 file changed, 96 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 3a80e5618089..8877e27222cc 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -49,24 +49,101 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] + start-publish-image-runner-0: + name: Start Build EC2 Runner 0 runs-on: ubuntu-latest needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} - # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-1: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 1 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-2: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 2 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-3: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 3 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-4: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 4 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -81,7 +158,6 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner - ec2-instance-id: ${{ matrix.ec2-instance-id }} preprocess-matrix: needs: start-publish-image-runner runs-on: ${{ needs.start-publish-image-runner.outputs.label }} @@ -292,11 +368,11 @@ jobs: matrix: ec2-instance-id: [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + "${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", ] runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs From 5f0c38b53db089d79bcf3322b27bf96c6e986bfe Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:18:09 +0100 Subject: [PATCH 18/38] multi test --- .github/workflows/publish-command.yml | 37 +++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 8877e27222cc..98804ef50bdf 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -357,10 +357,39 @@ jobs: # body: | # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. - stop-publish-image-runner: + stop-publish-image-runner-0: + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs name: Stop Build EC2 Runner needs: - - start-publish-image-runner # required to get output from the start-runner job + - start-publish-image-runner-0 # required to get output from the start-runner job + - preprocess-matrix + # - publish-image # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} + stop-publish-image-runner-multi: + if: github.event.inputs.parallel == 'true' + name: Stop Build EC2 Runner + needs: + - start-publish-image-runner-0 + - start-publish-image-runner-1 + - start-publish-image-runner-2 + - start-publish-image-runner-3 + - start-publish-image-runner-4 + - preprocess-matrix # - publish-image # required to wait when the main job is done - find_valid_pat strategy: @@ -368,14 +397,12 @@ jobs: matrix: ec2-instance-id: [ - "${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", ] runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -388,5 +415,5 @@ jobs: with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ matrix.ec2-instance-id }} From 3849459f393111bf98dbf96e5d76351a912f806f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:21:39 +0100 Subject: [PATCH 19/38] tryy --- .github/workflows/publish-command.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 98804ef50bdf..454991227c39 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -159,8 +159,8 @@ jobs: github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner preprocess-matrix: - needs: start-publish-image-runner - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} outputs: connectorjson: ${{ steps.preprocess.outputs.connectorjson }} steps: @@ -176,14 +176,14 @@ jobs: # publish-image: # timeout-minutes: 240 # needs: - # - start-publish-image-runner + # - start-publish-image-runner-0 # - preprocess-matrix # strategy: # max-parallel: 5 # fail-fast: false # matrix: # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + # runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} # environment: more-secrets # steps: # - name: Link comment to workflow run From 2a58e3d7be95b5122f81fc9963d9d5e825c9949b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:30:25 +0100 Subject: [PATCH 20/38] let's do this --- .github/workflows/publish-command.yml | 370 +++++++++++++------------- 1 file changed, 185 insertions(+), 185 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 454991227c39..b59121070189 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -173,189 +173,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - # publish-image: - # timeout-minutes: 240 - # needs: - # - start-publish-image-runner-0 - # - preprocess-matrix - # strategy: - # max-parallel: 5 - # fail-fast: false - # matrix: - # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - # environment: more-secrets - # steps: - # - name: Link comment to workflow run - # if: github.event.inputs.comment-id - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - # - name: Set up Cloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - # export_default_credentials: true - # - name: Search for valid connector name format - # id: regex - # uses: AsasInnab/regex-action@v1 - # with: - # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - # regex_flags: "i" # required to be set for this plugin - # search_string: ${{ matrix.connector }} - # - name: Validate input workflow format - # if: steps.regex.outputs.first_match != matrix.connector - # run: echo "The connector provided has an invalid format!" && exit 1 - # - name: Checkout Airbyte - # uses: actions/checkout@v2 - # with: - # repository: ${{ github.event.inputs.repo }} - # ref: ${{ github.event.inputs.gitref }} - # token: ${{ secrets.OCTAVIA_PAT }} - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: "17" - # - name: Install Python - # uses: actions/setup-python@v2 - # with: - # python-version: "3.9" - # - name: Install Pyenv and Tox - # run: | - # python3 -m pip install --quiet virtualenv==16.7.9 --user - # python3 -m virtualenv venv - # source venv/bin/activate - # pip install --quiet tox==3.24.4 - # - name: Install yq - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - # sudo add-apt-repository ppa:rmescandon/yq - # sudo apt update - # sudo apt install yq -y - # - name: Test and install CI scripts - # # all CI python packages have the prefix "ci_" - # run: | - # source venv/bin/activate - # tox -r -c ./tools/tox_ci.ini - # pip install --quiet -e ./tools/ci_* - # - name: Write Integration Test Credentials for ${{ matrix.connector }} - # run: | - # source venv/bin/activate - # ci_credentials ${{ matrix.connector }} - # env: - # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - # - name: Set Name and Version Environment Vars - # if: startsWith(matrix.connector, 'connectors') - # run: | - # source tools/lib/lib.sh - # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - # - name: Prepare Sentry - # if: startsWith(matrix.connector, 'connectors') - # run: | - # curl -sL https://sentry.io/get-cli/ | bash - # - name: Create Sentry Release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Publish ${{ matrix.connector }} - # run: | - # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - # id: publish - # env: - # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - # TZ: UTC - # - name: Finalize Sentry release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Add Published Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Successfully published ${{matrix.connector}} - # - name: Add Published Failure Comment - # if: github.event.inputs.comment-id && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Failed to publish ${{matrix.connector}} - # - name: Check if connector in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - # then exit 1 - # fi - # - name: Bump version in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - # if [ -z "$sourcename" ] - # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - # fi - # - name: Run gradle process changes - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # ./gradlew :airbyte-config:init:processResources - # - name: git config - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git config user.name 'Octavia Squidington III' - # git config user.email 'octavia-squidington-iii@users.noreply.github.com' - # - name: git commit and push - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git add -u - # git commit -m "auto-bump connector version" - # git pull origin ${{ github.event.inputs.gitref }} - # git push origin ${{ github.event.inputs.gitref }} - # - name: Add Version Bump Success Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Auto-bumped version for ${{matrix.connector}} - # - name: Add Version Bump Failure Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Couldn't auto-bump version for ${{matrix.connector}} - # - name: Add Final Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + environment: more-secrets + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ matrix.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != matrix.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ matrix.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ matrix.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Set Name and Version Environment Vars + if: startsWith(matrix.connector, 'connectors') + run: | + source tools/lib/lib.sh + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + - name: Prepare Sentry + if: startsWith(matrix.connector, 'connectors') + run: | + curl -sL https://sentry.io/get-cli/ | bash + - name: Create Sentry Release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Publish ${{ matrix.connector }} + run: | + echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + id: publish + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + - name: Finalize Sentry release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Add Published Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Successfully published ${{matrix.connector}} + - name: Add Published Failure Comment + if: github.event.inputs.comment-id && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Failed to publish ${{matrix.connector}} + - name: Check if connector in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + then exit 1 + fi + - name: Bump version in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + if [ -z "$sourcename" ] + then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + fi + - name: Run gradle process changes + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + ./gradlew :airbyte-config:init:processResources + - name: git config + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + - name: git commit and push + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git add -u + git commit -m "auto-bump connector version" + git pull origin ${{ github.event.inputs.gitref }} + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Auto-bumped version for ${{matrix.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't auto-bump version for ${{matrix.connector}} + - name: Add Final Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -363,7 +363,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -390,7 +390,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From db58ff42e69d06696e1c2193102b46258fe8c89f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:08:06 +0100 Subject: [PATCH 21/38] magico --- .github/workflows/publish-command.yml | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index b59121070189..1c74f63d4373 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -183,7 +183,7 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + runs-on: self-hosted environment: more-secrets steps: - name: Link comment to workflow run @@ -381,7 +381,7 @@ jobs: label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} stop-publish-image-runner-multi: - if: github.event.inputs.parallel == 'true' + if: ${{ always() && github.event.inputs.parallel == 'true' }} name: Stop Build EC2 Runner needs: - start-publish-image-runner-0 @@ -395,12 +395,24 @@ jobs: strategy: fail-fast: false matrix: - ec2-instance-id: + ec2-instance: [ - "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, ] runs-on: ubuntu-latest steps: @@ -415,5 +427,5 @@ jobs: with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner-0.outputs.label }} - ec2-instance-id: ${{ matrix.ec2-instance-id }} + label: ${{ matrix.ec2-instance.label }} + ec2-instance-id: ${{ matrix.ec2-instance.id }} From d1cf7f20fd45b4a64c4b893c04902f0204de50f5 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:28:26 +0100 Subject: [PATCH 22/38] fix --- .github/workflows/publish-command.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 1c74f63d4373..f5dafa16a39b 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -402,16 +402,16 @@ jobs: "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-2.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-3.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-4.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", }, ] runs-on: ubuntu-latest From 5232e2185718a87d63bfc479b548e6d662db8b26 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:34:54 +0100 Subject: [PATCH 23/38] label test --- .github/workflows/publish-command.yml | 373 +++++++++++++------------- 1 file changed, 190 insertions(+), 183 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index f5dafa16a39b..12f73a5799a4 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -158,6 +158,13 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner + manual-label-test: + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - id: label + run: | + ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} @@ -173,189 +180,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - publish-image: - timeout-minutes: 240 - needs: - - start-publish-image-runner-0 - - preprocess-matrix - strategy: - max-parallel: 5 - fail-fast: false - matrix: - connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: self-hosted - environment: more-secrets - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ matrix.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != matrix.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - token: ${{ secrets.OCTAVIA_PAT }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: "17" - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Install Pyenv and Tox - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - pip install --quiet tox==3.24.4 - - name: Install yq - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo apt update - sudo apt install yq -y - - name: Test and install CI scripts - # all CI python packages have the prefix "ci_" - run: | - source venv/bin/activate - tox -r -c ./tools/tox_ci.ini - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ matrix.connector }} - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Set Name and Version Environment Vars - if: startsWith(matrix.connector, 'connectors') - run: | - source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - - name: Prepare Sentry - if: startsWith(matrix.connector, 'connectors') - run: | - curl -sL https://sentry.io/get-cli/ | bash - - name: Create Sentry Release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ matrix.connector }} - run: | - echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - - name: Finalize Sentry release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - - name: Check if connector in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - then exit 1 - fi - - name: Bump version in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - if [ -z "$sourcename" ] - then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - fi - - name: Run gradle process changes - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - ./gradlew :airbyte-config:init:processResources - - name: git config - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git config user.name 'Octavia Squidington III' - git config user.email 'octavia-squidington-iii@users.noreply.github.com' - - name: git commit and push - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git add -u - git commit -m "auto-bump connector version" - git pull origin ${{ github.event.inputs.gitref }} - git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # publish-image: + # timeout-minutes: 240 + # needs: + # - start-publish-image-runner-0 + # - preprocess-matrix + # strategy: + # max-parallel: 5 + # fail-fast: false + # matrix: + # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + # runs-on: self-hosted + # environment: more-secrets + # steps: + # - name: Link comment to workflow run + # if: github.event.inputs.comment-id + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set up Cloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + # export_default_credentials: true + # - name: Search for valid connector name format + # id: regex + # uses: AsasInnab/regex-action@v1 + # with: + # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + # regex_flags: "i" # required to be set for this plugin + # search_string: ${{ matrix.connector }} + # - name: Validate input workflow format + # if: steps.regex.outputs.first_match != matrix.connector + # run: echo "The connector provided has an invalid format!" && exit 1 + # - name: Checkout Airbyte + # uses: actions/checkout@v2 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # token: ${{ secrets.OCTAVIA_PAT }} + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v2 + # with: + # python-version: "3.9" + # - name: Install Pyenv and Tox + # run: | + # python3 -m pip install --quiet virtualenv==16.7.9 --user + # python3 -m virtualenv venv + # source venv/bin/activate + # pip install --quiet tox==3.24.4 + # - name: Install yq + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + # sudo add-apt-repository ppa:rmescandon/yq + # sudo apt update + # sudo apt install yq -y + # - name: Test and install CI scripts + # # all CI python packages have the prefix "ci_" + # run: | + # source venv/bin/activate + # tox -r -c ./tools/tox_ci.ini + # pip install --quiet -e ./tools/ci_* + # - name: Write Integration Test Credentials for ${{ matrix.connector }} + # run: | + # source venv/bin/activate + # ci_credentials ${{ matrix.connector }} + # env: + # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # - name: Set Name and Version Environment Vars + # if: startsWith(matrix.connector, 'connectors') + # run: | + # source tools/lib/lib.sh + # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + # - name: Prepare Sentry + # if: startsWith(matrix.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Publish ${{ matrix.connector }} + # run: | + # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # - name: Finalize Sentry release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Add Published Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully published ${{matrix.connector}} + # - name: Add Published Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to publish ${{matrix.connector}} + # - name: Check if connector in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + # then exit 1 + # fi + # - name: Bump version in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + # if [ -z "$sourcename" ] + # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + # fi + # - name: Run gradle process changes + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # ./gradlew :airbyte-config:init:processResources + # - name: git config + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git config user.name 'Octavia Squidington III' + # git config user.email 'octavia-squidington-iii@users.noreply.github.com' + # - name: git commit and push + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git add -u + # git commit -m "auto-bump connector version" + # git pull origin ${{ github.event.inputs.gitref }} + # git push origin ${{ github.event.inputs.gitref }} + # - name: Add Version Bump Success Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Auto-bumped version for ${{matrix.connector}} + # - name: Add Version Bump Failure Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Couldn't auto-bump version for ${{matrix.connector}} + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs From 5d4a760085857d03c542d11c641e89d207316b66 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:48:44 +0100 Subject: [PATCH 24/38] couple more connector bumps --- .../connectors/destination-cassandra/Dockerfile | 2 +- .../connectors/destination-clickhouse-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- docs/integrations/destinations/clickhouse.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-cassandra/Dockerfile b/airbyte-integrations/connectors/destination-cassandra/Dockerfile index 2ee5c1d10a3b..5bb5b6b4dac1 100644 --- a/airbyte-integrations/connectors/destination-cassandra/Dockerfile +++ b/airbyte-integrations/connectors/destination-cassandra/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-cassandra COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-cassandra diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 2a88f7bf1248..8920d8d822e9 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index a909fe50fab8..fa443a93f73f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 4ad47c235121..2b3df385a8b2 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -79,6 +79,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:---------------------------------------------| +| 0.1.7 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | From 1cd9f993e1bfc36ed5d6f9b199dc70f87095e3fb Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:51:28 +0100 Subject: [PATCH 25/38] temp --- .github/workflows/publish-command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 12f73a5799a4..c3bd7b0e2fd4 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -370,7 +370,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -397,7 +397,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From 5b9ac81e035f9b71a6537e616ccd0a0f9a323ca1 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:02:56 +0100 Subject: [PATCH 26/38] things --- .github/workflows/publish-command.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index c3bd7b0e2fd4..3517c01957b1 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -161,10 +161,14 @@ jobs: manual-label-test: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + # ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL steps: - id: label run: | - ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL + ls + ls ../ + pwd + cwd preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From d43e431f2354f51323f9f622ab854addc946bf21 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:07:47 +0100 Subject: [PATCH 27/38] check this --- .github/workflows/publish-command.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 3517c01957b1..a67b96460535 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -165,10 +165,7 @@ jobs: steps: - id: label run: | - ls - ls ../ - pwd - cwd + ls /actions-runner/ preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From 2a760fa89fe374e9367ab56ef2574f93e00f688b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:13:47 +0100 Subject: [PATCH 28/38] lets gooo --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index a67b96460535..c48b3b7e791c 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -165,7 +165,7 @@ jobs: steps: - id: label run: | - ls /actions-runner/ + /actions-runner/config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},$GITHUB_RUN_ID-publisher preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From 8a9e07454c5960f8b757fd4900c3aa7dc3327f66 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:01:07 +0100 Subject: [PATCH 29/38] more connectors --- .github/workflows/publish-command.yml | 383 +++++++++--------- .../connectors/destination-csv/Dockerfile | 2 +- .../destination-databricks/Dockerfile | 2 +- .../destination-dev-null/Dockerfile | 2 +- .../destination-dynamodb/Dockerfile | 2 +- docs/integrations/destinations/databricks.md | 1 + docs/integrations/destinations/dynamodb.md | 1 + 7 files changed, 191 insertions(+), 202 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index c48b3b7e791c..8dbfb7552ba3 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -69,7 +69,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-1: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 1 @@ -91,7 +90,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-2: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 2 @@ -113,7 +111,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-3: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 3 @@ -135,7 +132,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-4: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 4 @@ -157,15 +153,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner - manual-label-test: - needs: start-publish-image-runner-0 - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - # ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL - steps: - - id: label - run: | - /actions-runner/config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},$GITHUB_RUN_ID-publisher preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} @@ -181,189 +168,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - # publish-image: - # timeout-minutes: 240 - # needs: - # - start-publish-image-runner-0 - # - preprocess-matrix - # strategy: - # max-parallel: 5 - # fail-fast: false - # matrix: - # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: self-hosted - # environment: more-secrets - # steps: - # - name: Link comment to workflow run - # if: github.event.inputs.comment-id - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - # - name: Set up Cloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - # export_default_credentials: true - # - name: Search for valid connector name format - # id: regex - # uses: AsasInnab/regex-action@v1 - # with: - # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - # regex_flags: "i" # required to be set for this plugin - # search_string: ${{ matrix.connector }} - # - name: Validate input workflow format - # if: steps.regex.outputs.first_match != matrix.connector - # run: echo "The connector provided has an invalid format!" && exit 1 - # - name: Checkout Airbyte - # uses: actions/checkout@v2 - # with: - # repository: ${{ github.event.inputs.repo }} - # ref: ${{ github.event.inputs.gitref }} - # token: ${{ secrets.OCTAVIA_PAT }} - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: "17" - # - name: Install Python - # uses: actions/setup-python@v2 - # with: - # python-version: "3.9" - # - name: Install Pyenv and Tox - # run: | - # python3 -m pip install --quiet virtualenv==16.7.9 --user - # python3 -m virtualenv venv - # source venv/bin/activate - # pip install --quiet tox==3.24.4 - # - name: Install yq - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - # sudo add-apt-repository ppa:rmescandon/yq - # sudo apt update - # sudo apt install yq -y - # - name: Test and install CI scripts - # # all CI python packages have the prefix "ci_" - # run: | - # source venv/bin/activate - # tox -r -c ./tools/tox_ci.ini - # pip install --quiet -e ./tools/ci_* - # - name: Write Integration Test Credentials for ${{ matrix.connector }} - # run: | - # source venv/bin/activate - # ci_credentials ${{ matrix.connector }} - # env: - # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - # - name: Set Name and Version Environment Vars - # if: startsWith(matrix.connector, 'connectors') - # run: | - # source tools/lib/lib.sh - # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - # - name: Prepare Sentry - # if: startsWith(matrix.connector, 'connectors') - # run: | - # curl -sL https://sentry.io/get-cli/ | bash - # - name: Create Sentry Release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Publish ${{ matrix.connector }} - # run: | - # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - # id: publish - # env: - # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - # TZ: UTC - # - name: Finalize Sentry release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Add Published Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Successfully published ${{matrix.connector}} - # - name: Add Published Failure Comment - # if: github.event.inputs.comment-id && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Failed to publish ${{matrix.connector}} - # - name: Check if connector in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - # then exit 1 - # fi - # - name: Bump version in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - # if [ -z "$sourcename" ] - # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - # fi - # - name: Run gradle process changes - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # ./gradlew :airbyte-config:init:processResources - # - name: git config - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git config user.name 'Octavia Squidington III' - # git config user.email 'octavia-squidington-iii@users.noreply.github.com' - # - name: git commit and push - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git add -u - # git commit -m "auto-bump connector version" - # git pull origin ${{ github.event.inputs.gitref }} - # git push origin ${{ github.event.inputs.gitref }} - # - name: Add Version Bump Success Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Auto-bumped version for ${{matrix.connector}} - # - name: Add Version Bump Failure Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Couldn't auto-bump version for ${{matrix.connector}} - # - name: Add Final Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + environment: more-secrets + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ matrix.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != matrix.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ matrix.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ matrix.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Set Name and Version Environment Vars + if: startsWith(matrix.connector, 'connectors') + run: | + source tools/lib/lib.sh + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + - name: Prepare Sentry + if: startsWith(matrix.connector, 'connectors') + run: | + curl -sL https://sentry.io/get-cli/ | bash + - name: Create Sentry Release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Publish ${{ matrix.connector }} + run: | + echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + id: publish + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + - name: Finalize Sentry release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Add Published Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Successfully published ${{matrix.connector}} + - name: Add Published Failure Comment + if: github.event.inputs.comment-id && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Failed to publish ${{matrix.connector}} + - name: Check if connector in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + then exit 1 + fi + - name: Bump version in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + if [ -z "$sourcename" ] + then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + fi + - name: Run gradle process changes + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + ./gradlew :airbyte-config:init:processResources + - name: git config + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + - name: git commit and push + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git add -u + git commit -m "auto-bump connector version" + git pull origin ${{ github.event.inputs.gitref }} + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Auto-bumped version for ${{matrix.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't auto-bump version for ${{matrix.connector}} + - name: Add Final Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -371,7 +358,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -398,7 +385,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false diff --git a/airbyte-integrations/connectors/destination-csv/Dockerfile b/airbyte-integrations/connectors/destination-csv/Dockerfile index 12791f0e238f..f9f3456bf150 100644 --- a/airbyte-integrations/connectors/destination-csv/Dockerfile +++ b/airbyte-integrations/connectors/destination-csv/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-csv COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/destination-csv diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile index edad5c72a657..ae5588e07ac4 100644 --- a/airbyte-integrations/connectors/destination-databricks/Dockerfile +++ b/airbyte-integrations/connectors/destination-databricks/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/destination-databricks diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile index 007170ec3fa5..e37fc860dd67 100644 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile index 487b94174678..5f80d086a636 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-dynamodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-dynamodb diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md index db105fd355bc..63635069f1af 100644 --- a/docs/integrations/destinations/databricks.md +++ b/docs/integrations/destinations/databricks.md @@ -104,6 +104,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.3 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.2.2 | 2022-06-13 | [\#13722](https://github.com/airbytehq/airbyte/pull/13722) | Rename to "Databricks Lakehouse". | | 0.2.1 | 2022-06-08 | [\#13630](https://github.com/airbytehq/airbyte/pull/13630) | Rename to "Databricks Delta Lake" and add field orders in the spec. | | 0.2.0 | 2022-05-15 | [\#12861](https://github.com/airbytehq/airbyte/pull/12861) | Use new public Databricks JDBC driver, and open source the connector. | diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index 2934fb01e205..727218d51386 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -58,6 +58,7 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.3 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.2 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.1 | 2022-12-05 | [\#9314](https://github.com/airbytehq/airbyte/pull/9314) | Rename dynamo_db_table_name to dynamo_db_table_name_prefix. | From e03adaa0a6bc0596ae098d925b2c957d3d42e6de Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Fri, 17 Jun 2022 10:06:26 -0400 Subject: [PATCH 30/38] Fix md and lg spacing scss vars (#13886) --- airbyte-webapp/src/scss/_variables.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/scss/_variables.scss b/airbyte-webapp/src/scss/_variables.scss index 4d1b7a6a7622..5d3267e3e276 100644 --- a/airbyte-webapp/src/scss/_variables.scss +++ b/airbyte-webapp/src/scss/_variables.scss @@ -5,8 +5,8 @@ $border-thick: 2px; $spacing-xs: 3px; $spacing-sm: 5px; -$spacing-m: 10px; -$spacing-l: 15px; +$spacing-md: 10px; +$spacing-lg: 15px; $spacing-xl: 20px; $spacing-2xl: 40px; $spacing-page-bottom: 150px; From 24344174164b6d15e0346fc29449a56e37f78244 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:17:38 +0100 Subject: [PATCH 31/38] Delete TEMP-testing-command.yml --- .github/workflows/TEMP-testing-command.yml | 114 --------------------- 1 file changed, 114 deletions(-) delete mode 100644 .github/workflows/TEMP-testing-command.yml diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml deleted file mode 100644 index d931ef501ee8..000000000000 --- a/.github/workflows/TEMP-testing-command.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: TEMP testing command -on: - workflow_dispatch: - inputs: - repo: - description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." - required: false - default: "airbytehq/airbyte" - gitref: - description: "The git ref to check out from the specified repository." - required: false - default: master - connector: - description: "Airbyte Connector" - required: true - run-tests: - description: "Should run tests when publishing" - required: true - default: "true" - comment-id: - description: "The comment-id of the slash command. Used to update the comment with the status." - required: false - auto-bump-version: - description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" - required: true - default: "true" - -jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} - ## Gradle Build - # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWCRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner - ec2-instance-id: ${{ matrix.ec2-instance-id }} - - stop-publish-image-runner: - name: Stop Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] - needs: - - start-publish-image-runner # required to get output from the start-runner job - # - publish-image # required to wait when the main job is done - - find_valid_pat - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ matrix.ec2-instance-id }} From 1748a5ec79251944679fd87cd9b744afc74f1f23 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 14:23:12 +0000 Subject: [PATCH 32/38] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index ea6afc10a8b6..5ba048fa3cf1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -73,7 +73,7 @@ - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/databricks icon: databricks.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 81d82333f41e..3ee588ead68f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -996,7 +996,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-databricks:0.2.2" +- dockerImage: "airbyte/destination-databricks:0.2.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/databricks" connectionSpecification: From 32951e12d59b77fa8f88ea3de9edf7b9bd5704b7 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:27:58 +0100 Subject: [PATCH 33/38] Delete publish-command.yml --- .github/workflows/publish-command.yml | 426 -------------------------- 1 file changed, 426 deletions(-) delete mode 100644 .github/workflows/publish-command.yml diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml deleted file mode 100644 index 8dbfb7552ba3..000000000000 --- a/.github/workflows/publish-command.yml +++ /dev/null @@ -1,426 +0,0 @@ -name: Publish Connector Image -on: - workflow_dispatch: - inputs: - repo: - description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." - required: false - default: "airbytehq/airbyte" - gitref: - description: "The git ref to check out from the specified repository." - required: false - default: master - connector: - description: "Airbyte Connector" - required: true - run-tests: - description: "Should run tests when publishing" - required: true - default: "true" - comment-id: - description: "The comment-id of the slash command. Used to update the comment with the status." - required: false - auto-bump-version: - description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" - required: true - default: "true" - parallel: - description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" - required: true - default: "false" - -jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} - ## Gradle Build - # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner-0: - name: Start Build EC2 Runner 0 - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - start-publish-image-runner-1: - if: github.event.inputs.parallel == 'true' && success() - name: Start Build EC2 Runner 1 - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - start-publish-image-runner-2: - if: github.event.inputs.parallel == 'true' && success() - name: Start Build EC2 Runner 2 - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - start-publish-image-runner-3: - if: github.event.inputs.parallel == 'true' && success() - name: Start Build EC2 Runner 3 - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - start-publish-image-runner-4: - if: github.event.inputs.parallel == 'true' && success() - name: Start Build EC2 Runner 4 - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - preprocess-matrix: - needs: start-publish-image-runner-0 - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - outputs: - connectorjson: ${{ steps.preprocess.outputs.connectorjson }} - steps: - - id: preprocess - run: | - start="[\"" - replace="\",\"" - end="\"]" - stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')" - middle=${stripped_connector//,/$replace} - full="$start$middle$end" - echo "::set-output name=connectorjson::$full" - publish-image: - timeout-minutes: 240 - needs: - - start-publish-image-runner-0 - - preprocess-matrix - strategy: - max-parallel: 5 - fail-fast: false - matrix: - connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that - environment: more-secrets - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ matrix.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != matrix.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - token: ${{ secrets.OCTAVIA_PAT }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: "17" - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Install Pyenv and Tox - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - pip install --quiet tox==3.24.4 - - name: Install yq - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo apt update - sudo apt install yq -y - - name: Test and install CI scripts - # all CI python packages have the prefix "ci_" - run: | - source venv/bin/activate - tox -r -c ./tools/tox_ci.ini - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ matrix.connector }} - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Set Name and Version Environment Vars - if: startsWith(matrix.connector, 'connectors') - run: | - source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - - name: Prepare Sentry - if: startsWith(matrix.connector, 'connectors') - run: | - curl -sL https://sentry.io/get-cli/ | bash - - name: Create Sentry Release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ matrix.connector }} - run: | - echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - - name: Finalize Sentry release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - - name: Check if connector in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - then exit 1 - fi - - name: Bump version in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - if [ -z "$sourcename" ] - then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - fi - - name: Run gradle process changes - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - ./gradlew :airbyte-config:init:processResources - - name: git config - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git config user.name 'Octavia Squidington III' - git config user.email 'octavia-squidington-iii@users.noreply.github.com' - - name: git commit and push - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git add -u - git commit -m "auto-bump connector version" - git pull origin ${{ github.event.inputs.gitref }} - git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - # In case of self-hosted EC2 errors, remove this block. - stop-publish-image-runner-0: - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - name: Stop Build EC2 Runner - needs: - - start-publish-image-runner-0 # required to get output from the start-runner job - - preprocess-matrix - - publish-image # required to wait when the main job is done - - find_valid_pat - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner-0.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} - stop-publish-image-runner-multi: - if: ${{ always() && github.event.inputs.parallel == 'true' }} - name: Stop Build EC2 Runner - needs: - - start-publish-image-runner-0 - - start-publish-image-runner-1 - - start-publish-image-runner-2 - - start-publish-image-runner-3 - - start-publish-image-runner-4 - - preprocess-matrix - - publish-image # required to wait when the main job is done - - find_valid_pat - strategy: - fail-fast: false - matrix: - ec2-instance: - [ - { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", - }, - { - "label": "${{ needs.start-publish-image-runner-2.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", - }, - { - "label": "${{ needs.start-publish-image-runner-3.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", - }, - { - "label": "${{ needs.start-publish-image-runner-4.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", - }, - ] - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ matrix.ec2-instance.label }} - ec2-instance-id: ${{ matrix.ec2-instance.id }} From 79cb87d3fa848faf8954f3c9ca09e9688bfaf554 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:50:26 +0100 Subject: [PATCH 34/38] added comment describing bash part --- .github/workflows/publish-command.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 8dbfb7552ba3..9e81a42034f5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -159,6 +159,9 @@ jobs: outputs: connectorjson: ${{ steps.preprocess.outputs.connectorjson }} steps: + # given a string input of a single connector or comma separated list of connectors e.g. connector1, connector2 + # this step builds an array, by removing whitespace, add in quotation marks around connectors and braces [ ] at the start and end + # finally, it sets it as output from this job so we can use this array of connectors as our matrix strategy for publishing - id: preprocess run: | start="[\"" From 4fe526db618c9371c5c6fd36068895c803bfad2d Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:01:53 +0100 Subject: [PATCH 35/38] running single thread --- .github/workflows/publish-command.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 9e81a42034f5..5cfa5e01e468 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -181,7 +181,8 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + # runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} environment: more-secrets steps: - name: Link comment to workflow run From bb7de7ad5ff48f4a079f3586393330752b810c24 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:05:35 +0100 Subject: [PATCH 36/38] catch sentry cli --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 5cfa5e01e468..1ee6fc4e0fd2 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -256,7 +256,7 @@ jobs: - name: Prepare Sentry if: startsWith(matrix.connector, 'connectors') run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed" - name: Create Sentry Release if: startsWith(matrix.connector, 'connectors') run: | From b690ac3d7a83832630d18314c04b89ec4fb40448 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 15:06:31 +0000 Subject: [PATCH 37/38] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index ea6afc10a8b6..b8f98b09dc29 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Databricks Lakehouse diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 81d82333f41e..1acd6b2c5eef 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -827,7 +827,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.6" +- dockerImage: "airbyte/destination-clickhouse:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: From 8f44baf5d55f5c9af916b0099f917cc62233843c Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:13:21 +0100 Subject: [PATCH 38/38] Update _variables.scss --- airbyte-webapp/src/scss/_variables.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/scss/_variables.scss b/airbyte-webapp/src/scss/_variables.scss index 5d3267e3e276..4d1b7a6a7622 100644 --- a/airbyte-webapp/src/scss/_variables.scss +++ b/airbyte-webapp/src/scss/_variables.scss @@ -5,8 +5,8 @@ $border-thick: 2px; $spacing-xs: 3px; $spacing-sm: 5px; -$spacing-md: 10px; -$spacing-lg: 15px; +$spacing-m: 10px; +$spacing-l: 15px; $spacing-xl: 20px; $spacing-2xl: 40px; $spacing-page-bottom: 150px;