From 5a09c7a354f2c383492408f2231a678d05eda44e Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Wed, 5 Apr 2023 13:42:23 -0400 Subject: [PATCH 1/3] Migrate collector layer publishing to shared workflow This needs testing, specifically around the part where I am defining `COLLECTOR_VERSION`. Also changed tag layer pattern to remove `release/` prefix. --- .github/workflows/layer-publish.yml | 2 +- .github/workflows/release-layer-collector.yml | 138 +++++++----------- .github/workflows/release-layer-java.yml | 29 +++- 3 files changed, 78 insertions(+), 91 deletions(-) diff --git a/.github/workflows/layer-publish.yml b/.github/workflows/layer-publish.yml index acd7ae8f1a..0eb08090ed 100644 --- a/.github/workflows/layer-publish.yml +++ b/.github/workflows/layer-publish.yml @@ -62,7 +62,7 @@ jobs: echo GITHUB_ENV: cat $GITHUB_ENV - if [[ $GITHUB_REF_NAME != *$LAYER_VERSION ]]; then + if [[ $GITHUB_REF_NAME != */$LAYER_VERSION ]]; then echo "Tag $GITHUB_REF_NAME doesn't end with $LAYER_VERSION" exit 1 fi diff --git a/.github/workflows/release-layer-collector.yml b/.github/workflows/release-layer-collector.yml index 6a7b279bea..faed1ebf0d 100644 --- a/.github/workflows/release-layer-collector.yml +++ b/.github/workflows/release-layer-collector.yml @@ -1,104 +1,76 @@ -name: "Release: Collector Lambda layer" +name: "Release Collector Lambda layer" on: - workflow_dispatch: - inputs: - layer_name_keyword: - description: 'Publish layer to keyword name by substituting architecture parameter (ie: opentelemetry-python-)' - required: true - aws_region: - description: 'Deploy to aws region' - required: true - default: "[ \"us-east-1\", \"us-east-2\", \"us-west-1\", \"us-west-2\", \"ap-south-1\", \"ap-northeast-2\", \"ap-southeast-1\", \"ap-southeast-2\", \"ap-northeast-1\", \"ca-central-1\", \"eu-central-1\", \"eu-west-1\", \"eu-west-2\", \"eu-west-3\", \"eu-north-1\", \"sa-east-1\" ]" - architecture: - description: 'Architecture-compatible distributions of the layer to release in each region' - required: true - default: "[ \"amd64\", \"arm64\" ]" + # (Using tag push instead of release to allow filtering by tag prefix.) + push: + tags: + - layer-collector/** permissions: id-token: write contents: read jobs: - validate-inputs: + build-layer: runs-on: ubuntu-latest - steps: - - name: Validate `layer_name_keyword` (${{ github.event.inputs.layer_name_keyword }}) - run: | - grep -Eq "opentelemetry-(collector|java-agent|java-wrapper|nodejs|python)(-dev)*-" <<< "${{ github.event.inputs.layer_name_keyword }}" - build-archive: - runs-on: ubuntu-latest - needs: validate-inputs strategy: matrix: - architecture: ${{ fromJson(github.event.inputs.architecture) }} + architecture: + - amd64 + - arm64 + outputs: + COLLECTOR_VERSION: ${{ steps.save-collector-version.outputs.COLLECTOR_VERSION }} steps: - - name: Get layer kind by parsing `${{ github.event.inputs.layer_name_keyword }}` - run: | - echo LAYER_KIND=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 2) | tee --append $GITHUB_ENV - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: go-version: '^1.19.4' - name: build - run: | - make -C ${{ env.LAYER_KIND }} package GOARCH=${{ matrix.architecture }} + run: make -C collector package GOARCH=${{ matrix.architecture }} - uses: actions/upload-artifact@v3 with: - name: dist - path: | - ${{ github.workspace }}/${{ env.LAYER_KIND }}/build/*.zip - publish-prod: - runs-on: ubuntu-latest - needs: build-archive + name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip + path: ${{ github.workspace }}/collector/build/*.zip + + - name: Save Collector Version + id: save-collector-version + shell: bash + # `./collector -v` output is in the form `otelcol-contrib version 0.75.0` + run: | + unzip ${{ github.workspace }}/collector/build/*.zip + COLLECTOR_VERSION=$(./extensions/otelcol-contrib -v | sed 's/^.* version //') + echo "COLLECTOR_VERSION=$COLLECTOR_VERSION" >> $GITHUB_OUTPUT + + publish-layer: + uses: ./.github/workflows/layer-publish.yml + needs: build-layer strategy: matrix: - architecture: ${{ fromJson(github.event.inputs.architecture) }} - aws_region: ${{ fromJson(github.event.inputs.aws_region) }} - steps: - - name: Get layer name by substituting `${{ matrix.architecture }}` into Workflow Input name keyword - run: | - echo LAYER_NAME=$(echo "${{ github.event.inputs.layer_name_keyword }}" | sed 's//${{ matrix.architecture }}/') | tee --append $GITHUB_ENV - - name: Get layer kind by parsing `${{ github.event.inputs.layer_name_keyword }}` - run: | - echo LAYER_KIND=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 2) | tee --append $GITHUB_ENV - - name: Get bucket name for release run - run: | - echo BUCKET_NAME=lambda-artifacts-${{ env.LAYER_KIND }}-${{ matrix.architecture }}-${{ github.run_id }} | tee --append $GITHUB_ENV - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: ${{ matrix.aws_region }} - mask-aws-account-id: false - - uses: actions/checkout@v3 - - name: add build directory - run: | - mkdir -p ${{ github.workspace }}/${{ env.LAYER_KIND }}/build - - uses: actions/download-artifact@v3 - with: - name: dist - path: ${{ github.workspace }}/${{ env.LAYER_KIND }}/build/ - - name: publish - run: | - mkdir -p ${{ env.LAYER_NAME }} - make -C ${{ env.LAYER_KIND }} publish LAYER_NAME=${{ env.LAYER_NAME }} GOARCH=${{ matrix.architecture }} >> ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} - - name: public layer - run: | - layerVersion=$( - aws lambda list-layer-versions \ - --layer-name ${{ env.LAYER_NAME }} \ - --query 'max_by(LayerVersions, &Version).Version' - ) - aws lambda add-layer-version-permission \ - --layer-name ${{ env.LAYER_NAME }} \ - --version-number $layerVersion \ - --principal "*" \ - --statement-id publish \ - --action lambda:GetLayerVersion - - name: upload layer arn artifact - if: ${{ success() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.LAYER_NAME }} - path: ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} + architecture: + - amd64 + - arm64 + aws_region: + - ap-northeast-1 + - ap-northeast-2 + - ap-south-1 + - ap-southeast-1 + - ap-southeast-2 + - ca-central-1 + - eu-central-1 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - sa-east-1 + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + with: + artifact-name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip + layer-name: opentelemetry-collector + layer-version: ${{needs.build-layer.outputs.COLLECTOR_VERSION}} + architecture: ${{ matrix.architecture }} + release-group: dev + aws_region: ${{ matrix.aws_region }} + secrets: inherit diff --git a/.github/workflows/release-layer-java.yml b/.github/workflows/release-layer-java.yml index a4db2f27ec..e7c1b559be 100644 --- a/.github/workflows/release-layer-java.yml +++ b/.github/workflows/release-layer-java.yml @@ -4,14 +4,14 @@ on: # (Using tag push instead of release to allow filtering by tag prefix.) push: tags: - - release/layer-javaagent/** + - layer-javaagent/** permissions: id-token: write contents: read jobs: - build: + build-layer: runs-on: ubuntu-latest outputs: JAVAAGENT_VERSION: ${{ steps.save-javaagent-version.outputs.JAVAAGENT_VERSION }} @@ -44,17 +44,32 @@ jobs: JAVAAGENT_VERSION=$(java -jar ./opentelemetry-javaagent.jar) echo "JAVAAGENT_VERSION=$JAVAAGENT_VERSION" >> $GITHUB_OUTPUT - publish: + publish-layer: uses: ./.github/workflows/layer-publish.yml - needs: build + needs: build-layer strategy: matrix: - aws_region: [ us-east-1, us-east-2 ] - # aws_region: "[ \"us-east-1\", \"us-east-2\", \"us-west-1\", \"us-west-2\", \"ap-south-1\", \"ap-northeast-2\", \"ap-southeast-1\", \"ap-southeast-2\", \"ap-northeast-1\", \"ca-central-1\", \"eu-central-1\", \"eu-west-1\", \"eu-west-2\", \"eu-west-3\", \"eu-north-1\", \"sa-east-1\" ]" + aws_region: + - ap-northeast-1 + - ap-northeast-2 + - ap-south-1 + - ap-southeast-1 + - ap-southeast-2 + - ca-central-1 + - eu-central-1 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - sa-east-1 + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 with: artifact-name: opentelemetry-javaagent-layer.zip layer-name: opentelemetry-javaagent - layer-version: ${{needs.build.outputs.JAVAAGENT_VERSION}} + layer-version: ${{needs.build-layer.outputs.JAVAAGENT_VERSION}} # architecture: release-group: dev aws_region: ${{ matrix.aws_region }} From cbf94d1af57d47af9f567ca298a0e39917e7f738 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Thu, 6 Apr 2023 13:37:30 -0400 Subject: [PATCH 2/3] Make layer public and annotate ARN --- .github/workflows/layer-publish.yml | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/layer-publish.yml b/.github/workflows/layer-publish.yml index 0eb08090ed..f0e9957438 100644 --- a/.github/workflows/layer-publish.yml +++ b/.github/workflows/layer-publish.yml @@ -80,10 +80,28 @@ jobs: mask-aws-account-id: false - name: Publish Lambda Layer - env: - LAYER_VERSION: ${{needs.build.outputs.LAYER_VERSION}} run: | - aws lambda publish-layer-version \ - --layer-name $LAYER_NAME \ - --license-info "Apache 2.0" \ - --zip-file fileb://${{ inputs.artifact-name }} + LAYER_ARN=$( + aws lambda publish-layer-version \ + --layer-name $LAYER_NAME \ + --license-info "Apache 2.0" \ + --zip-file fileb://${{ inputs.artifact-name }} \ + --query 'LayerVersionArn' \ + --output text + ) + echo "::notice title=LayerARN::$LAYER_ARN" + echo "* $LAYER_ARN" >> $GITHUB_STEP_SUMMARY + + - name: Make Layer Public + run: | + LAYER_VERSION=$( + aws lambda list-layer-versions \ + --layer-name $LAYER_NAME \ + --query 'max_by(LayerVersions, &Version).Version' + ) + aws lambda add-layer-version-permission \ + --layer-name $LAYER_NAME \ + --version-number $LAYER_VERSION \ + --principal "*" \ + --statement-id publish \ + --action lambda:GetLayerVersion From 44d039f62a30409874bfcb144b1508d21ccf8f12 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Thu, 6 Apr 2023 14:22:58 -0400 Subject: [PATCH 3/3] Limit regions while in dev mode. --- .github/workflows/release-layer-collector.yml | 28 +++++++++---------- .github/workflows/release-layer-java.yml | 28 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release-layer-collector.yml b/.github/workflows/release-layer-collector.yml index faed1ebf0d..6527541e5b 100644 --- a/.github/workflows/release-layer-collector.yml +++ b/.github/workflows/release-layer-collector.yml @@ -50,20 +50,20 @@ jobs: - amd64 - arm64 aws_region: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 +# - ap-northeast-1 +# - ap-northeast-2 +# - ap-south-1 +# - ap-southeast-1 +# - ap-southeast-2 +# - ca-central-1 +# - eu-central-1 +# - eu-north-1 +# - eu-west-1 +# - eu-west-2 +# - eu-west-3 +# - sa-east-1 +# - us-east-1 +# - us-east-2 - us-west-1 - us-west-2 with: diff --git a/.github/workflows/release-layer-java.yml b/.github/workflows/release-layer-java.yml index e7c1b559be..9d940faeac 100644 --- a/.github/workflows/release-layer-java.yml +++ b/.github/workflows/release-layer-java.yml @@ -50,20 +50,20 @@ jobs: strategy: matrix: aws_region: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 +# - ap-northeast-1 +# - ap-northeast-2 +# - ap-south-1 +# - ap-southeast-1 +# - ap-southeast-2 +# - ca-central-1 +# - eu-central-1 +# - eu-north-1 +# - eu-west-1 +# - eu-west-2 +# - eu-west-3 +# - sa-east-1 +# - us-east-1 +# - us-east-2 - us-west-1 - us-west-2 with: