From d76f3198bb173bf8ebd9da7228d1bb09b57eb1e2 Mon Sep 17 00:00:00 2001 From: Rashid N H M <95639609+rashidnhm@users.noreply.github.com> Date: Wed, 16 Aug 2023 09:51:18 -0400 Subject: [PATCH] Added upload per env, and added new staging env (#891) * Updated JSON building workflow into one upload job per environment * Update .github/workflows/build-json.yml Co-authored-by: Mikhail Andrenkov * Move aws secrets in upload job to be global at job level --------- Co-authored-by: Mikhail Andrenkov --- .github/workflows/build-json.yml | 138 ++++++++++-------------------- .github/workflows/upload-json.yml | 125 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/upload-json.yml diff --git a/.github/workflows/build-json.yml b/.github/workflows/build-json.yml index 693d5ca5ca..b292e1ad4a 100644 --- a/.github/workflows/build-json.yml +++ b/.github/workflows/build-json.yml @@ -22,103 +22,59 @@ jobs: skip_sphinx_build_file_aggregation: false sphinx_build_output_format: json - upload: - runs-on: ubuntu-latest - + upload-production: + if: github.ref_name == 'master' needs: - build + uses: ./.github/workflows/upload-json.yml + with: + branch: ${{ github.ref }} + aws_s3_bucket_dir: commits/${{ github.ref_name }}-${{ github.sha }} + artifact_name: json.zip + qml_react_webhook_event_type: build-pl-site-main - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Download HTML - id: qml_json - uses: actions/download-artifact@v3 - with: - name: json.zip - path: qml_json - - - name: Copy Metadata into Build - run: | - mkdir ${{ steps.qml_json.outputs.download-path }}/metadata - find demonstrations -name "*.metadata.json" -type f | xargs cp -vt ${{ steps.qml_json.outputs.download-path }}/metadata - - - name: Generate searchindex.js file - run: | - touch searchindex.js - echo -n 'Search.setIndex(' > searchindex.js - cat ${{ steps.qml_json.outputs.download-path }}/searchindex.json >> searchindex.js - echo -n ')' >> searchindex.js - - - name: Set Upload Info - id: upload_info - env: - INFRA_ENV: ${{ github.ref_name == 'master' && 'PROD' || 'DEV' }} - run: | - AWS_REGION=${{ secrets.AWS_REGION }} - AWS_ACCESS_KEY_ID=${{ secrets[format('PL_SITE_{0}_NON_REACT_ACCESS_KEY_ID', env.INFRA_ENV)] }} - AWS_SECRET_ACCESS_KEY=${{ secrets[format('PL_SITE_{0}_NON_REACT_SECRET_ACCESS_KEY', env.INFRA_ENV)] }} - AWS_S3_BUCKET_ID=${{ secrets[format('PL_SITE_{0}_QML_JSON_S3_BUCKET_ID', env.INFRA_ENV)] }} - AWS_S3_BUCKET_DIR='commits/${{ github.ref_name }}-${{ github.sha }}' - AWS_HTML_S3_BUCKET_ID=${{ secrets[format('PL_SITE_{0}_S3_BUCKET_NAME', env.INFRA_ENV)] }} - - echo "aws_region=$AWS_REGION" >> $GITHUB_OUTPUT - echo "aws_access_key_id=$AWS_ACCESS_KEY_ID" >> $GITHUB_OUTPUT - echo "aws_secret_access_key=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_OUTPUT - echo "aws_s3_bucket_id=$AWS_S3_BUCKET_ID" >> $GITHUB_OUTPUT - echo "aws_s3_bucket_dir=$AWS_S3_BUCKET_DIR" >> $GITHUB_OUTPUT - echo "aws_html_s3_bucket_id=$AWS_HTML_S3_BUCKET_ID" >> $GITHUB_OUTPUT - - - name: Upload - uses: XanaduAI/cloud-actions/push-to-s3-and-invalidate-cloudfront@main - with: - build-directory: ${{ steps.qml_json.outputs.download-path }} - aws-cloudfront-distribution-id: '' - aws-region: ${{ steps.upload_info.outputs.aws_region }} - aws-access-key-id: ${{ steps.upload_info.outputs.aws_access_key_id }} - aws-secret-access-key: ${{ steps.upload_info.outputs.aws_secret_access_key }} - s3-bucket: ${{ steps.upload_info.outputs.aws_s3_bucket_id }} - s3-directory: ${{ steps.upload_info.outputs.aws_s3_bucket_dir }} - s3-delete-stale-files: true - s3-action: upload - invalidate-cloudfront-cache: false - - - name: Set as latest upload - env: - AWS_REGION: ${{ steps.upload_info.outputs.aws_region }} - AWS_ACCESS_KEY_ID: ${{ steps.upload_info.outputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.upload_info.outputs.aws_secret_access_key }} - AWS_S3_BUCKET_ID: ${{ steps.upload_info.outputs.aws_s3_bucket_id }} - AWS_S3_BUCKET_DIR: ${{ steps.upload_info.outputs.aws_s3_bucket_dir }} - run: aws s3 sync s3://$AWS_S3_BUCKET_ID/$AWS_S3_BUCKET_DIR s3://$AWS_S3_BUCKET_ID/${{ github.ref_name }} --delete + secrets: + aws_region: ${{ secrets.AWS_REGION }} + aws_access_key_id: ${{ secrets.PL_SITE_PROD_NON_REACT_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.PL_SITE_PROD_NON_REACT_SECRET_ACCESS_KEY }} + aws_json_s3_bucket_id: ${{ secrets.PL_SITE_PROD_QML_JSON_S3_BUCKET_ID }} + aws_html_s3_bucket_id: ${{ secrets.PL_SITE_PROD_S3_BUCKET_NAME }} + qml_react_pat_token: ${{ secrets.PL_SITE_PROD_QML_REACT_PAT }} - - name: Sync searchindex to HTML Bucket - env: - AWS_REGION: ${{ steps.upload_info.outputs.aws_region }} - AWS_ACCESS_KEY_ID: ${{ steps.upload_info.outputs.aws_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.upload_info.outputs.aws_secret_access_key }} - AWS_S3_BUCKET_ID: ${{ steps.upload_info.outputs.aws_html_s3_bucket_id }} - run: aws s3 cp searchindex.js s3://$AWS_S3_BUCKET_ID/qml/searchindex.js + upload-staging: + if: github.ref_name == 'dev' + needs: + - build + uses: ./.github/workflows/upload-json.yml + with: + branch: ${{ github.ref }} + aws_s3_bucket_dir: commits/${{ github.ref_name }}-${{ github.sha }} + artifact_name: json.zip + qml_react_webhook_event_type: build-pl-site-staging - # Though this job only has one step to trigger a build in the pennylane.ai React repository, it has to be a separate - # job since we need the previous two to be successful before this can be done. - trigger-website-build: - runs-on: ubuntu-latest + secrets: + aws_region: ${{ secrets.AWS_REGION }} + aws_access_key_id: ${{ secrets.PL_SITE_STAGING_NON_REACT_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.PL_SITE_STAGING_NON_REACT_SECRET_ACCESS_KEY }} + aws_json_s3_bucket_id: ${{ secrets.PL_SITE_STAGING_QML_JSON_S3_BUCKET_ID }} + aws_html_s3_bucket_id: ${{ secrets.PL_SITE_STAGING_S3_BUCKET_NAME }} + qml_react_pat_token: ${{ secrets.PL_SITE_STAGING_QML_REACT_PAT }} + upload-dev: + if: github.ref_name == 'dev' needs: - build - - upload + uses: ./.github/workflows/upload-json.yml + with: + branch: ${{ github.ref }} + aws_s3_bucket_dir: commits/${{ github.ref_name }}-${{ github.sha }} + artifact_name: json.zip + qml_react_webhook_event_type: build-pl-site-dev - steps: - - name: Trigger Website Build - uses: peter-evans/repository-dispatch@v2 - env: - INFRA_ENV: ${{ github.ref_name == 'master' && 'PROD' || 'DEV' }} - EVENT_NAME: build-pl-site-${{ github.ref_name == 'master' && 'main' || 'dev' }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - with: - token: ${{ secrets[format('PL_SITE_{0}_QML_REACT_PAT', env.INFRA_ENV)] }} - repository: XanaduAI/pennylane.ai-react - event-type: ${{ env.EVENT_NAME }} - client-payload: '{"source_ref": "${{ github.ref_name }}", "source_sha": "${{ github.sha }}", "source_run_url": "${{ env.RUN_URL }}"}' + secrets: + aws_region: ${{ secrets.AWS_REGION }} + aws_access_key_id: ${{ secrets.PL_SITE_DEV_NON_REACT_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.PL_SITE_DEV_NON_REACT_SECRET_ACCESS_KEY }} + aws_json_s3_bucket_id: ${{ secrets.PL_SITE_DEV_QML_JSON_S3_BUCKET_ID }} + aws_html_s3_bucket_id: ${{ secrets.PL_SITE_DEV_S3_BUCKET_NAME }} + qml_react_pat_token: ${{ secrets.PL_SITE_DEV_QML_REACT_PAT }} diff --git a/.github/workflows/upload-json.yml b/.github/workflows/upload-json.yml new file mode 100644 index 0000000000..5d95266fd2 --- /dev/null +++ b/.github/workflows/upload-json.yml @@ -0,0 +1,125 @@ +name: Upload JSON to S3 +on: + workflow_call: + inputs: + branch: + description: The QML branch that is being built + type: string + required: true + aws_s3_bucket_dir: + description: The directory within the S3 bucket where the files will be pushed to + type: string + required: true + artifact_name: + description: The name of the artifact which contains all the JSON files to be downloaded + type: string + required: false + default: json.zip + qml_react_webhook_event_type: + description: The event-type of the webhook to trigger a build on the website once the files are uploaded + type: string + required: true + + secrets: + aws_region: + description: The AWS Region the infrastructure resides in + required: true + aws_access_key_id: + description: AWS Access Key to use when accessing the S3 bucket + required: true + aws_secret_access_key: + description: AWS Secret Access Key to use when accessing the S3 bucket + required: true + aws_json_s3_bucket_id: + description: The S3 bucket ID where the JSON files will be pushed + required: true + aws_html_s3_bucket_id: + description: The S3 bucket ID where the searchindex.js file will be pushed + required: true + qml_react_pat_token: + description: The token that will be used to trigger a build on the react website + required: true + +jobs: + upload: + runs-on: ubuntu-latest + + env: + AWS_REGION: ${{ secrets.aws_region }} + AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }} + AWS_S3_BUCKET_ID: ${{ secrets.aws_json_s3_bucket_id }} + AWS_S3_BUCKET_DIR: ${{ inputs.aws_s3_bucket_dir }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + fetch-depth: 1 + + - name: Download HTML + id: qml_json + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.artifact_name }} + path: qml_json + + - name: Copy Metadata into Build + run: | + mkdir ${{ steps.qml_json.outputs.download-path }}/metadata + find demonstrations -name "*.metadata.json" -type f | xargs cp -vt ${{ steps.qml_json.outputs.download-path }}/metadata + + - name: Generate searchindex.js file + run: | + touch searchindex.js + echo -n 'Search.setIndex(' > searchindex.js + cat ${{ steps.qml_json.outputs.download-path }}/searchindex.json >> searchindex.js + echo -n ')' >> searchindex.js + + - name: Upload + uses: XanaduAI/cloud-actions/push-to-s3-and-invalidate-cloudfront@main + with: + build-directory: ${{ steps.qml_json.outputs.download-path }} + aws-cloudfront-distribution-id: '' + aws-region: ${{ secrets.aws_region }} + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + s3-bucket: ${{ secrets.aws_json_s3_bucket_id }} + s3-directory: ${{ inputs.aws_s3_bucket_dir }} + s3-delete-stale-files: true + s3-action: upload + invalidate-cloudfront-cache: false + + - name: Get Branch Name from Ref + id: branch_name + run: | + input_branch="${{ inputs.branch }}" + ref_name=${input_branch#refs/heads/} + echo "name=$ref_name" >> $GITHUB_OUTPUT + + - name: Set as latest upload + run: aws s3 sync s3://$AWS_S3_BUCKET_ID/$AWS_S3_BUCKET_DIR s3://$AWS_S3_BUCKET_ID/${{ steps.branch_name.outputs.name }} --delete + + - name: Sync searchindex to HTML Bucket + run: aws s3 cp searchindex.js s3://$AWS_S3_BUCKET_ID/qml/searchindex.js + + trigger-website-build: + runs-on: ubuntu-latest + + needs: + - upload + + steps: + - name: Trigger Website Build + uses: peter-evans/repository-dispatch@v2 + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + token: ${{ secrets.qml_react_pat_token }} + repository: XanaduAI/pennylane.ai-react + event-type: ${{ inputs.qml_react_webhook_event_type }} + client-payload: '{"source_ref": "${{ github.ref_name }}", "source_sha": "${{ github.sha }}", "source_run_url": "${{ env.RUN_URL }}"}' + + +