merge

PennyLaneAI · Sep 26, 2023 · 273220a · 273220a
2 parents 0a3d393 + e150259
commit 273220a
Show file tree

Hide file tree

Showing 209 changed files with 4,945 additions and 1,031 deletions.
diff --git a/.github/workflows/build-branch.yml b/.github/workflows/build-branch.yml
@@ -54,17 +54,25 @@ on:
         type: boolean
         default: false
 
+      # TODO: This is temp workaround to add metadata previewImages validation to this job. Once the static asset files
+      #       are centralized properly, then this parameter can be removed.
+      validate_metadata_preview_images:
+        description: Validate the metadata previewImages after sphinx-build concludes
+        required: false
+        type: boolean
+        default: false
+
 jobs:
   validate-inputs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Validate inputs.sphinx_build_output_format
         run: |
           valid_choices='html json'
           [[ " $valid_choices " =~ " ${{ inputs.sphinx_build_output_format }} " ]] && exit 0 || exit 1
 
   compute-build-strategy-matrix:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -78,9 +86,7 @@ jobs:
           python-version: 3.9
 
       - name: Install QML Pipeline Utils
-        run: |
-          cd .github/workflows/qml_pipeline_utils
-          pip install .
+        run: pip install .github/workflows/qml_pipeline_utils
 
       - name: Execution Times Cache
         id: execution_times_cache
@@ -217,7 +223,7 @@ jobs:
       worker-load-artifact-name: ${{ steps.compute-strategy-matrix.outputs.worker_load_artifact_name }}
 
   validate-poetry-lock-file:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
@@ -244,7 +250,7 @@ jobs:
         run: ${{ steps.poetry.outputs.bin }} lock --check
 
   build-branch:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs:
       - compute-build-strategy-matrix
       - validate-inputs
@@ -279,9 +285,7 @@ jobs:
           echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH
 
       - name: Install QML Pipeline Utils
-        run: |
-          cd .github/workflows/qml_pipeline_utils
-          ${{ steps.venv.outputs.location }}/bin/python3 -m pip install .
+        run: ${{ steps.venv.outputs.location }}/bin/python3 -m pip install .github/workflows/qml_pipeline_utils
 
       - name: Install Poetry
         id: poetry
@@ -427,6 +431,25 @@ jobs:
           
           cat /tmp/execution_times/execution_times.json | jq
 
+      # TODO: This is a temporary step to include metadata validation as a part of the build step.
+      #       One the assets in QML are centralized with proper naming convention, this step can be removed
+      #       and the validation can be done with `validate-demo-metadata.yml` workflow.
+      - name: Validate metadata previewImages
+        run: |
+          metadata_files=()
+          input_demos_to_build=$(jq -r '[.[].name]|join(" ")' ${{ steps.worker_tasks.outputs.file_name }})
+          
+          for metadata_file in $input_demos_to_build
+          do
+            metadata_files+=("demonstrations/${metadata_file%.py}.metadata.json")
+          done
+          
+          metadata_files_str="${metadata_files[@]}"
+          
+          qml_pipeline_utils validate-metadata-preview-images \
+          --metadata-files $metadata_files_str \
+          --build-dir="${{ github.workspace }}/_build/html"
+
       - name: Upload Execution Times
         uses: actions/upload-artifact@v3
         with:
@@ -490,7 +513,7 @@ jobs:
             !_build/html/glossary
 
   aggregate_build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     if: inputs.skip_execution_times_aggregation == false || inputs.skip_sphinx_build_file_aggregation == false
     needs:
       - build-branch

diff --git a/.github/workflows/build-json.yml b/.github/workflows/build-json.yml
@@ -22,103 +22,40 @@ jobs:
       skip_sphinx_build_file_aggregation: false
       sphinx_build_output_format: json
 
-  upload:
-    runs-on: ubuntu-latest
-
+  upload-staging:
+    if: github.ref_name == 'master'
     needs:
       - build
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Download HTML
-        id: qml_json
-        uses: actions/download-artifact@v3
-        with:
-          name: json.zip
-          path: qml_json
-
-      - name: Copy Metadata into Build
-        run: |
-          mkdir ${{ steps.qml_json.outputs.download-path }}/metadata
-          find demonstrations -name "*.metadata.json" -type f | xargs cp -vt ${{ steps.qml_json.outputs.download-path }}/metadata
-
-      - name: Generate searchindex.js file
-        run: |
-          touch searchindex.js
-          echo -n 'Search.setIndex(' > searchindex.js
-          cat ${{ steps.qml_json.outputs.download-path }}/searchindex.json >> searchindex.js
-          echo -n ')' >> searchindex.js
-
-      - name: Set Upload Info
-        id: upload_info
-        env:
-          INFRA_ENV: ${{ github.ref_name == 'master' && 'PROD' || 'DEV' }}
-        run: |
-          AWS_REGION=${{ secrets.AWS_REGION }}
-          AWS_ACCESS_KEY_ID=${{ secrets[format('PL_SITE_{0}_NON_REACT_ACCESS_KEY_ID', env.INFRA_ENV)] }}
-          AWS_SECRET_ACCESS_KEY=${{ secrets[format('PL_SITE_{0}_NON_REACT_SECRET_ACCESS_KEY', env.INFRA_ENV)] }}
-          AWS_S3_BUCKET_ID=${{ secrets[format('PL_SITE_{0}_QML_JSON_S3_BUCKET_ID', env.INFRA_ENV)] }}
-          AWS_S3_BUCKET_DIR='commits/${{ github.ref_name }}-${{ github.sha }}'
-          AWS_HTML_S3_BUCKET_ID=${{ secrets[format('PL_SITE_{0}_S3_BUCKET_NAME', env.INFRA_ENV)] }}
-          
-          echo "aws_region=$AWS_REGION" >> $GITHUB_OUTPUT
-          echo "aws_access_key_id=$AWS_ACCESS_KEY_ID" >> $GITHUB_OUTPUT
-          echo "aws_secret_access_key=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_OUTPUT
-          echo "aws_s3_bucket_id=$AWS_S3_BUCKET_ID" >> $GITHUB_OUTPUT
-          echo "aws_s3_bucket_dir=$AWS_S3_BUCKET_DIR" >> $GITHUB_OUTPUT
-          echo "aws_html_s3_bucket_id=$AWS_HTML_S3_BUCKET_ID" >> $GITHUB_OUTPUT
-
-      - name: Upload
-        uses: XanaduAI/cloud-actions/push-to-s3-and-invalidate-cloudfront@main
-        with:
-          build-directory: ${{ steps.qml_json.outputs.download-path }}
-          aws-cloudfront-distribution-id: ''
-          aws-region: ${{ steps.upload_info.outputs.aws_region }}
-          aws-access-key-id: ${{ steps.upload_info.outputs.aws_access_key_id }}
-          aws-secret-access-key: ${{ steps.upload_info.outputs.aws_secret_access_key }}
-          s3-bucket: ${{ steps.upload_info.outputs.aws_s3_bucket_id }}
-          s3-directory: ${{ steps.upload_info.outputs.aws_s3_bucket_dir }}
-          s3-delete-stale-files: true
-          s3-action: upload
-          invalidate-cloudfront-cache: false
-
-      - name: Set as latest upload
-        env:
-          AWS_REGION: ${{ steps.upload_info.outputs.aws_region }}
-          AWS_ACCESS_KEY_ID: ${{ steps.upload_info.outputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY: ${{ steps.upload_info.outputs.aws_secret_access_key }}
-          AWS_S3_BUCKET_ID: ${{ steps.upload_info.outputs.aws_s3_bucket_id }}
-          AWS_S3_BUCKET_DIR: ${{ steps.upload_info.outputs.aws_s3_bucket_dir }}
-        run: aws s3 sync s3://$AWS_S3_BUCKET_ID/$AWS_S3_BUCKET_DIR s3://$AWS_S3_BUCKET_ID/${{ github.ref_name }} --delete
-
-      - name: Sync searchindex to HTML Bucket
-        env:
-          AWS_REGION: ${{ steps.upload_info.outputs.aws_region }}
-          AWS_ACCESS_KEY_ID: ${{ steps.upload_info.outputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY: ${{ steps.upload_info.outputs.aws_secret_access_key }}
-          AWS_S3_BUCKET_ID: ${{ steps.upload_info.outputs.aws_html_s3_bucket_id }}
-        run: aws s3 cp searchindex.js s3://$AWS_S3_BUCKET_ID/qml/searchindex.js
-
-  # Though this job only has one step to trigger a build in the pennylane.ai React repository, it has to be a separate
-  # job since we need the previous two to be successful before this can be done.
-  trigger-website-build:
-    runs-on: ubuntu-latest
-
+    uses: ./.github/workflows/upload-json.yml
+    with:
+      branch: ${{ github.ref }}
+      aws_s3_bucket_dir: commits/${{ github.ref_name }}-${{ github.sha }}
+      artifact_name: json.zip
+      qml_react_webhook_event_type: build-pl-site-staging
+
+    secrets:
+      aws_region: ${{ secrets.AWS_REGION }}
+      aws_access_key_id: ${{ secrets.PL_SITE_STAGING_NON_REACT_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.PL_SITE_STAGING_NON_REACT_SECRET_ACCESS_KEY }}
+      aws_json_s3_bucket_id: ${{ secrets.PL_SITE_STAGING_QML_JSON_S3_BUCKET_ID }}
+      aws_html_s3_bucket_id: ${{ secrets.PL_SITE_STAGING_S3_BUCKET_NAME }}
+      qml_react_pat_token: ${{ secrets.PL_SITE_STAGING_QML_REACT_PAT }}
+
+  upload-dev:
+    if: github.ref_name == 'dev'
     needs:
       - build
-      - upload
-
-    steps:
-      - name: Trigger Website Build
-        uses: peter-evans/repository-dispatch@v2
-        env:
-          INFRA_ENV: ${{ github.ref_name == 'master' && 'PROD' || 'DEV' }}
-          EVENT_NAME: build-pl-site-${{ github.ref_name == 'master' && 'main' || 'dev' }}
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-        with:
-          token: ${{ secrets[format('PL_SITE_{0}_QML_REACT_PAT', env.INFRA_ENV)] }}
-          repository: XanaduAI/pennylane.ai-react
-          event-type: ${{ env.EVENT_NAME }}
-          client-payload: '{"source_ref": "${{ github.ref_name }}", "source_sha": "${{ github.sha }}", "source_run_url": "${{ env.RUN_URL }}"}'
+    uses: ./.github/workflows/upload-json.yml
+    with:
+      branch: ${{ github.ref }}
+      aws_s3_bucket_dir: commits/${{ github.ref_name }}-${{ github.sha }}
+      artifact_name: json.zip
+      qml_react_webhook_event_type: build-pl-site-dev
+
+    secrets:
+      aws_region: ${{ secrets.AWS_REGION }}
+      aws_access_key_id: ${{ secrets.PL_SITE_DEV_NON_REACT_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.PL_SITE_DEV_NON_REACT_SECRET_ACCESS_KEY }}
+      aws_json_s3_bucket_id: ${{ secrets.PL_SITE_DEV_QML_JSON_S3_BUCKET_ID }}
+      aws_html_s3_bucket_id: ${{ secrets.PL_SITE_DEV_S3_BUCKET_NAME }}
+      qml_react_pat_token: ${{ secrets.PL_SITE_DEV_QML_REACT_PAT }}
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
@@ -65,6 +65,30 @@ jobs:
           done
           echo "files=${demo_filenames[@]}" >> $GITHUB_OUTPUT
 
+      - name: Detect Metadata Files that need to be changed
+        id: metadata_files_to_check
+        env:
+          DEMOS_TO_BUILD: ${{ steps.formatted_changed_demos.outputs.files }}
+        run: |
+          metadata_files=()
+          input_demos_to_build="$DEMOS_TO_BUILD"
+          
+          if [ -z "$input_demos_to_build" ]; then
+            readarray -d '' metadata_files < <(find demonstrations -name "*.metadata.json" -not -path "demonstrations/demonstrations_categories.metadata.json" -print0)
+          else
+            for metadata_file in $input_demos_to_build
+            do
+              metadata_files+=("demonstrations/${metadata_file%.py}.metadata.json")
+            done
+          fi
+          
+          metadata_files_str="${metadata_files[@]}"
+          
+          echo 'The following metadata files will be validated =>'
+          echo "$metadata_files_str"
+          
+          echo "file_names=$metadata_files_str" >> $GITHUB_OUTPUT
+
       - name: Set number of workers needed
         id: num_workers
         env:
@@ -81,12 +105,25 @@ jobs:
       num_workers: ${{ steps.num_workers.outputs.num_workers }}
       build_all_demos: ${{ steps.build_all_demos.outputs.build_all_demos }}
       demos_to_build: ${{ steps.formatted_changed_demos.outputs.files }}
+      metadata_file_names: ${{ steps.metadata_files_to_check.outputs.file_names }}
+
+  validate_metadata:
+    uses: ./.github/workflows/validate-demo-metadata.yml
+
+    needs:
+      - set_build_params
+
+    with:
+      branch: ${{ github.ref }}
+      metadata_files: ${{ needs.set_build_params.outputs.metadata_file_names }}
+      skip_metadata_preview_image_value_validation: true  # TODO: This can be reverted once the static assets are centralized
 
   build:
     uses: ./.github/workflows/build-branch.yml
 
     needs:
       - set_build_params
+      - validate_metadata
 
     # Only run if we need to build all the demos or a subset of demos.
     # If build_all_demos is false AND demos_to_build_is blank, this means
@@ -111,12 +148,12 @@ jobs:
       sphinx_build_output_format: html
       sphinx_examples_to_build: ${{ needs.set_build_params.outputs.demos_to_build }}
 
-
   save-build-context:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     needs:
       - build
+      - validate_metadata
 
     steps:
       - name: Save Pull Request Event Context