[sc-34057]: Parallelise build-dev and build-master (#710)

.github/workflows/build-branch.yml

@@ -1,59 +1,328 @@
 name: Build QML Branch
 on:
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: The QML branch to checkout and build demos for
-        required: true
-        type: string
   workflow_call:
     inputs:
       branch:
         description: The QML branch to checkout and build demos for
         required: true
         type: string
+      num_workers:
+        description: The number of workers to use for building the QML demos
+        required: false
+        type: string
+        default: '15'
+      enable_sphinx_cache:
+        description: Use actions/cache for sphinx and sphinx-gallery
+        required: false
+        type: boolean
+        default: false
+      refresh_sphinx_cache:
+        description: Build QML Demos without using cache and create new caches after the build
+        required: false
+        type: boolean
+        default: false
+      enable_python_cache:
+        description: Use actions/cache for python packages being installed
+        required: false
+        type: boolean
+        default: false
+      skip_execution_times_aggregation:
+        description: Skip aggregating all the execution times from all workers into one file
+        required: false
+        type: boolean
+        default: false
+      skip_sphinx_build_file_aggregation:
+        description: Skip aggregating the html files built from all workers into one zip file
+        required: false
+        type: boolean
+        default: false
 
 jobs:
+  compute-build-strategy-matrix:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install QML Pipeline Utils
+        run: |
+          cd .github/workflows/qml_pipeline_utils
+          pip install .
+
+      - name: Generate Build Matrix
+        id: compute-strategy-matrix
+        run: |
+          echo "strategy-matrix=$(qml_pipeline_utils \
+            build-strategy-matrix \
+            --num-workers=${{ inputs.num_workers }} \
+            --examples-dir='${{ github.workspace }}/demonstrations')" >> $GITHUB_OUTPUT
+
+    outputs:
+      strategy-matrix: ${{ steps.compute-strategy-matrix.outputs.strategy-matrix }}
+
   build-branch:
     runs-on: ubuntu-22.04
+    needs:
+      - compute-build-strategy-matrix
+    strategy:
+      matrix:
+        offset: ${{ fromJson(needs.compute-build-strategy-matrix.outputs.strategy-matrix) }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.branch }}
+          fetch-depth: 1
 
-      - name: Set up Python
-        uses: actions/setup-python@v3
+      - name: Set up Python Version
+        id: setup_python
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
+      - name: Python Environment Cache
+        if: inputs.enable_python_cache == true
+        uses: actions/cache@v3
+        with:
+          path: venv
+          key: pip-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_no_deps.txt') }}
+
+      - name: Setup Python Environment
+        id: venv
+        run: |
+          if [ ! -d "venv" ]; then python3 -m venv venv; fi
+          echo "location=${{ github.workspace }}/venv" >> $GITHUB_OUTPUT
+          echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH
+
       - name: Install QML Pipeline Utils
         run: |
           cd .github/workflows/qml_pipeline_utils
-          pip install .
+          ${{ steps.venv.outputs.location }}/bin/python3 -m pip install .
 
       - name: Install Python Dependencies
         run: |
-          pip install -r requirements.txt
-          pip install --no-deps -r requirements_no_deps.txt
-          pip install --upgrade pip 'setuptools<65' cmake
+          ${{ steps.venv.outputs.location }}/bin/python3 -m pip install -r requirements.txt
+          ${{ steps.venv.outputs.location }}/bin/python3 -m pip install --no-deps -r requirements_no_deps.txt
+          ${{ steps.venv.outputs.location }}/bin/python3 -m pip install --upgrade pip 'setuptools<65' cmake
+
+      # Creates a temp file with current matrix information, which includes:
+      #  - Current offset
+      #  - Total Workers
+      #  - Metadata regarding the Python version and dependencies
+      #  - The name of files that will be executed by this worker
+      # The hash of this file is used as portion of the key in subsequent caching steps.
+      # This ensures that if the number of worker, tutorials, or python evironment details change,
+      # it will invalidate the previous cache and build fresh.
+      - name: Set Matrix offset file
+        run: |
+          cat >matrix_info.txt <<EOL
+          workers: ${{ env.NUM_WORKERS }}
+          offset: ${{ matrix.offset }}
+          ---
+          python_version: ${{ steps.setup_python.outputs.python-version }}
+          requirements: ${{ hashFiles('requirements.txt') }}
+          requirements_no_deps: ${{ hashFiles('requirements_no_deps.txt') }}
+          EOL
+
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
+          show-worker-files \
+          --num-workers=${{ inputs.num_workers }} \
+          --examples-dir="${{ github.workspace }}/demonstrations" \
+          --offset=${{ matrix.offset }} >> matrix_info.txt
+
+          cat matrix_info.txt
+
+      - name: Generate hash of the matrix file
+        id: matrix_file
+        if: inputs.enable_sphinx_cache == true
+        run: |
+          echo "hash=${{ hashFiles('matrix_info.txt') }}" >> $GITHUB_OUTPUT
+
+      - name: Install OS build dependencies
+        run: |
+          sudo apt-get install -y pandoc --quiet
+
+      # Removes executable code from tutorials that are not relevant to current node
+      # See documentation in github_job_scheduler.py for more details.
+      - name: Remove extraneous executable code from demos
+        run: |
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
+          remove-executable-code-from-extraneous-demos \
+          --num-workers=${{ inputs.num_workers }} \
+          --examples-dir="${{ github.workspace }}/demonstrations" \
+          --offset=${{ matrix.offset }} \
+          --verbose
 
-      - name: Build tutorials
+      - name: Gallery Cache
+        if: inputs.enable_sphinx_cache == true
+        uses: actions/cache@v3
+        with:
+          path: demos
+          key: gallery-${{ steps.matrix_file.outputs.hash }}-${{ github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            gallery-${{ steps.matrix_file.outputs.hash }}-${{ github.ref_name }}-
+            gallery-${{ steps.matrix_file.outputs.hash }}-
+
+      - name: Sphinx Cache
+        if: inputs.enable_sphinx_cache == true
+        uses: actions/cache@v3
+        with:
+          path: sphinx_cache-${{ steps.matrix_file.outputs.hash }}
+          key: sphinx-${{ steps.matrix_file.outputs.hash }}-${{ github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            sphinx-${{ steps.matrix_file.outputs.hash }}-${{ github.ref_name }}-
+            sphinx-${{ steps.matrix_file.outputs.hash }}-
+
+      - name: Clear Cache
+        if: inputs.refresh_sphinx_cache == true
+        env:
+          sphinx_cache_filename: sphinx_cache-${{ steps.matrix_file.outputs.hash }}
+        run: |
+          if [ -d demos ]; then rm -rf demos; fi
+          if [ -d ${{ env.sphinx_cache_filename }} ]; then rm -rf ${{ env.sphinx_cache_filename }}; fi
+
+      - name: Build Tutorials
         run: |
           make download
-          make html
+          make SPHINXBUILD="${{ steps.venv.outputs.location }}/bin/sphinx-build" SPHINXOPTS="-d sphinx_cache-${{ steps.matrix_file.outputs.hash }}" html
 
       - name: Generate Execution Time Map
         run: |
           mkdir /tmp/execution_times
 
-          qml_pipeline_utils \
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
           parse-execution-times \
+          --num-workers=${{ inputs.num_workers }} \
+          --offset=${{ matrix.offset }} \
+          --examples-dir="${{ github.workspace }}/demonstrations" \
           --build-dir="${{ github.workspace }}/_build/html" > /tmp/execution_times/execution_times.json
           
           cat /tmp/execution_times/execution_times.json | jq
 
       - name: Upload Execution Times
         uses: actions/upload-artifact@v3
         with:
-          name: execution_times_${{ inputs.branch }}
+          name: execution_times_${{ matrix.offset }}.zip
+          if-no-files-found: error
+          retention-days: 1
+          path: /tmp/execution_times
+
+      # These are files that are generated as part of sphinx-build but are not needed and supported on the live website
+      # There does not seem to be an option to "not" generate them, therefore this step deletes these files before they
+      # are published to the live website.
+      - name: Update sitemap.xml
+        run: |
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
+          clean-sitemap \
+          --build-dir="${{ github.workspace }}/_build/html" \
+          --html-files="demos/sg_execution_times.html" \
+          --verbose
+
+      # Removes built html files that are not relevant to current node
+      # See documentation in github_job_scheduler.py for more details.
+      - name: Clean HTML Files
+        if: matrix.offset == 0
+        run: |
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
+          remove-extraneous-built-html-files \
+          --num-workers=${{ inputs.num_workers }} \
+          --build-dir="${{ github.workspace }}/_build/html" \
+          --examples-dir="${{ github.workspace }}/demonstrations" \
+          --offset=${{ matrix.offset }} \
+          --preserve-non-sphinx-images \
+          --verbose
+
+      - name: Clean HTML Files and Images
+        if: matrix.offset != 0
+        run: |
+          ${{ steps.venv.outputs.location }}/bin/qml_pipeline_utils \
+          remove-extraneous-built-html-files \
+          --num-workers=${{ inputs.num_workers }} \
+          --build-dir="${{ github.workspace }}/_build/html" \
+          --examples-dir="${{ github.workspace }}/demonstrations" \
+          --offset=${{ matrix.offset }} \
+          --verbose
+
+      - name: Upload Html
+        if: matrix.offset == 0
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-${{ matrix.offset }}.zip
+          if-no-files-found: error
+          retention-days: 1
+          path: _build/html
+
+      # Only upload demos since all other html files are pushed as artifact from offset 0
+      # This step excludes static files (files that are the same across all workers) from being included in the
+      # built artifact. This is done as a performance boost.
+      # The step above this is executed by only one worker which uploads all static content.
+      - name: Upload Demo Html
+        if: matrix.offset != 0
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-${{ matrix.offset }}.zip
+          if-no-files-found: error
+          retention-days: 1
+          path: |
+            _build/html
+            !_build/html/*.html
+            !_build/html/*.js
+            !_build/html/*.xml
+            !_build/html/_static
+            !_build/html/glossary
+
+  aggregate_build:
+    runs-on: ubuntu-22.04
+    if: inputs.skip_execution_times_aggregation == false || inputs.skip_sphinx_build_file_aggregation == false
+    needs:
+      - build-branch
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+
+      - name: Merge Execution Times
+        if: inputs.skip_execution_times_aggregation == false
+        run: |
+          cd artifacts
+          mkdir execution_times_all /tmp/execution_times
+          for f in execution_times_*.zip; do
+            new_name=execution_times-\($f\).json
+            mv $f/execution_times.json execution_times_all/$new_name
+            echo execution_times_all/$new_name
+            cat execution_times_all/$new_name | jq
+          done
+          jq -s 'reduce .[] as $item ({}; . * $item)' execution_times_all/* | tee /tmp/execution_times/execution_times.json
+
+          cat /tmp/execution_times/execution_times.json | jq
+
+      - name: Merge Sphinx Build Files
+        if: inputs.skip_sphinx_build_file_aggregation == false
+        run: |
+          cd artifacts
+          mkdir -p website/demos
+          for f in html-*.zip; do
+            rsync -a --progress "$f/" website
+          done
+
+      - name: Upload Sphinx Build files
+        if: inputs.skip_sphinx_build_file_aggregation == false
+        uses: actions/upload-artifact@v3
+        with:
+          name: html.zip
+          if-no-files-found: error
+          path: artifacts/website
+
+      - name: Upload Execution Times
+        if: inputs.skip_execution_times_aggregation == false
+        uses: actions/upload-artifact@v3
+        with:
+          name: execution_times.zip
           path: /tmp/execution_times

.github/workflows/qml_pipeline_utils/qml_pipeline_utils/cli.py

@@ -128,7 +128,8 @@ def cli_parser():
         description="Parse the `sg_exection_times.html` file generated by sphinx "
         "to get the time to execute each example",
     )
-    add_flags_to_subparser(subparsers_parse_execution_times, "build-dir", "gallery-dir-name")
+    add_flags_to_subparser(subparsers_parse_execution_times, "num-workers", "offset", "examples-dir", "build-dir",
+                           "gallery-dir-name", "glob-pattern")
 
     parser_results = parser.parse_args()
 
@@ -193,6 +194,10 @@ def cli_parser():
         "parse-execution-times": {
             "func": qml_pipeline_utils.services.parse_execution_times,
             "kwargs": {
+                "num_workers": getattr(parser_results, "num_workers", None),
+                "offset": getattr(parser_results, "offset", None),
+                "sphinx_examples_dir": Path(getattr(parser_results, "examples_dir", "")),
+                "glob_pattern": getattr(parser_results, "glob_pattern", None),
                 "sphinx_build_directory": Path(getattr(parser_results, "build_dir", "")),
                 "sphinx_gallery_dir_name": getattr(parser_results, "gallery_dir_name", "")
             },