Implement build system using self-hosted aarch64 runners, GitHub needs jobs feature and reusable workflows

.github/actions/create-dev-env/action.yml

@@ -0,0 +1,16 @@
+name: "Build environment"
+description: "Create build environment"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set Up Python 🐍
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.x
+
+    - name: Install Dev Dependencies 📦
+      run: |
+        pip install --upgrade pip
+        pip install -r requirements-dev.txt
+      shell: bash

.github/actions/download-manifests/action.yml

@@ -0,0 +1,101 @@
+name: "Download manifests"
+description: "Download all manifests and history lines"
+
+# Unfortunately, `actions/download-artifact` doesn't support wildcard download
+# To make this workflow fast, we manually download all manifests and history lines
+# https://github.com/actions/download-artifact/issues/6
+
+inputs:
+  histLineDir:
+    description: "Directory to store history lines"
+    required: true
+    type: string
+  manifestDir:
+    description: "Directory to store manifest files"
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: tensorflow-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: datascience-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-amd64-history_line
+        path: ${{ inputs.histLineDir }}
+
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: base-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: minimal-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: scipy-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: r-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: tensorflow-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: datascience-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: pyspark-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}
+    - name: Download artifact 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: all-spark-notebook-amd64-manifest
+        path: ${{ inputs.manifestDir }}

.github/actions/load-image/action.yml

@@ -0,0 +1,29 @@
+name: "Load image"
+description: "Download image artifact and load it to docker"
+
+inputs:
+  image:
+    description: "Image name"
+    required: true
+    type: string
+  platform:
+    description: "Image platform"
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download built image 📥
+      uses: actions/download-artifact@v3
+      with:
+        name: ${{ inputs.image }}-${{ inputs.platform }}
+        path: /tmp/
+    - name: Load downloaded image to docker 📥
+      run: |
+        docker load --input /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        docker image ls -a
+      shell: bash
+    - name: Delete the file 🗑️
+      run: rm /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+      shell: bash

.github/workflows/docker-build-upload.yml

@@ -0,0 +1,55 @@
+name: Download parent image, build new one, upload it to GitHub artifacts
+
+on:
+  workflow_call:
+    inputs:
+      parentImage:
+        description: "Parent image name"
+        required: true
+        type: string
+      image:
+        description: "Image name"
+        required: true
+        type: string
+      platform:
+        description: "Image platform"
+        required: true
+        type: string
+      runsOn:
+        description: "GitHub Actions Runner image"
+        required: true
+        type: string
+
+jobs:
+  build-upload:
+    runs-on: ${{ inputs.runsOn }}
+    steps:
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v3
+      - name: Create dev environment 📦
+        uses: ./.github/actions/create-dev-env
+
+      - name: Load parent built image to docker 📥
+        if: ${{ inputs.parentImage != '' }}
+        uses: ./.github/actions/load-image
+        with:
+          image: ${{ inputs.parentImage }}
+          platform: ${{ inputs.platform }}
+
+      - name: Build image 🛠
+        run: docker build --rm --force-rm --tag jupyter/${{ inputs.image }} ${{ inputs.image }}/
+        env:
+          DOCKER_BUILDKIT: 1
+          # Full logs for CI build
+          BUILDKIT_PROGRESS: plain
+        shell: bash
+      - name: Save image as a file 💾
+        run: docker save jupyter/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        shell: bash
+
+      - name: Upload artifact 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.image }}-${{ inputs.platform }}
+          path: /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+          retention-days: 3

.github/workflows/docker-tag-manifest-push.yml

@@ -0,0 +1,69 @@
+name: Download image artifact from GitHub artifacts, tag it and push to DockerHub
+
+on:
+  workflow_call:
+    inputs:
+      images:
+        description: "Stringified JSON object listing image names"
+        required: true
+        type: string
+      platform:
+        description: "Image platform"
+        required: true
+        type: string
+      runsOn:
+        description: "GitHub Actions Runner image"
+        required: true
+        type: string
+
+jobs:
+  tag-push:
+    runs-on: ${{ inputs.runsOn }}
+
+    strategy:
+      matrix:
+        image: ${{ fromJson(inputs.images) }}
+
+    steps:
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v3
+      - name: Create dev environment 📦
+        uses: ./.github/actions/create-dev-env
+
+      - name: Load image to docker 📥
+        uses: ./.github/actions/load-image
+        with:
+          image: ${{ matrix.image }}
+          platform: ${{ inputs.platform }}
+
+      - name: Create tags 🏷
+        run: python3 -m tagging.tag_image --short-image-name ${{ matrix.image }}
+        shell: bash
+
+      - name: Write manifest files 🏷
+        run: python3 -m tagging.write_manifests --short-image-name ${{ matrix.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/
+        shell: bash
+      - name: Upload manifest file 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.image }}-${{ inputs.platform }}-manifest
+          path: /tmp/manifests/*.md
+          retention-days: 3
+      - name: Upload build history line 💾
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.image }}-${{ inputs.platform }}-history_line
+          path: /tmp/hist_lines/*.txt
+          retention-days: 3
+
+      - name: Login to Docker Hub 🔐
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule'
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # dependabot updates to latest release
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push Images to Docker Hub 📤
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule'
+        run: docker push --all-tags jupyter/${{ matrix.image }}
+        shell: bash