CD - Feature Branch #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #TODO: Due to lack of time, this could not be de-duplicated | |
| # from cd-syft.yml, which have a similar structure | |
| name: CD - Feature Branch | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release_version: | |
| description: "Syft Version to Release" | |
| required: true | |
| type: string | |
| release_branch: | |
| description: "Branch to Release from" | |
| required: true | |
| type: string | |
| release_id: | |
| description: "A Unique Identifier for the Release, which would be appended as version+release_id, Can contain only [A-Z][a-z][0-9][.]" | |
| required: true | |
| type: string | |
| release_platform: | |
| description: "Release Platform" | |
| required: true | |
| default: "TEST_PYPI" | |
| type: choice | |
| options: | |
| - TEST_PYPI | |
| # - REAL_PYPI | |
| # - REAL_AND_TEST_PYPI | |
| # Prevents concurrent runs of the same workflow | |
| # while the previous run is still in progress | |
| concurrency: | |
| group: "CD - Feature Branch" | |
| cancel-in-progress: false | |
| jobs: | |
| build-and-push-docker-images: | |
| strategy: | |
| matrix: | |
| runner: [sh-arc-linux-x64, sh-arc-linux-arm64] | |
| runs-on: ${{ matrix.runner }} | |
| outputs: | |
| server_version: ${{ steps.release_metadata.outputs.server_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.release_branch }} | |
| # actions/setup-python doesn't yet support ARM | |
| - name: Setup Python on x64 | |
| if: ${{ !endsWith(matrix.runner, '-arm64') }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| # Currently psutil package requires gcc to be installed on arm | |
| # for building psutil from source | |
| # as linux/aarch64 wheels not avaialble for psutil | |
| # We could remove once we have aarch64 wheels | |
| # https://github.com/giampaolo/psutil/issues/1972 | |
| - name: Install Metadata packages for arm64 | |
| if: ${{ endsWith(matrix.runner, '-arm64') }} | |
| run: | | |
| sudo apt update -y | |
| sudo apt install software-properties-common -y | |
| sudo apt install gcc curl -y | |
| sudo apt-get install python3-dev -y | |
| - name: Setup Python on arm64 | |
| if: ${{ endsWith(matrix.runner, '-arm64') }} | |
| uses: deadsnakes/[email protected] | |
| with: | |
| python-version: "3.12" | |
| - name: Install Git | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install git -y | |
| - name: Check python version | |
| run: | | |
| python --version | |
| python3 --version | |
| which python | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.2.17 tox tox-uv==1.9.0 bump2version==1.0.1 | |
| uv --version | |
| - name: Generate Release Metadata | |
| id: release_metadata | |
| run: | | |
| if [[ ${{matrix.runner}} == *"x64"* ]]; then | |
| echo "release_platform=linux/amd64" >> $GITHUB_OUTPUT | |
| echo "short_release_platform=amd64" >> $GITHUB_OUTPUT | |
| else | |
| echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT | |
| echo "short_release_platform=arm64" >> $GITHUB_OUTPUT | |
| fi | |
| echo "server_version=${{ github.event.inputs.release_version }}+${{github.events.inputs.release_id}}.$(git rev-parse --short ${{ github.sha }})" >> $GITHUB_OUTPUT | |
| - name: Bump to Final Release version | |
| run: | | |
| python scripts/bump_version.py --bump-to-stable ${{ steps.release_metadata.outputs.server_version}} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_LOGIN }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build and push `syft-backend` image to DockerHub | |
| id: syft-backend-build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages | |
| file: ./packages/grid/backend/backend.dockerfile | |
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | |
| target: backend | |
| outputs: type=image,name=openmined/syft-backend,push-by-digest=true,name-canonical=true,push=true | |
| cache-from: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }} | |
| cache-to: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max | |
| - name: Export digest for syft-backend | |
| run: | | |
| mkdir -p /tmp/digests/syft-backend | |
| digest="${{ steps.syft-backend-build.outputs.digest }}" | |
| touch "/tmp/digests/syft-backend/${digest#sha256:}" | |
| - name: Build and push `syft-frontend` image to DockerHub | |
| id: syft-frontend-build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/grid/frontend | |
| file: ./packages/grid/frontend/frontend.dockerfile | |
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | |
| outputs: type=image,name=openmined/syft-frontend,push-by-digest=true,name-canonical=true,push=true | |
| target: syft-ui-development | |
| cache-from: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }} | |
| cache-to: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max | |
| - name: Export digest for syft-frontend | |
| run: | | |
| mkdir -p /tmp/digests/syft-frontend | |
| digest="${{ steps.syft-frontend-build.outputs.digest }}" | |
| touch "/tmp/digests/syft-frontend/${digest#sha256:}" | |
| - name: Build and push `syft-seaweedfs` image to DockerHub | |
| id: syft-seaweedfs-build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/grid/seaweedfs | |
| file: ./packages/grid/seaweedfs/seaweedfs.dockerfile | |
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | |
| outputs: type=image,name=openmined/syft-seaweedfs,push-by-digest=true,name-canonical=true,push=true | |
| cache-from: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }} | |
| cache-to: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max | |
| - name: Export digest for syft-seaweedfs | |
| run: | | |
| mkdir -p /tmp/digests/syft-seaweedfs | |
| digest="${{ steps.syft-seaweedfs-build.outputs.digest }}" | |
| touch "/tmp/digests/syft-seaweedfs/${digest#sha256:}" | |
| # Some of the dependencies of syft-enclave-attestation are not available for arm64 | |
| # Hence, we are building syft-enclave-attestation only for x64 (see the `if` conditional) | |
| - name: Build and push `syft-enclave-attestation` image to DockerHub | |
| if: ${{ endsWith(matrix.runner, '-x64') }} | |
| id: syft-enclave-attestation-build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/grid/enclave/attestation | |
| file: ./packages/grid/enclave/attestation/attestation.dockerfile | |
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | |
| outputs: type=image,name=openmined/syft-enclave-attestation,push-by-digest=true,name-canonical=true,push=true | |
| cache-from: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform }} | |
| cache-to: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max | |
| - name: Export digest for syft-enclave-attestation | |
| if: ${{ endsWith(matrix.runner, '-x64') }} | |
| run: | | |
| mkdir -p /tmp/digests/syft-enclave-attestation | |
| digest="${{ steps.syft-enclave-attestation-build.outputs.digest }}" | |
| touch "/tmp/digests/syft-enclave-attestation/${digest#sha256:}" | |
| - name: Build and push `syft` image to registry | |
| id: syft-build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./packages/ | |
| file: ./packages/grid/syft-client/syft.Dockerfile | |
| outputs: type=image,name=openmined/syft-client,push-by-digest=true,name-canonical=true,push=true | |
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | |
| cache-from: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }} | |
| cache-to: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max | |
| - name: Export digest for `syft` image | |
| run: | | |
| mkdir -p /tmp/digests/syft | |
| digest="${{ steps.syft-build.outputs.digest }}" | |
| touch "/tmp/digests/syft/${digest#sha256:}" | |
| - name: Upload digests | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digests-${{ steps.release_metadata.outputs.server_version }}-${{ steps.release_metadata.outputs.short_release_platform }} | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| #Used to merge x64 and arm64 into one docker image | |
| merge-docker-images: | |
| needs: [build-and-push-docker-images] | |
| if: always() && (needs.build-and-push-docker-images.result == 'success') | |
| runs-on: sh-arc-linux-x64 | |
| outputs: | |
| server_version: ${{ needs.build-and-push-docker-images.outputs.server_version }} | |
| steps: | |
| - name: Download digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp/digests | |
| pattern: digests-${{ needs.build-and-push-docker-images.outputs.server_version }}-* | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_LOGIN }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Create manifest list and push for syft-backend | |
| working-directory: /tmp/digests/syft-backend | |
| run: | | |
| docker buildx imagetools create \ | |
| -t openmined/syft-backend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ | |
| $(printf 'openmined/syft-backend@sha256:%s ' *) | |
| - name: Create manifest list and push for syft-frontend | |
| working-directory: /tmp/digests/syft-frontend | |
| run: | | |
| docker buildx imagetools create \ | |
| -t openmined/syft-frontend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ | |
| $(printf 'openmined/syft-frontend@sha256:%s ' *) | |
| - name: Create manifest list and push for syft-seaweedfs | |
| working-directory: /tmp/digests/syft-seaweedfs | |
| run: | | |
| docker buildx imagetools create \ | |
| -t openmined/syft-seaweedfs:${{ needs.build-and-push-docker-images.outputs.server_version }} \ | |
| $(printf 'openmined/syft-seaweedfs@sha256:%s ' *) | |
| - name: Create manifest list and push for syft-enclave-attestation | |
| working-directory: /tmp/digests/syft-enclave-attestation | |
| run: | | |
| docker buildx imagetools create \ | |
| -t openmined/syft-enclave-attestation:${{ needs.build-and-push-docker-images.outputs.server_version }} \ | |
| $(printf 'openmined/syft-enclave-attestation@sha256:%s ' *) | |
| - name: Create manifest list and push for syft client | |
| working-directory: /tmp/digests/syft | |
| run: | | |
| docker buildx imagetools create \ | |
| -t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.server_version }} \ | |
| $(printf 'openmined/syft-client@sha256:%s ' *) | |
| deploy-syft: | |
| needs: [merge-docker-images] | |
| if: always() && needs.merge-docker-images.result == 'success' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Permission to home directory | |
| run: | | |
| sudo chown -R $USER:$USER $HOME | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} | |
| ref: ${{ github.event.inputs.release_branch }} | |
| # free 10GB of space | |
| - name: Remove unnecessary files | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| docker image prune --all --force | |
| docker builder prune --all --force | |
| docker system prune --all --force | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv==0.2.17 tox tox-uv==1.9.0 setuptools wheel twine bump2version PyYAML | |
| uv --version | |
| - name: Bump to Final Release version | |
| run: | | |
| python scripts/bump_version.py --bump-to-stable ${{ needs.merge-docker-images.outputs.server_version }} | |
| - name: Build Helm Chart | |
| shell: bash | |
| run: | | |
| # install k3d | |
| K3D_VERSION=v5.6.3 | |
| wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 | |
| mv k3d-linux-amd64 k3d | |
| chmod +x k3d | |
| export PATH=`pwd`:$PATH | |
| k3d version | |
| #Install Devspace | |
| DEVSPACE_VERSION=v6.3.12 | |
| curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace | |
| chmod +x devspace | |
| devspace version | |
| # Install helm | |
| curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash | |
| helm version | |
| tox -e syft.build.helm | |
| tox -e syft.package.helm | |
| - name: Linting | |
| run: | | |
| tox -e lint || true | |
| - name: Manual Build and Publish | |
| run: | | |
| tox -e syft.publish | |
| if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then | |
| twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* | |
| fi | |
| # Checkout to gh-pages and update helm repo | |
| - name: Checkout to gh-pages | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} | |
| path: ghpages | |
| - name: Copy helm repo files from Syft Repo | |
| run: | | |
| rm -rf ghpages/helm/* | |
| cp -R packages/grid/helm/repo/. ghpages/helm/ | |
| - name: Commit changes to gh-pages | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| author_name: ${{ secrets.OM_BOT_NAME }} | |
| author_email: ${{ secrets.OM_BOT_EMAIL }} | |
| message: "Update Helm package from Syft Repo" | |
| add: "helm/" | |
| push: "origin gh-pages" | |
| cwd: "./ghpages/" |