[GSK-852] Change Tensorflow to latest version, to enable arm64 test a…

…gain (#1409)

* Rewrite Dockerfile and CI workflow to build images

* Handle missing tensorflow for integration tests

* Splitting tests into two docker steps

* [GSK-852] Change Tensorflow to latest version, to enable arm64 test again

* [GSK-1763] Adding python 3.11 support

Contributes to GSK-1762

* Fix lock file (weird changes at regeneration)

* Using setup tools as build backend (not working with pdm)

* Minor changes to Github installed package

.github/workflows/build-images.yml

@@ -1,59 +1,252 @@
 name: Build & Push dev images
 
 on:
-#  schedule:
-#    - cron: '0 5 * * *'
+  push:
+    branches:
+      - main
+    tags:
+      - "v*.*.*"
   workflow_dispatch:
+    inputs:
+      run-tests:
+        description: 'If all python tests should be run during build'
+        required: true
+        type: boolean
+        default: true
+      build_only:
+        description: 'If images should be built and not push'
+        required: true
+        type: boolean
+        default: false
 
+  workflow_call:
+    inputs:
+      run-tests:
+        description: 'If all python tests should be run during build'
+        required: true
+        type: boolean
+        default: false
+      build_only:
+        description: 'If images should be built and not push'
+        required: true
+        type: boolean
+        default: true
+
+  pull_request: # This will allow to trigger on PR only with a specific label
+    types: [opened, reopened, synchronize, labeled, unlabeled]
+# Concurrency : auto-cancel "old" jobs ie when pushing again
+# https://docs.github.com/fr/actions/using-jobs/using-concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+env:
+  RUN_TESTS: false
+  BUILD_ONLY: false
+  REGISTRY_IMAGE: giskardai/giskard
+  DOCKERHUB_USER: giskardai
+# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
 jobs:
   build-images:
+    # Debug
+    strategy:
+      matrix:
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
+    if: ${{ !github.event.pull_request || contains( github.event.pull_request.labels.*.name, 'Docker') }}
     runs-on: ubuntu-latest
     steps:
+      - name: Check if python tests should be run
+        if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' || inputs.run-tests }}
+        run: echo 'RUN_TESTS=true' >> $GITHUB_ENV
+
+      - name: Check if it should be build only (if yes, do not push the images)
+        if: ${{ github.event_name == 'pull_request' || inputs.build_only }}
+        run: echo 'BUILD_ONLY=true' >> $GITHUB_ENV
+
+      - name: Check disk space
+        if: ${{ env.RUN_TESTS }}
+        run: |
+          df -h
+      - name: List installed packages
+        run: |
+          dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}'
+          apt list --installed | wc -l
+
+      - name: Free disk space
+        if: ${{ env.RUN_TESTS }}
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo rm -rf /usr/local/lib/android
+          docker rmi $(docker image ls -aq)
+          docker system prune -f
+          sudo apt remove -y \
+            alsa-topology-conf alsa-ucm-conf \
+            google-cloud-cli azure-cli microsoft-edge-stable dotnet-sdk-7.0 dotnet-sdk-6.0 temurin-17-jdk \
+            google-chrome-stable temurin-11-jdk llvm-14-dev llvm-13-dev llvm-12-dev firefox temurin-8-jdk \
+            powershell mysql-server-core-8.0 libllvm15 libllvm14 libllvm13 libllvm12 libclang-common-13-dev
+          sudo apt autoremove -y
+          sudo apt autoclean -y
+
+      - name: Check new disk space
+        if: ${{ env.RUN_TESTS }}
+        run: |
+          df -h
+      - name: List remaning installed packages
+        if: ${{ env.RUN_TESTS }}
+        run: |
+          dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}'
+          apt list --installed | wc -l
+
       - name: Checkout code
-        uses: actions/[email protected]
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
 
       - name: Set up Docker Buildx
         id: builder
-        uses: docker/setup-buildx-action@master
+        uses: docker/setup-buildx-action@v3
 
       - name: "Append ARM buildx builder from Oracle cloud"
+        if: ${{ matrix.platform == 'linux/arm64'}}
         uses: baschny/append-buildx-action@v1
         with:
           builder: ${{ steps.builder.outputs.name }}
           endpoint: ssh://${{ secrets.ARM_SSH_CONNECTION_STRING }}
           ssh_private_key: ${{ secrets.ARM_SSH_PRIVATE_KEY }}
 
-      - name: Extract branch name
-        shell: bash
-        run: |
-          BRANCH=${GITHUB_REF#refs/heads/}
-          SANITIZED_BRANCH=${BRANCH//"/"/-}
-          
-          echo $BRANCH
-          echo $SANITIZED_BRANCH
-          
-          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
-          echo "tag=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT
-        id: extract_branch
-
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY_IMAGE }}
+          tags: |
+            type=edge,branch=main
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+      
       - name: Login to Docker registry
-        uses: docker/login-action@v2
+        if: ${{ env.BUILD_ONLY != 'true' }}
+        uses: docker/login-action@v3
         with:
-          username: giskardai
+          username: ${{ env.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push image based on the current branch
-        env:
-          IMAGE: docker.io/giskardai/giskard
+      - name: Install deps inside docker
+        if: ${{ env.RUN_TESTS }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: full-install-python
+          push: false
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          builder: ${{ steps.builder.outputs.name }}
+          platforms: |
+            ${{ matrix.platform}}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run python test inside docker
+        if: ${{ env.RUN_TESTS }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: test-python
+          push: false
+          load: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          builder: ${{ steps.builder.outputs.name }}
+          platforms: |
+            ${{ matrix.platform}}
+          cache-from: type=gha
+
+      # - name: Run python integration test inside docker
+      #   if: ${{ env.RUN_TESTS }}
+      #   uses: docker/build-push-action@v5
+      #   with:
+      #     context: .
+      #     target: integration-test-python
+      #     push: false
+      #     load: false
+      #     tags: ${{ steps.meta.outputs.tags }}
+      #     labels: ${{ steps.meta.outputs.labels }}
+      #     builder: ${{ steps.builder.outputs.name }}
+      #     platforms: |
+      #       ${{ matrix.platform}}
+      #     cache-from: type=gha
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: prod
+          push: ${{ env.BUILD_ONLY != 'true' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          builder: ${{ steps.builder.outputs.name }}
+          platforms: |
+            ${{ matrix.platform}}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ env.BUILD_ONLY != 'true' }}
+
+      # For details, see link below
+      # https://docs.docker.com/build/ci/github-actions/multi-platform/
+      - name: Export digest
         run: |
-          docker buildx build \
-          --platform linux/amd64,linux/arm64 \
-          --tag $IMAGE:${{ steps.extract_branch.outputs.tag }} \
-          --file Dockerfile \
-          --push \
-          .
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
 
-      - name: Invoke deployment hook
-        if: ${{ steps.extract_branch.outputs.branch == 'main' }}
+  merge:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' && !inputs.build_only }}
+    needs:
+      - build-images
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
         run: |
-          curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}          
+        
+
+      # - name: Invoke deployment hook
+      #   if: ${{ steps.extract_branch.outputs.branch == 'main' && env.BUILD_ONLY != 'true' }}
+      #   run: |
+      #     curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev