convert.yml

name: Convert & Check Images

on:
  schedule:
    # Do conversion every day at 00:03 clock UTC
    - cron: "3 0 * * *"
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always
  REGISTRY: ghcr.io
  ORGANIZATION: ${{ github.repository }}
  IMAGE_LIST_PATH: misc/top_images/image_list.txt
  FSCK_PATCH_PATH: misc/top_images/fsck.patch

jobs:
  nydusify-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Setup Golang
      uses: actions/setup-go@v4
      with:
        go-version-file: 'go.work'
        cache-dependency-path: "**/*.sum"
    - name: Build Contrib
      run: |
        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.54.2
        make -e DOCKER=false nydusify-release
    - name: Upload Nydusify
      uses: actions/upload-artifact@v3
      with:
        name: nydusify-artifact
        path: contrib/nydusify/cmd/nydusify

  nydus-build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    - name: Rust Cache
      uses: Swatinem/rust-cache@v2.7.0
      with:
        cache-on-failure: true
        shared-key: Linux-cargo-amd64
    - uses: dsherret/rust-toolchain-file@v1
    - name: Build Nydus
      run: |
        make release
    - name: Upload Nydus Binaries
      uses: actions/upload-artifact@v3
      with:
        name: nydus-artifact
        path: |
          target/release/nydus-image
          target/release/nydusd

  fsck-erofs-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Build fsck.erofs
        run: |
          sudo apt-get update && sudo apt-get install -y build-essential git autotools-dev automake libtool pkg-config uuid-dev liblz4-dev
          git clone https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git
          cd erofs-utils && git checkout v1.6 && git apply ../${{ env.FSCK_PATCH_PATH }} && ./autogen.sh && ./configure && make && cd ..
          sudo cp erofs-utils/fsck/fsck.erofs /usr/local/bin/
      - name: Upload fsck.erofs
        uses: actions/upload-artifact@v3
        with:
          name: fsck-erofs-artifact
          path: |
            /usr/local/bin/fsck.erofs

  convert-zran:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build, fsck-erofs-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@v3
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@v3
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Download fsck.erofs
        uses: actions/download-artifact@v3
        with:
          name: fsck-erofs-artifact
          path: /usr/local/bin
      - name: Convert and check zran images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo chmod +x /usr/local/bin/fsck.erofs
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-zran
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-oci-ref"
            ghcr_repo=${{ env.REGISTRY }}/${{ env.ORGANIZATION }}

            # push oci image to ghcr/local for zran reference
            sudo docker pull $I:latest
            sudo docker tag $I:latest $ghcr_repo/$I
            sudo docker tag $I:latest localhost:5000/$I
            sudo DOCKER_CONFIG=$HOME/.docker docker push $ghcr_repo/$I
            sudo docker push localhost:5000/$I

            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --oci-ref \
                 --source $ghcr_repo/$I \
                 --target $ghcr_repo/$I:nydus-nightly-oci-ref \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --oci-ref \
                 --source localhost:5000/$I \
                 --target localhost:5000/$I:nydus-nightly-oci-ref \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-zran/${I}.json

            # check zran image and referenced oci image
            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check \
                --source localhost:5000/$I \
                --target localhost:5000/$I:nydus-nightly-oci-ref

            sudo fsck.erofs -d1 output/nydus_bootstrap
            sudo rm -rf ./output
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-zran-metric
          path: convert-zran

  convert-native-v5:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@v3
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@v3
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Convert and check RAFS v5 images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-native-v5
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-v5"
            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v5 \
                 --fs-version 5 \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target localhost:5000/$I:nydus-nightly-v5 \
                 --fs-version 5 \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-native-v5/${I}.json

            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
                --target localhost:5000/$I:nydus-nightly-v5
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-native-v5-metric
          path: convert-native-v5

  convert-native-v6:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build, fsck-erofs-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@v3
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@v3
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Download fsck.erofs
        uses: actions/download-artifact@v3
        with:
          name: fsck-erofs-artifact
          path: /usr/local/bin
      - name: Convert and check RAFS v6 images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo chmod +x /usr/local/bin/fsck.erofs
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-native-v6
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-v6"
            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v6 \
                 --fs-version 6 \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target localhost:5000/$I:nydus-nightly-v6 \
                 --fs-version 6 \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-native-v6/${I}.json

            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
                --target localhost:5000/$I:nydus-nightly-v6

            sudo fsck.erofs -d1 output/nydus_bootstrap
            sudo rm -rf ./output
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-native-v6-metric
          path: convert-native-v6

  convert-native-v6-batch:
    runs-on: ubuntu-latest
    needs: [nydusify-build, nydus-build, fsck-erofs-build]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Login ghcr registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Nydus
        uses: actions/download-artifact@master
        with:
          name: nydus-artifact
          path: /usr/local/bin
      - name: Download Nydusify
        uses: actions/download-artifact@master
        with:
          name: nydusify-artifact
          path: /usr/local/bin
      - name: Download fsck.erofs
        uses: actions/download-artifact@master
        with:
          name: fsck-erofs-artifact
          path: /usr/local/bin
      - name: Convert and check RAFS v6 batch images
        run: |
          sudo chmod +x /usr/local/bin/nydus*
          sudo chmod +x /usr/local/bin/fsck.erofs
          sudo docker run -d --restart=always -p 5000:5000 registry
          sudo mkdir convert-native-v6-batch
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            echo "converting $I:latest to $I:nydus-nightly-v6-batch"
            # for pre-built images
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v6-batch \
                 --fs-version 6 \
                 --batch-size 0x100000 \
                 --platform linux/amd64,linux/arm64

            # use local registry for speed
            sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
                 --source $I:latest \
                 --target localhost:5000/$I:nydus-nightly-v6-batch \
                 --fs-version 6 \
                 --batch-size 0x100000 \
                 --platform linux/amd64,linux/arm64 \
                 --output-json convert-native-v6-batch/${I}.json

            sudo rm -rf ./tmp
            sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
                --target localhost:5000/$I:nydus-nightly-v6-batch

            sudo fsck.erofs -d1 output/nydus_bootstrap
            sudo rm -rf ./output
          done
      - name: Save Nydusify Metric
        uses: actions/upload-artifact@v3
        with:
          name: convert-native-v6-batch-metric
          path: convert-native-v6-batch

  convert-metric:
    runs-on: ubuntu-latest
    needs: [convert-zran, convert-native-v5, convert-native-v6, convert-native-v6-batch]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Download Zran Metric
        uses: actions/download-artifact@v3
        with:
          name: convert-zran-metric
          path: convert-zran
      - name: Download V5 Metric
        uses: actions/download-artifact@v3
        with:
          name: convert-native-v5-metric
          path: convert-native-v5
      - name: Download V6 Metric
        uses: actions/download-artifact@v3
        with:
          name: convert-native-v6-metric
          path: convert-native-v6
      - name: Download V6 Batch Metric
        uses: actions/download-artifact@master
        with:
          name: convert-native-v6-batch-metric
          path: convert-native-v6-batch
      - name: Summary
        run: |
          echo "## Image Size(MB)" > $GITHUB_STEP_SUMMARY
          echo "> Compare the size of OCI image and Nydus image."
          echo "|image name|oci/nydus-zran|oci/nydus-v5|oci/nydus-v6|oci/nydus-batch|" >> $GITHUB_STEP_SUMMARY
          echo "|:--------:|:------------:|:----------:|:----------:|:-------------:|" >> $GITHUB_STEP_SUMMARY
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            zranSourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-zran/${I}.json) / 1048576")")
            zranTargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-zran/${I}.json) / 1048576")")
            v5SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v5/${I}.json) / 1048576")")
            v5TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v5/${I}.json) / 1048576")")
            v6SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v6/${I}.json) / 1048576")")
            v6TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v6/${I}.json) / 1048576")")
            batchSourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v6-batch/${I}.json) / 1048576")")
            batchTargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v6-batch/${I}.json) / 1048576")")
            echo "|${I}:latest|${zranSourceImageSize}/${zranTargetImageSize}|${v5SourceImageSize}/${v5TargetImageSize}|${v6SourceImageSize}/${v6TargetImageSize}|${batchSourceImageSize}/${batchTargetImageSize}|" >> $GITHUB_STEP_SUMMARY
          done
          echo "## Conversion Time(ms)" >> $GITHUB_STEP_SUMMARY
          echo "> Time elapsed to convert OCI image to Nydus image."
          echo "|image name|nydus-zran|nydus-v5|nydus-v6|nydus-batch|" >> $GITHUB_STEP_SUMMARY
          echo "|:---:|:--:|:-------:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY
          for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
            zranConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-zran/${I}.json) / 1000000")")
            v5ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v5/${I}.json) / 1000000")")
            v6ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6/${I}.json) / 1000000")")
            batchConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6-batch/${I}.json) / 1000000")")
            echo "|${I}:latest|${zranConversionElapsed}|${v5ConversionElapsed}|${v6ConversionElapsed}|${batchConversionElapsed}|" >> $GITHUB_STEP_SUMMARY
          done
      - uses: geekyeggo/delete-artifact@v2
        with:
          name: '*'