CI: Make docker builds fast again (#756)

* ♻️ refactor dockerfile and makefile.d/docker.mk Signed-off-by: Rintaro Okamura <[email protected]> 🐳 :tag: revise golang image tag Signed-off-by: Rintaro Okamura <[email protected]> * 💚 revise GitHub Actions yaml Signed-off-by: Rintaro Okamura <[email protected]> * 🎨 add .github/**/*.yml to format objects Signed-off-by: Rintaro Okamura <[email protected]> * 📝 update README table Signed-off-by: Rintaro Okamura <[email protected]> * 📝 update README: use badge for DockerHub Signed-off-by: Rintaro Okamura <[email protected]> * 📝 update README: use badge for ghcr.io Signed-off-by: Rintaro Okamura <[email protected]> * 📝 update CHANGELOG template Signed-off-by: Rintaro Okamura <[email protected]> * 📝 simplify README table Signed-off-by: Rintaro Okamura <[email protected]> * 🤖 Update license headers / Format go codes and yaml files Signed-off-by: vdaas-ci <[email protected]> Co-authored-by: Yusuke Kato <[email protected]> Co-authored-by: vdaas-ci <[email protected]>
vdaas · Oct 13, 2020 · c81622e · c81622e
1 parent 819ecb0
commit c81622e
Show file tree

Hide file tree

Showing 50 changed files with 2,575 additions and 2,543 deletions.
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -66,7 +66,6 @@ area/tools/cli/loadtest:
   - cmd/tools/cli/loadtest/**/*
   - pkg/tools/cli/loadtest/**/*
 
-
 area/internal:
   - internal/**/*
 

diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml
@@ -1,4 +1,4 @@
-name: 'Upload artifacts to release'
+name: "Upload artifacts to release"
 on:
   release:
     types:

diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml
@@ -5,11 +5,11 @@ on:
       - master
     paths:
       - "apis/proto/**"
-      - 'versions/GO_VERSION'
+      - "versions/GO_VERSION"
   pull_request:
     paths:
       - "apis/proto/**"
-      - 'versions/GO_VERSION'
+      - "versions/GO_VERSION"
 
 jobs:
   build:

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -5,16 +5,16 @@ on:
     branches:
       - master
     tags:
-       - '*.*.*'
-       - 'v*.*.*'
-       - '*.*.*-*'
-       - 'v*.*.*-*'
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
   pull_request:
     paths:
-      - '.github/workflows/codeql-analysis.yml'
-      - '**.go'
+      - ".github/workflows/codeql-analysis.yml"
+      - "**.go"
   schedule:
-    - cron: '0 1 * * *'
+    - cron: "0 1 * * *"
 
 jobs:
   CodeQL-Build:

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -1,19 +1,19 @@
-name: 'Coverage'
+name: "Coverage"
 on:
   push:
     branches:
       - master
     paths:
-      - '.github/workflows/coverage.yml'
-      - 'internal/**'
-      - 'pkg/**'
-      - 'cmd/**'
+      - ".github/workflows/coverage.yml"
+      - "internal/**"
+      - "pkg/**"
+      - "cmd/**"
   pull_request:
     paths:
-      - '.github/workflows/coverage.yml'
-      - 'internal/**'
-      - 'pkg/**'
-      - 'cmd/**'
+      - ".github/workflows/coverage.yml"
+      - "internal/**"
+      - "pkg/**"
+      - "cmd/**"
 
 jobs:
   coverage:

diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml
@@ -1,9 +1,9 @@
-name: 'Detect internal config changes'
+name: "Detect internal config changes"
 on:
   pull_request:
     paths:
-      - 'internal/config/**'
-      - '!internal/config/**/*_test.go'
+      - "internal/config/**"
+      - "!internal/config/**/*_test.go"
 
 jobs:
   warning:

diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml
@@ -1,160 +1,149 @@
-name: 'Build docker image: agent-ngt'
+name: "Build docker image: agent-ngt"
 on:
   push:
     branches:
       - master
     tags:
-       - '*.*.*'
-       - 'v*.*.*'
-       - '*.*.*-*'
-       - 'v*.*.*-*'
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
     paths:
-      - 'go.mod'
-      - 'go.sum'
-      - 'internal/**'
-      - '!internal/**/*_test.go'
-      - '!internal/db/**'
-      - '!internal/k8s/**'
-      - 'apis/grpc/**'
-      - 'pkg/agent/core/ngt/**'
-      - 'cmd/agent/core/ngt/**'
-      - 'dockers/base/Dockerfile'
-      - 'dockers/agent/core/ngt/Dockerfile'
-      - 'versions/GO_VERSION'
-      - 'versions/NGT_VERSION'
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/agent/core/ngt/**"
+      - "cmd/agent/core/ngt/**"
+      - "dockers/base/Dockerfile"
+      - "dockers/agent/core/ngt/Dockerfile"
+      - "versions/GO_VERSION"
+      - "versions/NGT_VERSION"
   pull_request:
     paths:
-      - 'go.mod'
-      - 'go.sum'
-      - 'internal/**'
-      - '!internal/**/*_test.go'
-      - '!internal/db/**'
-      - '!internal/k8s/**'
-      - 'apis/grpc/**'
-      - 'pkg/agent/core/ngt/**'
-      - 'cmd/agent/core/ngt/**'
-      - 'dockers/base/Dockerfile'
-      - 'dockers/agent/core/ngt/Dockerfile'
-      - 'versions/GO_VERSION'
-      - 'versions/NGT_VERSION'
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/agent/core/ngt/**"
+      - "cmd/agent/core/ngt/**"
+      - "dockers/base/Dockerfile"
+      - "dockers/agent/core/ngt/Dockerfile"
+      - "versions/GO_VERSION"
+      - "versions/NGT_VERSION"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup QEMU
-      uses: docker/setup-qemu-action@v1
-      with:
-        platforms: all
-    - name: Setup Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@v1
-      with:
-        buildkitd-flags: "--debug"
-    - name: Cache Docker layers (base)
-      uses: actions/cache@v2
-      with:
-        path: /tmp/.buildx-cache-base
-        key: ${{ runner.os }}-buildx-vald-base-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-vald-base-
-    - name: Cache Docker layers
-      uses: actions/cache@v2
-      with:
-        path: /tmp/.buildx-cache
-        key: ${{ runner.os }}-buildx-vald-agent-ngt-${{ github.sha }}
-        restore-keys: |
-          ${{ runner.os }}-buildx-vald-agent-ngt-
-    - name: Login to DockerHub
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_PASS }}
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ secrets.PACKAGE_USER }}
-        password: ${{ secrets.PACKAGE_TOKEN }}
-    - name: Image name
-      id: image_name
-      run: |
-        image_name=`make docker/name/agent-ngt`
-        base_platforms=`make docker/platforms`
-        echo "IMAGE_NAME=${image_name}" >> $GITHUB_ENV
-        echo "::set-output name=IMAGE_NAME::${image_name}"
-        echo "::set-output name=BASE_PLATFORMS::${base_platforms}"
-    - name: Determine tag name (master)
-      if: github.ref == 'refs/heads/master'
-      run: |
-        echo "PRIMARY_TAG=nightly" >> $GITHUB_ENV
-        echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-      env:
-        PLATFORMS: linux/amd64,linux/arm64
-    - name: Determine tag name (pull request)
-      if: github.event_name == 'pull_request'
-      run: |
-        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
-        echo "PR-${pr_num}" > versions/VALD_VERSION
-        echo "PRIMARY_TAG=pr-${pr_num}" >> $GITHUB_ENV
-        echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-      env:
-        PLATFORMS: linux/amd64
-    - name: Determine tag name (tags)
-      if: startsWith( github.ref, 'refs/tags/')
-      id: determine_tag
-      run: |
-        tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
-        echo "::set-output name=TAG_NAME::${tag_name}"
-        echo "PRIMARY_TAG=${tag_name}" >> $GITHUB_ENV
-        echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
-      env:
-        PLATFORMS: linux/amd64,linux/arm64
-    - name: Build and Push
-      run: |
-        make \
-          DOCKER="docker buildx" \
-          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} --push" \
-          DOCKER_OPTS_BASE="--platform ${BASE_PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS_BASE} --push" \
-          TAG="${PRIMARY_TAG}" \
-          docker/build/agent-ngt
-        make \
-          REPO="ghcr.io/vdaas/vald" \
-          DOCKER="docker buildx" \
-          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} --push" \
-          DOCKER_OPTS_BASE="--platform ${BASE_PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS_BASE} --push" \
-          TAG="${PRIMARY_TAG}" \
-          docker/build/agent-ngt
-      env:
-        DOCKER_BUILDKIT: 1
-        BASE_PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
-        BUILDER: ${{ steps.buildx.outputs.name }}
-        CACHE_OPTS: "--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache"
-        CACHE_OPTS_BASE: "--cache-from=type=local,src=/tmp/.buildx-cache-base"
-        LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
-    - name: Initialize CodeQL
-      if: startsWith( github.ref, 'refs/tags/')
-      uses: github/codeql-action/init@v1
-    - name: Run vulnerability scanner (table)
-      if: startsWith( github.ref, 'refs/tags/')
-      uses: aquasecurity/trivy-action@master
-      with:
-        image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
-        format: 'table'
-    - name: Run vulnerability scanner (sarif)
-      if: startsWith( github.ref, 'refs/tags/')
-      uses: aquasecurity/trivy-action@master
-      with:
-        image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
-        format: 'template'
-        template: '@/contrib/sarif.tpl'
-        output: 'trivy-results.sarif'
-    - name: Upload Trivy scan results to Security tab
-      if: startsWith( github.ref, 'refs/tags/')
-      uses: github/codeql-action/upload-sarif@v1
-      with:
-        sarif_file: 'trivy-results.sarif'
+      - uses: actions/checkout@v2
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          buildkitd-flags: "--debug"
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-vald-agent-ngt-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-vald-agent-ngt-
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.PACKAGE_USER }}
+          password: ${{ secrets.PACKAGE_TOKEN }}
+      - name: Image name
+        id: image_name
+        run: |
+          image_name=`make docker/name/agent-ngt`
+          base_platforms=`make docker/platforms`
+          echo "IMAGE_NAME=${image_name}" >> $GITHUB_ENV
+          echo "::set-output name=IMAGE_NAME::${image_name}"
+          echo "::set-output name=BASE_PLATFORMS::${base_platforms}"
+      - name: Determine tag name (master)
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "PRIMARY_TAG=nightly" >> $GITHUB_ENV
+          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
+        env:
+          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
+      - name: Determine tag name (pull request)
+        if: github.event_name == 'pull_request'
+        run: |
+          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+          echo "PR-${pr_num}" > versions/VALD_VERSION
+          echo "PRIMARY_TAG=pr-${pr_num}" >> $GITHUB_ENV
+          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
+        env:
+          PLATFORMS: linux/amd64
+      - name: Determine tag name (tags)
+        if: startsWith( github.ref, 'refs/tags/')
+        id: determine_tag
+        run: |
+          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
+          echo "::set-output name=TAG_NAME::${tag_name}"
+          echo "PRIMARY_TAG=${tag_name}" >> $GITHUB_ENV
+          echo "PLATFORMS=${PLATFORMS}" >> $GITHUB_ENV
+        env:
+          PLATFORMS: ${{ steps.image_name.outputs.BASE_PLATFORMS }}
+      - name: Build and Push
+        run: |
+          make \
+            DOCKER="docker buildx" \
+            DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} --push" \
+            TAG="${PRIMARY_TAG}" \
+            docker/build/agent-ngt
+          make \
+            REPO="ghcr.io/vdaas/vald" \
+            DOCKER="docker buildx" \
+            DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${CACHE_OPTS} ${LABEL_OPTS} --push" \
+            TAG="${PRIMARY_TAG}" \
+            docker/build/agent-ngt
+        env:
+          DOCKER_BUILDKIT: 1
+          BUILDER: ${{ steps.buildx.outputs.name }}
+          CACHE_OPTS: "--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,mode=max,dest=/tmp/.buildx-cache"
+          LABEL_OPTS: "--label org.opencontainers.image.url=${{ github.event.repository.html_url }} --label org.opencontainers.image.source=${{ github.event.repository.html_url }} --label org.opencontainers.image.revision=${{ github.sha }}"
+      - name: Initialize CodeQL
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: github/codeql-action/init@v1
+      - name: Run vulnerability scanner (table)
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          format: "table"
+      - name: Run vulnerability scanner (sarif)
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ steps.image_name.outputs.IMAGE_NAME }}:${{ steps.determine_tag.outputs.TAG_NAME }}"
+          format: "template"
+          template: "@/contrib/sarif.tpl"
+          output: "trivy-results.sarif"
+      - name: Upload Trivy scan results to Security tab
+        if: startsWith( github.ref, 'refs/tags/')
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: "trivy-results.sarif"
   slack:
     name: Slack notification
     needs: build