Merge branch 'master' into feature/benchmark/fix_load_test_using_inte…

…rnal_client
vdaas · Aug 19, 2020 · b429c56 · b429c56
2 parents 8d090d6 + cd8be5b
commit b429c56
Show file tree

Hide file tree

Showing 50 changed files with 3,346 additions and 2,667 deletions.
diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml
@@ -0,0 +1,74 @@
+name: 'Upload artifacts to release'
+on:
+  release:
+    types:
+      - created
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    container:
+      image: vdaas/vald-ci-container:nightly
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 10
+      - name: Fetch golang version
+        run: |
+          GO_VERSION=`make version/go`
+          echo "::set-output name=version::${GO_VERSION}"
+        id: golang_version
+      - uses: actions/setup-go@v1
+        with:
+          go-version: ${{ steps.golang_version.outputs.version }}
+      - name: Build and zip
+        run: |
+          make binary/build/zip
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifacts-linux
+          path: ./artifacts/
+  # build-macos: ## or using cross-compiler?
+  #   runs-on: macos-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #       with:
+  #         fetch-depth: 10
+  #     - name: Fetch golang version
+  #       run: |
+  #         GO_VERSION=`make version/go`
+  #         echo "::set-output name=version::${GO_VERSION}"
+  #       id: golang_version
+  #     - uses: actions/setup-go@v1
+  #       with:
+  #         go-version: ${{ steps.golang_version.outputs.version }}
+  #     - name: Build and zip
+  #       run: |
+  #         export PATH=$PATH:$(go env GOPATH)/bin
+  #         brew install llvm libomp protobuf ngt
+  #         make CXXFLAGS="-I/usr/local/opt/llvm/include -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl" binary/build/zip
+  #     - name: Upload artifact
+  #       uses: actions/upload-artifact@v2
+  #       with:
+  #         name: artifacts-macos
+  #         path: ./artifacts
+  publish:
+    runs-on: ubuntu-latest
+    needs:
+      - build-linux
+      # - build-macos
+      # - build-windows
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifacts-linux
+          path: tmp/linux
+      # - uses: actions/download-artifact@v2
+      #   with:
+      #     name: artifacts-macos
+      #     path: tmp/macos
+      - uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: tmp/linux/vald-*.zip
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -14,7 +14,7 @@ on:
       - '.github/workflows/codeql-analysis.yml'
       - '**.go'
   schedule:
-    - cron: '0 1 * * 2'
+    - cron: '0 1 * * *'
 
 jobs:
   CodeQL-Build:

diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml
@@ -41,6 +41,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/agent-ngt
@@ -65,6 +70,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/agent-ngt`
         docker push ${imagename}:latest
@@ -73,6 +79,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build

diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml
@@ -41,6 +41,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/agent-sidecar
@@ -65,6 +70,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/agent-sidecar`
         docker push ${imagename}:latest
@@ -73,6 +79,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build

diff --git a/.github/workflows/dockers-backup-manager-cassandra-image.yml b/.github/workflows/dockers-backup-manager-cassandra-image.yml
@@ -43,6 +43,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/backup-manager-cassandra
@@ -67,6 +72,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/backup-manager-cassandra`
         docker push ${imagename}:latest
@@ -75,6 +81,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build

diff --git a/.github/workflows/dockers-backup-manager-mysql-image.yml b/.github/workflows/dockers-backup-manager-mysql-image.yml
@@ -41,6 +41,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/backup-manager-mysql
@@ -65,6 +70,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/backup-manager-mysql`
         docker push ${imagename}:latest
@@ -73,6 +79,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build

diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml
@@ -37,6 +37,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/discoverer-k8s
@@ -61,6 +66,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/discoverer-k8s`
         docker push ${imagename}:latest
@@ -69,6 +75,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build

diff --git a/.github/workflows/dockers-gateway-vald-image.yml b/.github/workflows/dockers-gateway-vald-image.yml
@@ -39,6 +39,11 @@ jobs:
     - uses: actions/checkout@v1
       with:
         fetch-depth: 10
+    - name: Overwrite version name
+      if: github.event_name == 'pull_request'
+      run: |
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        echo "PR-${pr_num}" > versions/VALD_VERSION
     - name: Build the Docker image
       run: |
         make docker/build/gateway-vald
@@ -63,6 +68,7 @@ jobs:
         docker push ${imagename}:pr-${pr_num}
     - name: push to DockerHub (tags)
       if: startsWith( github.ref, 'refs/tags/')
+      id: push_to_dockerhub_tags
       run: |
         imagename=`make docker/name/gateway-vald`
         docker push ${imagename}:latest
@@ -71,6 +77,30 @@ jobs:
         docker push ${imagename}:${tag_name}
         docker tag ${imagename} ${imagename}:nightly
         docker push ${imagename}:nightly
+        echo "::set-output name=IMAGE_NAME::${imagename}"
+        echo "::set-output name=TAG_NAME::${tag_name}"
+    - name: Initialize CodeQL
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/init@v1
+    - name: Run vulnerability scanner (table)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'table'
+    - name: Run vulnerability scanner (sarif)
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}"
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to Security tab
+      if: startsWith( github.ref, 'refs/tags/')
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
   slack:
     name: Slack notification
     needs: build