diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 0000000000..b6b2eb4b39 --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,74 @@ +name: 'Upload artifacts to release' +on: + release: + types: + - created + +jobs: + build-linux: + runs-on: ubuntu-latest + container: + image: vdaas/vald-ci-container:nightly + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 10 + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "::set-output name=version::${GO_VERSION}" + id: golang_version + - uses: actions/setup-go@v1 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: Build and zip + run: | + make binary/build/zip + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: artifacts-linux + path: ./artifacts/ + # build-macos: ## or using cross-compiler? + # runs-on: macos-latest + # steps: + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 10 + # - name: Fetch golang version + # run: | + # GO_VERSION=`make version/go` + # echo "::set-output name=version::${GO_VERSION}" + # id: golang_version + # - uses: actions/setup-go@v1 + # with: + # go-version: ${{ steps.golang_version.outputs.version }} + # - name: Build and zip + # run: | + # export PATH=$PATH:$(go env GOPATH)/bin + # brew install llvm libomp protobuf ngt + # make CXXFLAGS="-I/usr/local/opt/llvm/include -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl" binary/build/zip + # - name: Upload artifact + # uses: actions/upload-artifact@v2 + # with: + # name: artifacts-macos + # path: ./artifacts + publish: + runs-on: ubuntu-latest + needs: + - build-linux + # - build-macos + # - build-windows + steps: + - uses: actions/download-artifact@v2 + with: + name: artifacts-linux + path: tmp/linux + # - uses: actions/download-artifact@v2 + # with: + # name: artifacts-macos + # path: tmp/macos + - uses: shogo82148/actions-upload-release-asset@v1 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: tmp/linux/vald-*.zip diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index aaff1324b2..9f794dd0d4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,7 +14,7 @@ on: - '.github/workflows/codeql-analysis.yml' - '**.go' schedule: - - cron: '0 1 * * 2' + - cron: '0 1 * * *' jobs: CodeQL-Build: diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index d5b1c073b6..0a2b9e8d5a 100755 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -41,6 +41,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/agent-ngt @@ -65,6 +70,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/agent-ngt` docker push ${imagename}:latest @@ -73,6 +79,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml index 4372138d7d..ed511c5661 100644 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ b/.github/workflows/dockers-agent-sidecar-image.yml @@ -41,6 +41,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/agent-sidecar @@ -65,6 +70,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/agent-sidecar` docker push ${imagename}:latest @@ -73,6 +79,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-backup-manager-cassandra-image.yml b/.github/workflows/dockers-backup-manager-cassandra-image.yml index b1658e8867..5eaf5ea652 100644 --- a/.github/workflows/dockers-backup-manager-cassandra-image.yml +++ b/.github/workflows/dockers-backup-manager-cassandra-image.yml @@ -43,6 +43,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/backup-manager-cassandra @@ -67,6 +72,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/backup-manager-cassandra` docker push ${imagename}:latest @@ -75,6 +81,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-backup-manager-mysql-image.yml b/.github/workflows/dockers-backup-manager-mysql-image.yml index b90129b7ed..1b5e5e06d8 100644 --- a/.github/workflows/dockers-backup-manager-mysql-image.yml +++ b/.github/workflows/dockers-backup-manager-mysql-image.yml @@ -41,6 +41,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/backup-manager-mysql @@ -65,6 +70,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/backup-manager-mysql` docker push ${imagename}:latest @@ -73,6 +79,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml index d9024dcee9..4842a650e3 100755 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ b/.github/workflows/dockers-discoverer-k8s-image.yml @@ -37,6 +37,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/discoverer-k8s @@ -61,6 +66,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/discoverer-k8s` docker push ${imagename}:latest @@ -69,6 +75,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-gateway-vald-image.yml b/.github/workflows/dockers-gateway-vald-image.yml index 62d11d7322..2388008694 100755 --- a/.github/workflows/dockers-gateway-vald-image.yml +++ b/.github/workflows/dockers-gateway-vald-image.yml @@ -39,6 +39,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/gateway-vald @@ -63,6 +68,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/gateway-vald` docker push ${imagename}:latest @@ -71,6 +77,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index 41d147e672..2bee039531 100755 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -57,6 +57,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/operator/helm` docker push ${imagename}:latest @@ -65,6 +66,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml new file mode 100644 index 0000000000..3968e7bd6f --- /dev/null +++ b/.github/workflows/dockers-image-scan.yml @@ -0,0 +1,378 @@ +name: 'Docker image scanning' +on: + schedule: + - cron: '0 1 * * *' + +jobs: + agent-ngt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-ngt + imagename=`make docker/name/agent-ngt` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + agent-sidecar: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-sidecar + imagename=`make docker/name/agent-sidecar` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + discoverer-k8s: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/discoverer-k8s + imagename=`make docker/name/discoverer-k8s` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + gateway-vald: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/gateway-vald + imagename=`make docker/name/gateway-vald` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + meta-redis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/meta-redis + imagename=`make docker/name/meta-redis` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + meta-cassandra: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/meta-cassandra + imagename=`make docker/name/meta-cassandra` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + backup-manager-mysql: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/backup-manager-mysql + imagename=`make docker/name/backup-manager-mysql` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + backup-manager-cassandra: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/backup-manager-cassandra + imagename=`make docker/name/backup-manager-cassandra` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + compressor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/manager-compressor + imagename=`make docker/name/manager-compressor` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + manager-index: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/manager-index + imagename=`make docker/name/manager-index` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + operator-helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/operator/helm + imagename=`make docker/name/operator/helm` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' + loadtest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: Build the Docker image + id: build_image + run: | + make docker/build/loadtest + imagename=`make docker/name/loadtest` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "::set-output name=IMAGE_NAME::${imagename}" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'table' + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index 670c885963..9cab8a1d67 100755 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -39,6 +39,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/loadtest @@ -63,6 +68,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/loadtest` docker push ${imagename}:latest @@ -71,6 +77,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-manager-compressor-image.yml b/.github/workflows/dockers-manager-compressor-image.yml index 2b1fe96141..dc6799b220 100644 --- a/.github/workflows/dockers-manager-compressor-image.yml +++ b/.github/workflows/dockers-manager-compressor-image.yml @@ -39,6 +39,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/manager-compressor @@ -63,6 +68,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/manager-compressor` docker push ${imagename}:latest @@ -71,6 +77,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml index 7946a8fc21..ce7ed593ab 100644 --- a/.github/workflows/dockers-manager-index-image.yml +++ b/.github/workflows/dockers-manager-index-image.yml @@ -39,6 +39,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/manager-index @@ -63,6 +68,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/manager-index` docker push ${imagename}:latest @@ -71,6 +77,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-meta-cassandra-image.yml b/.github/workflows/dockers-meta-cassandra-image.yml index a36a93d3dd..16dafa9de2 100644 --- a/.github/workflows/dockers-meta-cassandra-image.yml +++ b/.github/workflows/dockers-meta-cassandra-image.yml @@ -41,6 +41,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/meta-cassandra @@ -65,6 +70,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/meta-cassandra` docker push ${imagename}:latest @@ -73,6 +79,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/.github/workflows/dockers-meta-redis-image.yml b/.github/workflows/dockers-meta-redis-image.yml index 712479ec04..2eb03aa12f 100755 --- a/.github/workflows/dockers-meta-redis-image.yml +++ b/.github/workflows/dockers-meta-redis-image.yml @@ -41,6 +41,11 @@ jobs: - uses: actions/checkout@v1 with: fetch-depth: 10 + - name: Overwrite version name + if: github.event_name == 'pull_request' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION - name: Build the Docker image run: | make docker/build/meta-redis @@ -65,6 +70,7 @@ jobs: docker push ${imagename}:pr-${pr_num} - name: push to DockerHub (tags) if: startsWith( github.ref, 'refs/tags/') + id: push_to_dockerhub_tags run: | imagename=`make docker/name/meta-redis` docker push ${imagename}:latest @@ -73,6 +79,30 @@ jobs: docker push ${imagename}:${tag_name} docker tag ${imagename} ${imagename}:nightly docker push ${imagename}:nightly + echo "::set-output name=IMAGE_NAME::${imagename}" + echo "::set-output name=TAG_NAME::${tag_name}" + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v1 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'table' + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.push_to_dockerhub_tags.outputs.IMAGE_NAME }}:${{ steps.push_to_dockerhub_tags.outputs.TAG_NAME }}" + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' slack: name: Slack notification needs: build diff --git a/Makefile b/Makefile index 26264c1cda..049d71d218 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ REPO ?= vdaas NAME = vald GOPKG = github.com/$(REPO)/$(NAME) +DATETIME = $(eval DATETIME := $(shell date -u +%Y/%m/%d_%H:%M:%S%z))$(DATETIME) TAG = $(eval TAG := $(shell date -u +%Y%m%d-%H%M%S))$(TAG) BASE_IMAGE = $(NAME)-base AGENT_IMAGE = $(NAME)-agent-ngt @@ -39,6 +40,8 @@ NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VER NGT_REPO = github.com/yahoojapan/NGT GO_VERSION := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION) +GOOS := $(eval GOOS := $(shell go env GOOS))$(GOOS) +GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH) GOPATH := $(eval GOPATH := $(shell go env GOPATH))$(GOPATH) GOCACHE := $(eval GOCACHE := $(shell go env GOCACHE))$(GOCACHE) @@ -47,9 +50,9 @@ TENSORFLOW_C_VERSION := $(eval TENSORFLOW_C_VERSION := $(shell cat versions/TENS OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION) KIND_VERSION ?= v0.8.1 -HELM_VERSION ?= v3.2.1 +HELM_VERSION ?= v3.2.4 HELM_DOCS_VERSION ?= 0.13.0 -VALDCLI_VERSION ?= v0.0.38 +VALDCLI_VERSION ?= v0.0.50 TELEPRESENCE_VERSION ?= 0.105 SWAP_DEPLOYMENT_TYPE ?= deployment @@ -59,16 +62,13 @@ SWAP_TAG ?= latest BINDIR ?= /usr/local/bin UNAME := $(eval UNAME := $(shell uname))$(UNAME) +CPU_INFO_FLAGS := $(eval CPU_INFO_FLAGS := $(shell cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1))$(CPU_INFO_FLAGS) +GIT_COMMIT := $(eval GIT_COMMIT := $(shell git rev-list -1 HEAD))$(GIT_COMMIT) MAKELISTS := Makefile $(shell find Makefile.d -type f -regex ".*\.mk") ROOTDIR = $(eval ROOTDIR := $(shell git rev-parse --show-toplevel))$(ROOTDIR) PROTODIRS := $(eval PROTODIRS := $(shell find apis/proto -type d | sed -e "s%apis/proto/%%g" | grep -v "apis/proto"))$(PROTODIRS) -PBGODIRS = $(PROTODIRS:%=apis/grpc/%) -SWAGGERDIRS = $(PROTODIRS:%=apis/swagger/%) -GRAPHQLDIRS = $(PROTODIRS:%=apis/graphql/%) -PBDOCDIRS = $(PROTODIRS:%=apis/docs/%) - BENCH_DATASET_BASE_DIR = hack/benchmark/assets BENCH_DATASET_MD5_DIR_NAME = checksum BENCH_DATASET_HDF5_DIR_NAME = dataset @@ -82,6 +82,9 @@ GRAPHQLS = $(PROTOS:apis/proto/%.proto=apis/graphql/%.pb.graphqls) GQLCODES = $(GRAPHQLS:apis/graphql/%.pb.graphqls=apis/graphql/%.generated.go) PBDOCS = $(PROTOS:apis/proto/%.proto=apis/docs/%.md) +CFLAGS ?= -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl +CXXFLAGS ?= $(CFLAGS) + BENCH_DATASET_MD5S := $(eval BENCH_DATASET_MD5S := $(shell find $(BENCH_DATASET_MD5_DIR) -type f -regex ".*\.md5"))$(BENCH_DATASET_MD5S) BENCH_DATASETS = $(BENCH_DATASET_MD5S:$(BENCH_DATASET_MD5_DIR)/%.md5=$(BENCH_DATASET_HDF5_DIR)/%.hdf5) @@ -143,9 +146,21 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ -regex '.*options?\.go' \ -not -name '*_test.go' \ -not -name 'doc.go'))$(GO_OPTION_SOURCES) + +GO_SOURCES_INTERNAL = $(eval GO_SOURCES_INTERNAL := $(shell find \ + ./internal \ + -type f \ + -name '*.go' \ + -not -name '*_test.go' \ + -not -name 'doc.go'))$(GO_SOURCES_INTERNAL) + GO_TEST_SOURCES = $(GO_SOURCES:%.go=%_test.go) GO_OPTION_TEST_SOURCES = $(GO_OPTION_SOURCES:%.go=%_test.go) +DISTROLESS_IMAGE ?= gcr.io/distroless/static +DISTROLESS_IMAGE_TAG ?= nonroot +UPX_OPTIONS ?= -9 + COMMA := , SHELL = bash @@ -248,7 +263,7 @@ format/yaml: deps: \ proto/deps \ goimports/install \ - prettier/install \ + prettier/install go mod tidy .PHONY: goimports/install @@ -302,6 +317,7 @@ ngt/install: /usr/local/include/NGT/Capi.h make install -C /tmp/NGT-$(NGT_VERSION) rm -rf v$(NGT_VERSION).tar.gz rm -rf /tmp/NGT-$(NGT_VERSION) + ldconfig .PHONY: tensorflow/install ## install TensorFlow for C @@ -335,6 +351,7 @@ changelog/next/print: @echo "$$BODY" include Makefile.d/bench.mk +include Makefile.d/build.mk include Makefile.d/docker.mk include Makefile.d/git.mk include Makefile.d/helm.mk diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk new file mode 100644 index 0000000000..80a6be0cbe --- /dev/null +++ b/Makefile.d/build.mk @@ -0,0 +1,390 @@ +# +# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: binary/build +## build all binaries +binary/build: \ + cmd/agent/core/ngt/ngt \ + cmd/agent/sidecar/sidecar \ + cmd/discoverer/k8s/discoverer \ + cmd/gateway/vald/vald \ + cmd/meta/redis/meta \ + cmd/meta/cassandra/meta \ + cmd/manager/backup/mysql/backup \ + cmd/manager/backup/cassandra/backup \ + cmd/manager/compressor/compressor \ + cmd/manager/index/index + +cmd/agent/core/ngt/ngt: \ + ngt/install \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/agent/core/ngt -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/agent/core/ngt ./pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CFLAGS="$(CFLAGS)" \ + && export CXXFLAGS="$(CXXFLAGS)" \ + && export CGO_ENABLED=1 \ + && export CGO_CXXFLAGS="-g -Ofast -march=native" \ + && export CGO_FFLAGS="-g -Ofast -march=native" \ + && export CGO_LDFLAGS="-g -Ofast -march=native" \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static -fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags "cgo netgo" \ + -trimpath \ + -installsuffix "cgo netgo" \ + -o $@ \ + $(dir $@)main.go + +cmd/agent/sidecar/sidecar: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/agent/sidecar -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/agent/sidecar ./pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -trimpath \ + -installsuffix netgo \ + -o $@ \ + $(dir $@)main.go + +cmd/discoverer/k8s/discoverer: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/discoverer/k8s -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/discoverer/k8s -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/gateway/vald/vald: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/gateway/vald -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/gateway/vald -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/meta/redis/meta: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/meta/redis -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/meta/redis -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/meta/cassandra/meta: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/meta/cassandra -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/meta/cassandra -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/backup/mysql/backup: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/backup/mysql -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/backup/mysql -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/backup/cassandra/backup: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/backup/cassandra -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/backup/cassandra -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -installsuffix netgo \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/compressor/compressor: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/compressor -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/compressor -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -trimpath \ + -installsuffix netgo \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/index/index: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/index -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/index -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -trimpath \ + -installsuffix netgo \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/replication/agent/agent: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/replication/agent -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/replication/agent -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -trimpath \ + -installsuffix netgo \ + -o $@ \ + $(dir $@)main.go + +cmd/manager/replication/controller/controller: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/manager/replication/controller -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/manager/replication/controller -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + export CGO_ENABLED=1 \ + && export GO111MODULE=on \ + && go build \ + --ldflags "-s -w -linkmode 'external' \ + -extldflags '-static' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)'" \ + -a \ + -tags netgo \ + -trimpath \ + -installsuffix netgo \ + -o $@ \ + $(dir $@)main.go + +.PHONY: binary/build/zip +## build all binaries and zip them +binary/build/zip: \ + artifacts/vald-agent-ngt-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-agent-sidecar-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-gateway-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-meta-redis-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-meta-cassandra-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-manager-backup-mysql-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-manager-backup-cassandra-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-manager-compressor-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-manager-index-$(GOOS)-$(GOARCH).zip + +artifacts/vald-agent-ngt-$(GOOS)-$(GOARCH).zip: cmd/agent/core/ngt/ngt + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-agent-sidecar-$(GOOS)-$(GOARCH).zip: cmd/agent/sidecar/sidecar + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip: cmd/discoverer/k8s/discoverer + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-gateway-$(GOOS)-$(GOARCH).zip: cmd/gateway/vald/vald + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-meta-redis-$(GOOS)-$(GOARCH).zip: cmd/meta/redis/meta + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-meta-cassandra-$(GOOS)-$(GOARCH).zip: cmd/meta/cassandra/meta + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-manager-backup-mysql-$(GOOS)-$(GOARCH).zip: cmd/manager/backup/mysql/backup + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-manager-backup-cassandra-$(GOOS)-$(GOARCH).zip: cmd/manager/backup/cassandra/backup + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-manager-compressor-$(GOOS)-$(GOARCH).zip: cmd/manager/compressor/compressor + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-manager-index-$(GOOS)-$(GOARCH).zip: cmd/manager/index/index + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 208b1eea00..5c1d02b7d6 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -45,7 +45,12 @@ docker/name/agent-ngt: .PHONY: docker/build/agent-ngt ## build agent-ngt image docker/build/agent-ngt: docker/build/base - docker build -f dockers/agent/core/ngt/Dockerfile -t $(REPO)/$(AGENT_IMAGE) . + docker build \ + -f dockers/agent/core/ngt/Dockerfile \ + -t $(REPO)/$(AGENT_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/agent-sidecar docker/name/agent-sidecar: @@ -54,7 +59,12 @@ docker/name/agent-sidecar: .PHONY: docker/build/agent-sidecar ## build agent-sidecar image docker/build/agent-sidecar: docker/build/base - docker build -f dockers/agent/sidecar/Dockerfile -t $(REPO)/$(AGENT_SIDECAR_IMAGE) . + docker build \ + -f dockers/agent/sidecar/Dockerfile \ + -t $(REPO)/$(AGENT_SIDECAR_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/discoverer-k8s docker/name/discoverer-k8s: @@ -63,7 +73,12 @@ docker/name/discoverer-k8s: .PHONY: docker/build/discoverer-k8s ## build discoverer-k8s image docker/build/discoverer-k8s: docker/build/base - docker build -f dockers/discoverer/k8s/Dockerfile -t $(REPO)/$(DISCOVERER_IMAGE) . + docker build \ + -f dockers/discoverer/k8s/Dockerfile \ + -t $(REPO)/$(DISCOVERER_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/gateway-vald docker/name/gateway-vald: @@ -72,7 +87,12 @@ docker/name/gateway-vald: .PHONY: docker/build/gateway-vald ## build gateway-vald image docker/build/gateway-vald: docker/build/base - docker build -f dockers/gateway/vald/Dockerfile -t $(REPO)/$(GATEWAY_IMAGE) . + docker build \ + -f dockers/gateway/vald/Dockerfile \ + -t $(REPO)/$(GATEWAY_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/meta-redis docker/name/meta-redis: @@ -81,7 +101,12 @@ docker/name/meta-redis: .PHONY: docker/build/meta-redis ## build meta-redis image docker/build/meta-redis: docker/build/base - docker build -f dockers/meta/redis/Dockerfile -t $(REPO)/$(META_REDIS_IMAGE) . + docker build \ + -f dockers/meta/redis/Dockerfile \ + -t $(REPO)/$(META_REDIS_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/meta-cassandra docker/name/meta-cassandra: @@ -90,7 +115,12 @@ docker/name/meta-cassandra: .PHONY: docker/build/meta-cassandra ## build meta-cassandra image docker/build/meta-cassandra: docker/build/base - docker build -f dockers/meta/cassandra/Dockerfile -t $(REPO)/$(META_CASSANDRA_IMAGE) . + docker build \ + -f dockers/meta/cassandra/Dockerfile \ + -t $(REPO)/$(META_CASSANDRA_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/backup-manager-mysql docker/name/backup-manager-mysql: @@ -99,7 +129,12 @@ docker/name/backup-manager-mysql: .PHONY: docker/build/backup-manager-mysql ## build backup-manager-mysql image docker/build/backup-manager-mysql: docker/build/base - docker build -f dockers/manager/backup/mysql/Dockerfile -t $(REPO)/$(MANAGER_BACKUP_MYSQL_IMAGE) . + docker build \ + -f dockers/manager/backup/mysql/Dockerfile \ + -t $(REPO)/$(MANAGER_BACKUP_MYSQL_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/backup-manager-cassandra docker/name/backup-manager-cassandra: @@ -108,7 +143,12 @@ docker/name/backup-manager-cassandra: .PHONY: docker/build/backup-manager-cassandra ## build backup-manager-cassandra image docker/build/backup-manager-cassandra: docker/build/base - docker build -f dockers/manager/backup/cassandra/Dockerfile -t $(REPO)/$(MANAGER_BACKUP_CASSANDRA_IMAGE) . + docker build \ + -f dockers/manager/backup/cassandra/Dockerfile \ + -t $(REPO)/$(MANAGER_BACKUP_CASSANDRA_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/manager-compressor docker/name/manager-compressor: @@ -117,7 +157,12 @@ docker/name/manager-compressor: .PHONY: docker/build/manager-compressor ## build manager-compressor image docker/build/manager-compressor: docker/build/base - docker build -f dockers/manager/compressor/Dockerfile -t $(REPO)/$(MANAGER_COMPRESSOR_IMAGE) . + docker build \ + -f dockers/manager/compressor/Dockerfile \ + -t $(REPO)/$(MANAGER_COMPRESSOR_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/manager-index docker/name/manager-index: @@ -126,7 +171,12 @@ docker/name/manager-index: .PHONY: docker/build/manager-index ## build manager-index image docker/build/manager-index: docker/build/base - docker build -f dockers/manager/index/Dockerfile -t $(REPO)/$(MANAGER_INDEX_IMAGE) . + docker build \ + -f dockers/manager/index/Dockerfile \ + -t $(REPO)/$(MANAGER_INDEX_IMAGE) . \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg UPX_OPTIONS=$(UPX_OPTIONS) .PHONY: docker/name/ci-container docker/name/ci-container: diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk index a44a372510..129648244d 100644 --- a/Makefile.d/proto.mk +++ b/Makefile.d/proto.mk @@ -68,7 +68,8 @@ proto/deps: \ $(GOPATH)/bin/swagger \ $(GOPATH)/src/google.golang.org/genproto \ $(GOPATH)/src/github.com/protocolbuffers/protobuf \ - $(GOPATH)/src/github.com/googleapis/googleapis + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate $(GOPATH)/src/github.com/protocolbuffers/protobuf: git clone \ @@ -82,6 +83,12 @@ $(GOPATH)/src/github.com/googleapis/googleapis: https://github.com/googleapis/googleapis \ $(GOPATH)/src/github.com/googleapis/googleapis +$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate: + git clone \ + --depth 1 \ + https://github.com/envoyproxy/protoc-gen-validate \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate + $(GOPATH)/src/google.golang.org/genproto: $(call go-get, google.golang.org/genproto/...) @@ -119,7 +126,7 @@ $(GOPATH)/bin/protoc-gen-gqlgencfg: $(call go-get-no-mod, github.com/danielvladco/go-proto-gql/protoc-gen-gqlgencfg) $(GOPATH)/bin/protoc-gen-validate: - $(call go-get-no-mod, github.com/envoyproxy/protoc-gen-validate) + $(call go-get, github.com/envoyproxy/protoc-gen-validate) $(GOPATH)/bin/prototool: $(call go-get, github.com/uber/prototool/cmd/prototool) @@ -128,49 +135,134 @@ $(GOPATH)/bin/protoc-gen-doc: $(call go-get, github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc) $(GOPATH)/bin/swagger: - $(call go-get-no-mod, github.com/go-swagger/go-swagger/cmd/swagger) + $(call go-get, github.com/go-swagger/go-swagger/cmd/swagger) $(GOPATH)/bin/gqlgen: $(call go-get, github.com/99designs/gqlgen) -$(PBGODIRS): - $(call mkdir, $@) - $(call rm, -rf, $@/*) - -$(SWAGGERDIRS): - $(call mkdir, $@) - $(call rm, -rf, $@/*) - -$(GRAPHQLDIRS): - $(call mkdir, $@) - $(call rm, -rf, $@/*) - -$(PBDOCDIRS): - $(call mkdir, $@) - $(call rm, -rf, $@/*) - -$(PBPYDIRS): - $(call mkdir, $@) - $(call rm, -rf, $@/*) - -$(PBGOS): proto/deps $(PBGODIRS) +$(PBGOS): \ + $(PROTOS) \ + $(GOPATH)/bin/gqlgen \ + $(GOPATH)/bin/protoc-gen-doc \ + $(GOPATH)/bin/protoc-gen-go \ + $(GOPATH)/bin/protoc-gen-gogo \ + $(GOPATH)/bin/protoc-gen-gofast \ + $(GOPATH)/bin/protoc-gen-gogofast \ + $(GOPATH)/bin/protoc-gen-gogofaster \ + $(GOPATH)/bin/protoc-gen-gogoslick \ + $(GOPATH)/bin/protoc-gen-gogqlgen \ + $(GOPATH)/bin/protoc-gen-gql \ + $(GOPATH)/bin/protoc-gen-gqlgencfg \ + $(GOPATH)/bin/protoc-gen-grpc-gateway \ + $(GOPATH)/bin/protoc-gen-swagger \ + $(GOPATH)/bin/protoc-gen-validate \ + $(GOPATH)/bin/swagger \ + $(GOPATH)/src/google.golang.org/genproto \ + $(GOPATH)/src/github.com/protocolbuffers/protobuf \ + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate @$(call green, "generating pb.go files...") + $(call mkdir, $(dir $@)) $(call protoc-gen, $(patsubst apis/grpc/%.pb.go,apis/proto/%.proto,$@), --gogofast_out=plugins=grpc:$(GOPATH)/src) # we have to enable validate after https://github.com/envoyproxy/protoc-gen-validate/pull/257 is merged # $(call protoc-gen, $(patsubst apis/grpc/%.pb.go,apis/proto/%.proto,$@), --gogofast_out=plugins=grpc:$(GOPATH)/src --validate_out=lang=gogo:$(GOPATH)/src) -$(SWAGGERS): proto/deps $(SWAGGERDIRS) +$(SWAGGERS): \ + $(PROTOS) \ + $(GOPATH)/bin/gqlgen \ + $(GOPATH)/bin/protoc-gen-doc \ + $(GOPATH)/bin/protoc-gen-go \ + $(GOPATH)/bin/protoc-gen-gogo \ + $(GOPATH)/bin/protoc-gen-gofast \ + $(GOPATH)/bin/protoc-gen-gogofast \ + $(GOPATH)/bin/protoc-gen-gogofaster \ + $(GOPATH)/bin/protoc-gen-gogoslick \ + $(GOPATH)/bin/protoc-gen-gogqlgen \ + $(GOPATH)/bin/protoc-gen-gql \ + $(GOPATH)/bin/protoc-gen-gqlgencfg \ + $(GOPATH)/bin/protoc-gen-grpc-gateway \ + $(GOPATH)/bin/protoc-gen-swagger \ + $(GOPATH)/bin/protoc-gen-validate \ + $(GOPATH)/bin/swagger \ + $(GOPATH)/src/google.golang.org/genproto \ + $(GOPATH)/src/github.com/protocolbuffers/protobuf \ + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate @$(call green, "generating swagger.json files...") + $(call mkdir, $(dir $@)) $(call protoc-gen, $(patsubst apis/swagger/%.swagger.json,apis/proto/%.proto,$@), --swagger_out=json_names_for_fields=true:$(dir $@)) -$(GRAPHQLS): proto/deps $(GRAPHQLDIRS) +$(GRAPHQLS): \ + $(PROTOS) \ + $(GOPATH)/bin/gqlgen \ + $(GOPATH)/bin/protoc-gen-doc \ + $(GOPATH)/bin/protoc-gen-go \ + $(GOPATH)/bin/protoc-gen-gogo \ + $(GOPATH)/bin/protoc-gen-gofast \ + $(GOPATH)/bin/protoc-gen-gogofast \ + $(GOPATH)/bin/protoc-gen-gogofaster \ + $(GOPATH)/bin/protoc-gen-gogoslick \ + $(GOPATH)/bin/protoc-gen-gogqlgen \ + $(GOPATH)/bin/protoc-gen-gql \ + $(GOPATH)/bin/protoc-gen-gqlgencfg \ + $(GOPATH)/bin/protoc-gen-grpc-gateway \ + $(GOPATH)/bin/protoc-gen-swagger \ + $(GOPATH)/bin/protoc-gen-validate \ + $(GOPATH)/bin/swagger \ + $(GOPATH)/src/google.golang.org/genproto \ + $(GOPATH)/src/github.com/protocolbuffers/protobuf \ + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate @$(call green, "generating pb.graphqls files...") + $(call mkdir, $(dir $@)) $(call protoc-gen, $(patsubst apis/graphql/%.pb.graphqls,apis/proto/%.proto,$@), --gql_out=paths=source_relative:$(dir $@)) -$(GQLCODES): proto/deps $(GRAPHQLS) +$(GQLCODES): \ + $(PROTOS) \ + $(GOPATH)/bin/gqlgen \ + $(GOPATH)/bin/protoc-gen-doc \ + $(GOPATH)/bin/protoc-gen-go \ + $(GOPATH)/bin/protoc-gen-gogo \ + $(GOPATH)/bin/protoc-gen-gofast \ + $(GOPATH)/bin/protoc-gen-gogofast \ + $(GOPATH)/bin/protoc-gen-gogofaster \ + $(GOPATH)/bin/protoc-gen-gogoslick \ + $(GOPATH)/bin/protoc-gen-gogqlgen \ + $(GOPATH)/bin/protoc-gen-gql \ + $(GOPATH)/bin/protoc-gen-gqlgencfg \ + $(GOPATH)/bin/protoc-gen-grpc-gateway \ + $(GOPATH)/bin/protoc-gen-swagger \ + $(GOPATH)/bin/protoc-gen-validate \ + $(GOPATH)/bin/swagger \ + $(GOPATH)/src/google.golang.org/genproto \ + $(GOPATH)/src/github.com/protocolbuffers/protobuf \ + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate @$(call green, "generating graphql generated.go files...") + $(call mkdir, $(dir $@)) sh hack/graphql/gqlgen.sh $(dir $@) $(patsubst apis/graphql/%.generated.go,apis/graphql/%.pb.graphqls,$@) $@ -$(PBDOCS): proto/deps $(PBDOCDIRS) +$(PBDOCS): \ + $(PROTOS) \ + $(GOPATH)/bin/gqlgen \ + $(GOPATH)/bin/protoc-gen-doc \ + $(GOPATH)/bin/protoc-gen-go \ + $(GOPATH)/bin/protoc-gen-gogo \ + $(GOPATH)/bin/protoc-gen-gofast \ + $(GOPATH)/bin/protoc-gen-gogofast \ + $(GOPATH)/bin/protoc-gen-gogofaster \ + $(GOPATH)/bin/protoc-gen-gogoslick \ + $(GOPATH)/bin/protoc-gen-gogqlgen \ + $(GOPATH)/bin/protoc-gen-gql \ + $(GOPATH)/bin/protoc-gen-gqlgencfg \ + $(GOPATH)/bin/protoc-gen-grpc-gateway \ + $(GOPATH)/bin/protoc-gen-swagger \ + $(GOPATH)/bin/protoc-gen-validate \ + $(GOPATH)/bin/swagger \ + $(GOPATH)/src/google.golang.org/genproto \ + $(GOPATH)/src/github.com/protocolbuffers/protobuf \ + $(GOPATH)/src/github.com/googleapis/googleapis \ + $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate @$(call green, "generating documents...") + $(call mkdir, $(dir $@)) $(call protoc-gen, $(patsubst apis/docs/%.md,apis/proto/%.proto,$@), --plugin=protoc-gen-doc=$(GOPATH)/bin/protoc-gen-doc --doc_opt=markdown$(COMMA)docs.md --doc_out=$(dir $@)) diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index f09b7abf84..fe5050d1a2 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -37,72 +42,30 @@ COPY pkg/${PKG_INTERNAL} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . -ENV CFLAGS "-mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl" -ENV CXXFLAGS ${CFLAGS} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . -WORKDIR /tmp -COPY versions/NGT_VERSION . -RUN export NGT_VERSION="$(cat NGT_VERSION)" \ - && curl -LO "https://github.com/yahoojapan/NGT/archive/v${NGT_VERSION}.tar.gz" \ - && tar zxf "v${NGT_VERSION}.tar.gz" -C /tmp \ - && cd "/tmp/NGT-${NGT_VERSION}" \ - && cmake . \ - && make -j -C "/tmp/NGT-${NGT_VERSION}" \ - && make install -C "/tmp/NGT-${NGT_VERSION}" +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . -COPY versions/NGT_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && NGT_VERSION="$(cat NGT_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=1 \ - && CGO_CXXFLAGS="-g -Ofast -march=native" \ - CGO_FFLAGS="-g -Ofast -march=native" \ - CGO_LDFLAGS="-g -Ofast -march=native" \ - GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static -fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.NGTVersion=${NGT_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags "cgo netgo" \ - -trimpath \ - -installsuffix "cgo netgo" \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} RUN cp sample.yaml /tmp/config.yaml -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest -LABEL maintainer "kpango " +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME ngt -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/ngt"] diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index f72d93b8fe..373460da1f 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -37,49 +42,26 @@ COPY pkg/${PKG_INTERNAL} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -trimpath \ - -installsuffix netgo \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest -LABEL maintainer "kpango " + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME sidecar -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/sidecar"] diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index b1bc4709e9..6708722ef4 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -35,6 +35,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ npm \ jq \ sed \ + zip \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index 4c55e78194..48297e9cb7 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -33,49 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest -LABEL maintainer "kpango " + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME discoverer -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/discoverer"] diff --git a/dockers/gateway/vald/Dockerfile b/dockers/gateway/vald/Dockerfile index 3420066d58..d45c873d86 100644 --- a/dockers/gateway/vald/Dockerfile +++ b/dockers/gateway/vald/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME vald -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/vald"] diff --git a/dockers/manager/backup/cassandra/Dockerfile b/dockers/manager/backup/cassandra/Dockerfile index 599c5deae0..d5a907f36f 100644 --- a/dockers/manager/backup/cassandra/Dockerfile +++ b/dockers/manager/backup/cassandra/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald -ENV APP_NAME cassandra -ENV PKG manager/backup/${APP_NAME} +ENV PKG manager/backup/cassandra +ENV APP_NAME backup WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest - -ENV APP_NAME cassandra - -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " + +ENV APP_NAME backup + COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} -ENTRYPOINT ["/go/bin/cassandra"] +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/backup"] diff --git a/dockers/manager/backup/mysql/Dockerfile b/dockers/manager/backup/mysql/Dockerfile index cf012b700c..bcacf04bfd 100644 --- a/dockers/manager/backup/mysql/Dockerfile +++ b/dockers/manager/backup/mysql/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald -ENV APP_NAME mysql -ENV PKG manager/backup/${APP_NAME} +ENV PKG manager/backup/mysql +ENV APP_NAME backup WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest - -ENV APP_NAME mysql - -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " + +ENV APP_NAME backup + COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} -ENTRYPOINT ["/go/bin/mysql"] +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/backup"] diff --git a/dockers/manager/compressor/Dockerfile b/dockers/manager/compressor/Dockerfile index 91e6a56d9c..a7223bb05b 100644 --- a/dockers/manager/compressor/Dockerfile +++ b/dockers/manager/compressor/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald +ENV PKG manager/compressor ENV APP_NAME compressor -ENV PKG manager/${APP_NAME} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -trimpath \ - -installsuffix netgo \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME compressor -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/compressor"] diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index 8bfed6fdbf..4603697227 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald +ENV PKG manager/index ENV APP_NAME index -ENV PKG manager/${APP_NAME} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -trimpath \ - -installsuffix netgo \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME index -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/index"] diff --git a/dockers/manager/replication/agent/Dockerfile b/dockers/manager/replication/agent/Dockerfile index 505169b5fd..499cb1337c 100644 --- a/dockers/manager/replication/agent/Dockerfile +++ b/dockers/manager/replication/agent/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald +ENV PKG manager/replication/agent ENV APP_NAME agent -ENV PKG manager/replication/${APP_NAME} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -trimpath \ - -installsuffix netgo \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME agent -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/agent"] diff --git a/dockers/manager/replication/controller/Dockerfile b/dockers/manager/replication/controller/Dockerfile index 36f0099733..85b1baf00f 100644 --- a/dockers/manager/replication/controller/Dockerfile +++ b/dockers/manager/replication/controller/Dockerfile @@ -14,12 +14,17 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald +ENV PKG manager/replication/controller ENV APP_NAME controller -ENV PKG manager/replication/${APP_NAME} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -trimpath \ - -installsuffix netgo \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME controller -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/controller"] diff --git a/dockers/meta/cassandra/Dockerfile b/dockers/meta/cassandra/Dockerfile index 4a0694df68..6134e24553 100644 --- a/dockers/meta/cassandra/Dockerfile +++ b/dockers/meta/cassandra/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME meta -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/meta"] diff --git a/dockers/meta/redis/Dockerfile b/dockers/meta/redis/Dockerfile index b037b95788..1cff3b90f2 100644 --- a/dockers/meta/redis/Dockerfile +++ b/dockers/meta/redis/Dockerfile @@ -14,7 +14,12 @@ # limitations under the License. # +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG UPX_OPTIONS=-9 + FROM vdaas/vald-base:latest AS builder +ARG UPX_OPTIONS ENV ORG vdaas ENV REPO vald @@ -33,48 +38,26 @@ COPY pkg/${PKG} . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} COPY cmd/${PKG} . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -COPY versions/GO_VERSION . -COPY versions/VALD_VERSION . +COPY Makefile . COPY .git . -RUN GO_VERSION="$(cat GO_VERSION)" \ - && VALD_VERSION="$(cat VALD_VERSION)" \ - && GIT_COMMIT="$(git rev-list -1 HEAD)" \ - && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \ - && GOOS="$(go env GOOS)" \ - && GOARCH="$(go env GOARCH)" \ - && CGO_ENABLED=0 \ - && GO111MODULE=on \ - go build \ - --ldflags "-s -w -linkmode 'external' \ - -extldflags '-static' \ - -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \ - -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \ - -a \ - -tags netgo \ - -installsuffix netgo \ - -trimpath \ - -o "${APP_NAME}" \ - "cmd/${PKG}/main.go" \ - && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}" - -# Start From Scratch For Running Environment -FROM scratch -# Start From Alpine For Debug Environment -# FROM alpine:latest + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && upx ${UPX_OPTIONS} -o "/usr/bin/${APP_NAME}" "cmd/${PKG}/${APP_NAME}" + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer "Vald team " ENV APP_NAME meta -# Copy certificates for SSL/TLS -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Copy permissions -COPY --from=builder /etc/passwd /etc/passwd -# Copy our static executable COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +USER nonroot:nonroot + ENTRYPOINT ["/go/bin/meta"] diff --git a/go.mod b/go.mod index 1c1173f93e..a9447eab19 100755 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.33.21 github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1 github.com/cockroachdb/errors => github.com/cockroachdb/errors v1.5.1-0.20200617111016-cc0024f9c4d3 github.com/coreos/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 diff --git a/hack/go.mod.default b/hack/go.mod.default index d2c6575f85..2bcce5dd68 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -4,6 +4,7 @@ go 1.14 replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.33.21 github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1 github.com/cockroachdb/errors => github.com/cockroachdb/errors master github.com/coreos/etcd => go.etcd.io/etcd master diff --git a/internal/info/info.go b/internal/info/info.go index e80142cd39..5cd3d18f52 100644 --- a/internal/info/info.go +++ b/internal/info/info.go @@ -25,7 +25,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/vdaas/vald/internal/log" ) @@ -59,7 +58,7 @@ var ( GitCommit = "master" Organization = "vdaas" Repository = "vald" - BuildTime = time.Now().Format(time.RFC1123) + BuildTime = "" GoVersion string GoOS string