From 3589baee7e734c331987bf9e014411be03179a59 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 14:09:31 +0900 Subject: [PATCH 001/139] add action to dump context Signed-off-by: hlts2 --- .github/actions/dump-context/action.yaml | 36 +++++++++++++++++++ .github/workflows/build-binaries.yml | 28 ++------------- .github/workflows/build-protobuf.yml | 28 ++------------- .github/workflows/chatops-help.yml | 29 +++------------ .github/workflows/chatops.yml | 32 ++++------------- .github/workflows/codeql-analysis.yml | 28 ++------------- .github/workflows/coverage.yml | 28 ++------------- .../detect-internal-config-changes.yml | 28 ++------------- .github/workflows/dockers-agent-ngt-image.yml | 28 ++------------- .../workflows/dockers-agent-sidecar-image.yml | 28 ++------------- .../workflows/dockers-ci-container-image.yml | 28 ++------------- .../workflows/dockers-dev-container-image.yml | 28 ++------------- .../dockers-discoverer-k8s-image.yml | 28 ++------------- .../dockers-gateway-filter-image.yml | 28 ++------------- .../workflows/dockers-gateway-lb-image.yml | 28 ++------------- .../workflows/dockers-helm-operator-image.yml | 28 ++------------- .github/workflows/dockers-image-scan.yml | 28 ++------------- .github/workflows/dockers-loadtest-image.yml | 28 ++------------- .../workflows/dockers-manager-index-image.yml | 28 ++------------- .github/workflows/e2e-chaos.yaml | 28 ++------------- .github/workflows/e2e-code-bench-agent.yaml | 28 ++------------- .github/workflows/e2e-deploy.yml | 28 ++------------- .github/workflows/e2e-max-dim.yml | 28 ++------------- .github/workflows/e2e-profiling.yml | 28 ++------------- .github/workflows/format.yml | 29 +++------------ .github/workflows/fossa.yml | 28 ++------------- .github/workflows/helm-lint.yml | 28 ++------------- .github/workflows/helm.yml | 28 ++------------- .github/workflows/labeler.yml | 28 ++------------- .github/workflows/reviewdog-hadolint.yml | 28 ++------------- .github/workflows/reviewdog-k8s.yml | 28 ++------------- .github/workflows/reviewdog-markdown.yml | 28 ++------------- .github/workflows/reviewdog.yml | 28 ++------------- .github/workflows/semver.yml | 28 ++------------- .github/workflows/test-hack.yml | 28 ++------------- .github/workflows/test.yml | 28 ++------------- .github/workflows/update-protobuf.yml | 28 ++------------- ...update_pull_request_and_issue_template.yml | 28 ++------------- 38 files changed, 152 insertions(+), 926 deletions(-) create mode 100644 .github/actions/dump-context/action.yaml diff --git a/.github/actions/dump-context/action.yaml b/.github/actions/dump-context/action.yaml new file mode 100644 index 0000000000..442e2821a4 --- /dev/null +++ b/.github/actions/dump-context/action.yaml @@ -0,0 +1,36 @@ +name: "Dump context to log" +description: "Dump context to log" + +runs: + using: "composite" + steps: + - name: "Dump GitHub context" + shell: bash + run: echo $JSON + env: + JSON: ${{toJSON(github)}} + - name: "Dump job context" + shell: bash + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: "Dump steps context" + shell: bash + run: echo $JSON + env: + JSON: ${{toJSON(steps)}} + - name: "Dump runner context" + shell: bash + run: echo $JSON + env: + JSON: ${{toJSON(runner)}} + - name: "Dump strategy context" + shell: bash + run: echo $JSON + env: + JSON: ${{toJSON(strategy)}} + - name: "Dump matrix context" + shell: bash + run: echo $JSON + env: + JSON: ${{toJSON(matrix)}} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 2f68eb5ba0..541ef1fcd8 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -23,31 +23,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build-linux: runs-on: ubuntu-latest container: diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index ced2f7cdf2..334c0dce58 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -30,31 +30,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: name: build all runs-on: ubuntu-latest diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index 42478c5fbb..af699cbdac 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -17,35 +17,14 @@ name: "ChatOps help" on: pull_request: types: [opened] + jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + print-help: name: print help runs-on: ubuntu-latest diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index d3c77552ed..fe37956172 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -13,41 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # +name: ChatOps on: issue_comment: types: [created] + env: YQ_VERSION: 3.2.1 -name: ChatOps + jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + label: name: Add labels runs-on: ubuntu-latest diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e95b13890c..1a35c9dd04 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,31 +37,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + CodeQL-Build: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 78c7abeace..05659811be 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -34,31 +34,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + coverage: name: Coverage runs-on: ubuntu-latest diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index 4ff5904d7a..ab6cc38c91 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -24,31 +24,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + warning: name: warn internal config changes runs-on: ubuntu-latest diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index 188ad2b517..1ebacd2ef6 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -79,31 +79,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml index f067deb575..75b21935c8 100644 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ b/.github/workflows/dockers-agent-sidecar-image.yml @@ -79,31 +79,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml index d6e6636ba1..ed3dd19b60 100644 --- a/.github/workflows/dockers-ci-container-image.yml +++ b/.github/workflows/dockers-ci-container-image.yml @@ -53,31 +53,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml index a3ec496967..376a813d6c 100644 --- a/.github/workflows/dockers-dev-container-image.yml +++ b/.github/workflows/dockers-dev-container-image.yml @@ -46,31 +46,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml index 767c1b6a6c..7207c2f2eb 100644 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ b/.github/workflows/dockers-discoverer-k8s-image.yml @@ -73,31 +73,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-gateway-filter-image.yml b/.github/workflows/dockers-gateway-filter-image.yml index 58e2a91628..a9bbc01684 100644 --- a/.github/workflows/dockers-gateway-filter-image.yml +++ b/.github/workflows/dockers-gateway-filter-image.yml @@ -82,31 +82,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-gateway-lb-image.yml b/.github/workflows/dockers-gateway-lb-image.yml index 4a4b7c0a7c..c1840b70a8 100644 --- a/.github/workflows/dockers-gateway-lb-image.yml +++ b/.github/workflows/dockers-gateway-lb-image.yml @@ -82,31 +82,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index 6593b02653..cd7d2a7fc9 100644 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -67,31 +67,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index f6dd2c1581..a8f063a178 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -26,31 +26,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + agent-ngt: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index e87654247c..97ab72f69b 100644 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -76,31 +76,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml index f20f561434..b53d7a5d89 100644 --- a/.github/workflows/dockers-manager-index-image.yml +++ b/.github/workflows/dockers-manager-index-image.yml @@ -76,31 +76,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + build: strategy: max-parallel: 4 diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index d9cf6b5839..3b81f488c3 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -29,31 +29,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + agent-failure: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index a996e58921..422a6c900a 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -46,31 +46,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + grpc-sequential: name: grpc-sequential runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 57678708f7..a23c7f39f5 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -28,31 +28,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + e2e-stream-crud: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 701dd343ed..0ab0c58bf9 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -29,31 +29,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + e2e-max-dimension-insert: name: "E2E test (Max Dimension Insert: skip strict exist check)" runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index d8faf01554..5edbfea183 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -29,31 +29,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + e2e-profiling: name: "E2E profiling" runs-on: ubuntu-latest diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 083a064214..ca2ba48adc 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -18,35 +18,14 @@ on: push: branches: - main + jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + format: name: Run formatter runs-on: ubuntu-latest diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 72658c978c..86c715c277 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -26,31 +26,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + fossa-scan: name: "trigger FOSSA scan" runs-on: ubuntu-latest diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 7836e5f88d..6276e8c52b 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -23,31 +23,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + lint-vald-chart: name: lint for vald chart runs-on: ubuntu-latest diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 5bad5d0ea5..369d6cd2d2 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -26,31 +26,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + update-k8s-manifest: name: Update k8s manifest runs-on: ubuntu-latest diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1f22df3926..fb2ab40a20 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -21,31 +21,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + triage: name: Triage runs-on: ubuntu-latest diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index 6d8403b3f6..a725a1aaf3 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -24,31 +24,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + hadolint: name: runner / hadolint runs-on: ubuntu-latest diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index 5d233404df..ae87d86852 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -25,31 +25,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + kubeval-conflint: name: runner / kubeval-conflint runs-on: ubuntu-latest diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index ad84dc315f..c21b926444 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -25,31 +25,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + languagetool_apis_charts: name: runner / LanguageTool / apis & charts runs-on: ubuntu-latest diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index c0777b7d4f..ee16a468c7 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -24,31 +24,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + golangci-lint: name: runner / golangci-lint runs-on: ubuntu-latest diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 8a1063b5d3..2f960ddfd4 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -23,31 +23,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + semver-auto: name: Run semver-auto runs-on: ubuntu-latest diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 54c4c929b3..e396b37f69 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -40,31 +40,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + test-hack: name: Run tests for hack packages runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f17b061a9..0e2c7347a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,31 +38,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + test-cmd: name: Run tests for cmd packages runs-on: ubuntu-latest diff --git a/.github/workflows/update-protobuf.yml b/.github/workflows/update-protobuf.yml index 67455803f3..121eaa6d06 100644 --- a/.github/workflows/update-protobuf.yml +++ b/.github/workflows/update-protobuf.yml @@ -26,31 +26,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + dispatches: name: Dispatches runs-on: ubuntu-latest diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update_pull_request_and_issue_template.yml index ac7ba7fa86..adc62b940d 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -27,31 +27,9 @@ jobs: dump_contexts_to_log: runs-on: ubuntu-latest steps: - - name: Dump GitHub context - id: github_context_step - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - name: Dump job context - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - name: Dump steps context - run: echo $JSON - env: - JSON: ${{ toJSON(steps) }} - - name: Dump runner context - run: echo $JSON - env: - JSON: ${{ toJSON(runner) }} - - name: Dump strategy context - run: echo $JSON - env: - JSON: ${{ toJSON(strategy) }} - - name: Dump matrix context - run: echo $JSON - env: - JSON: ${{ toJSON(matrix) }} + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + update-pr-issue-template: name: Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE runs-on: ubuntu-latest From 813fa874adad65e5cf98a36448bde5e7f7a70502 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 14:35:20 +0900 Subject: [PATCH 002/139] add reusable workflow to build and push target component Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 105 ++++++++++++++++++ .github/workflows/dockers-agent-ngt-image.yml | 97 +--------------- 2 files changed, 108 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/_docker-image.yaml diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml new file mode 100644 index 0000000000..2188af7570 --- /dev/null +++ b/.github/workflows/_docker-image.yaml @@ -0,0 +1,105 @@ +name: "Build docker image" + +on: + workflow_call: + inputs: + target: + type: string + description: 'Build target. e.g. `agent-ngt`' + required: true + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + + build: + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Test + run: | + echo ${{ steps.ref.outputs.ref }} + echo ${{ steps.buildx.outputs.name }} + echo ${{ inputs.target }} + exit 1 + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: ${{ inputs.target }} + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + + slack: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack + with: + author_name: ${{ inputs.target }} image build + slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index 1ebacd2ef6..cb12e5ae8a 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -71,99 +71,8 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: agent-ngt - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: agent-ngt image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-ngt From 91c91f8faa2a2462e9def88d6b9b255ac47479aa Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 14:39:30 +0900 Subject: [PATCH 003/139] test build Signed-off-by: hlts2 --- .github/workflows/dockers-agent-ngt-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index cb12e5ae8a..a64c73dfe2 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -71,6 +71,7 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" + jobs: build: uses: ./.github/workflows/_docker-image.yaml From bdaba929ed8b4656d1ae09347b55b04e8419bd30 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 14:42:25 +0900 Subject: [PATCH 004/139] deleted new line Signed-off-by: hlts2 --- .github/workflows/dockers-agent-ngt-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index a64c73dfe2..cb12e5ae8a 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -71,7 +71,6 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" - jobs: build: uses: ./.github/workflows/_docker-image.yaml From 86e33264a8854f82ce6ecf397c1174b47eeebe63 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 14:47:56 +0900 Subject: [PATCH 005/139] add inherit secret Signed-off-by: hlts2 --- .github/workflows/dockers-agent-ngt-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index cb12e5ae8a..ef163c654b 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -76,3 +76,4 @@ jobs: uses: ./.github/workflows/_docker-image.yaml with: target: agent-ngt + secrets: inherit From 7bf6b9a9c292e0047bc5e451c9873fdf7536748f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 15:27:02 +0900 Subject: [PATCH 006/139] update workflow for ci container Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 12 ++-- .../workflows/dockers-ci-container-image.yml | 61 ++----------------- 2 files changed, 11 insertions(+), 62 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 2188af7570..a4535f5293 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -7,6 +7,11 @@ on: type: string description: 'Build target. e.g. `agent-ngt`' required: true + platforms: + type: string + description: "if it is specified, specified platforms will be used." + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} @@ -59,17 +64,12 @@ jobs: registry: ghcr.io username: ${{ secrets.PACKAGE_USER }} password: ${{ secrets.PACKAGE_TOKEN }} - - name: Test - run: | - echo ${{ steps.ref.outputs.ref }} - echo ${{ steps.buildx.outputs.name }} - echo ${{ inputs.target }} - exit 1 - name: Build and Publish id: build_and_publish uses: ./.github/actions/docker-build with: target: ${{ inputs.target }} + platforms: ${{ inputs.platforms }} builder: ${{ steps.buildx.outputs.name }} - name: Initialize CodeQL if: startsWith( github.ref, 'refs/tags/') diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml index ed3dd19b60..3cd52484b7 100644 --- a/.github/workflows/dockers-ci-container-image.yml +++ b/.github/workflows/dockers-ci-container-image.yml @@ -45,61 +45,10 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: ci-container - builder: ${{ steps.buildx.outputs.name }} - platforms: linux/amd64 + uses: ./.github/workflows/_docker-image.yaml + with: + target: ci-container + platforms: linux/amd64 + secrets: inherit From 7000dbdd346a7dc0c0053403fe2d971ee966b1a4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 16:03:22 +0900 Subject: [PATCH 007/139] move docker related workflow to reusable workflow Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 1 + .../workflows/dockers-dev-container-image.yml | 61 +--------- .../dockers-discoverer-k8s-image.yml | 98 +--------------- .../dockers-gateway-filter-image.yml | 105 +----------------- .../workflows/dockers-gateway-lb-image.yml | 98 +--------------- .../workflows/dockers-helm-operator-image.yml | 98 +--------------- .github/workflows/dockers-loadtest-image.yml | 100 +---------------- .../workflows/dockers-manager-index-image.yml | 98 +--------------- 8 files changed, 31 insertions(+), 628 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index a4535f5293..2d9e880e87 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -97,6 +97,7 @@ jobs: slack: runs-on: ubuntu-latest needs: [build] + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 - uses: ./.github/actions/notify-slack diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml index 376a813d6c..0f9fbb6d03 100644 --- a/.github/workflows/dockers-dev-container-image.yml +++ b/.github/workflows/dockers-dev-container-image.yml @@ -38,61 +38,10 @@ on: schedule: - cron: "0 1 * * *" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: dev-container - builder: ${{ steps.buildx.outputs.name }} - platforms: linux/amd64 + uses: ./.github/workflows/_docker-image.yaml + with: + target: dev-container + platforms: linux/amd64 + secrets: inherit diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml index 7207c2f2eb..dff4b8339b 100644 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ b/.github/workflows/dockers-discoverer-k8s-image.yml @@ -65,99 +65,9 @@ on: - "dockers/discoverer/k8s/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: discoverer-k8s - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: discoverer-k8s image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: discoverer-k8s + secrets: inherit diff --git a/.github/workflows/dockers-gateway-filter-image.yml b/.github/workflows/dockers-gateway-filter-image.yml index a9bbc01684..537ff837fb 100644 --- a/.github/workflows/dockers-gateway-filter-image.yml +++ b/.github/workflows/dockers-gateway-filter-image.yml @@ -74,106 +74,9 @@ on: - "dockers/gateway/filter/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-vald-gateway-filter-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-vald-gateway-filter- - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: gateway-filter - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: vald-filter-gateway image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-filter + secrets: inherit diff --git a/.github/workflows/dockers-gateway-lb-image.yml b/.github/workflows/dockers-gateway-lb-image.yml index c1840b70a8..43a02279f9 100644 --- a/.github/workflows/dockers-gateway-lb-image.yml +++ b/.github/workflows/dockers-gateway-lb-image.yml @@ -74,99 +74,9 @@ on: - "dockers/gateway/lb/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: gateway-lb - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: vald-lb-gateway image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-lb + secrets: inherit diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index cd7d2a7fc9..3cb4f5ed8e 100644 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -59,99 +59,9 @@ on: - "charts/vald-helm-operator/templates/**" - "versions/OPERATOR_SDK_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: operator/helm - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: helm-operator image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: operator/helm + secrets: inherit diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index 97ab72f69b..7c29f5c6b8 100644 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -68,100 +68,10 @@ on: - "dockers/tools/cli/loadtest/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: loadtest - builder: ${{ steps.buildx.outputs.name }} - platforms: linux/amd64 - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: vald-loadtest image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: loadtest + platforms: linux/amd64 + secrets: inherit diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml index b53d7a5d89..2ff1f2b3db 100644 --- a/.github/workflows/dockers-manager-index-image.yml +++ b/.github/workflows/dockers-manager-index-image.yml @@ -68,99 +68,9 @@ on: - "dockers/manager/index/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: manager-index - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: manager-index image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: manager-index + secrets: inherit From 929682843f040ceb53fc14eac80a95398d1aeab6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 16:06:15 +0900 Subject: [PATCH 008/139] fix agent sidecar image Signed-off-by: hlts2 --- .../workflows/dockers-agent-sidecar-image.yml | 98 +------------------ 1 file changed, 4 insertions(+), 94 deletions(-) diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml index 75b21935c8..375a89e281 100644 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ b/.github/workflows/dockers-agent-sidecar-image.yml @@ -71,99 +71,9 @@ on: - "dockers/agent/sidecar/Dockerfile" - "versions/GO_VERSION" -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} - cancel-in-progress: true - jobs: - dump_contexts_to_log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - build: - strategy: - max-parallel: 4 - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: agent-sidecar - builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - slack: - name: Slack notification - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 - with: - author_name: agent-sidecar image build - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-sidecar + secrets: inherit From c9942e2b244732752ba315e4c364a7eb0e290c47 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 16:21:01 +0900 Subject: [PATCH 009/139] delete unnecessary of coverage workflow Signed-off-by: hlts2 --- .github/actions/setup-go/action.yaml | 28 ++++++++++++++++++++++++++++ .github/workflows/coverage.yml | 8 +++----- 2 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 .github/actions/setup-go/action.yaml diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml new file mode 100644 index 0000000000..fdf491978e --- /dev/null +++ b/.github/actions/setup-go/action.yaml @@ -0,0 +1,28 @@ +name: "Setup Go environment" +description: "Setup Go environment" + +inputs: + go_version: + description: "The Go version to use. The default version is `versions/GO_VERSION`" + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Fetch go version + id: go_version + shell: bash + run: | + GO_VERSION=${GO_VERSION:-`make version/go`} + + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + env: + GO_VERSION: ${{ inputs.go_version }} + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.go_version.outputs.version }} + - name: Check go version + shell: bash + run: | + go version diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 05659811be..8acca32c1c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -43,13 +43,11 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 10 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - name: Run coverage continue-on-error: true run: | From da2a7a03d0adea4188a619fd57c4412e7b9230dd Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 16:42:13 +0900 Subject: [PATCH 010/139] add action to deploy chaos mesh Signed-off-by: hlts2 --- .github/actions/deploy-chaos-mesh/action.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/actions/deploy-chaos-mesh/action.yaml diff --git a/.github/actions/deploy-chaos-mesh/action.yaml b/.github/actions/deploy-chaos-mesh/action.yaml new file mode 100644 index 0000000000..27ae0b7d4a --- /dev/null +++ b/.github/actions/deploy-chaos-mesh/action.yaml @@ -0,0 +1,47 @@ +name: "Deploy Chaos Mesh" +description: "Deploy Chaos Mesh" + +inputs: + chaos_mesh_version: + description: "The Chaos Mesh version to use. The default version is `versions/CHAOS_MESH_VERSION`" + required: false + default: "" + helm_extra_options: + description: "Extra options that passed to Helm command" + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Fetch Chaos Mesh version + id: chaos_mesh_version + shell: bash + run: | + CHAOS_MESH_VERSION=${CHAOS_MESH_VERSION:-`cat versions/CHAOS_MESH_VERSION`} + + echo "version=${CHAOS_MESH_VERSION}" >> $GITHUB_OUTPUT + env: + CHAOS_MESH_VERSION: ${{ inputs.chaos_mesh_version }} + - name: Check k3d + shell: bash + id: check_k3d + run: | + kubectl cluster-info + - name: Check Helm + shell: bash + run: | + helm version + - name: Deploy Chaos Mesh + shell: bash + run: | + make kubectl/install + + curl -sSL "https://mirrors.chaos-mesh.org/v${CHAOS_MESH_VERSION}/install.sh" | bash -s -- --k3s + + helm install \ + ${HELM_EXTRA_OPTIONS} \ + vald-chaos-test tests/chaos/chart + env: + HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} + CHAOS_MESH_VERSION: ${{ steps.chaos_mesh_version.outputs.version }} From a53307dee8c24f3fd592d86fc540ec2cb407c28b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 16:44:19 +0900 Subject: [PATCH 011/139] add action to notify slack Signed-off-by: hlts2 --- .github/actions/notify-slack/action.yaml | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/actions/notify-slack/action.yaml diff --git a/.github/actions/notify-slack/action.yaml b/.github/actions/notify-slack/action.yaml new file mode 100644 index 0000000000..fc776d119c --- /dev/null +++ b/.github/actions/notify-slack/action.yaml @@ -0,0 +1,31 @@ +name: "Notify Slack" +description: "Notify Slack" + +inputs: + author_name: + description: "User name for slack notification" + required: false + token: + description: "Github Token" + default: ${{ github.token }} + required: false + slack_notify_webhook_url: + description: "Slack Notify Webhook URL" + required: true + status: + description: "Specify success or failure or cancelled or custom." + required: false + +runs: + using: "composite" + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ inputs.token }} + - uses: 8398a7/action-slack@v3 + with: + author_name: ${{ inputs.author_name }} + status: ${{ inputs.status != '' && inputs.status || env.WORKFLOW_CONCLUSION }} + token: ${{ inputs.token }} + env: + SLACK_WEBHOOK_URL: ${{ inputs.slack_notify_webhook_url }} From 70c04adfefc752bc67f4eff0d89ee382533192f0 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 18:06:03 +0900 Subject: [PATCH 012/139] add reusable workflow to scan image Signed-off-by: hlts2 --- .github/workflows/_docker-image_scan.yaml | 44 ++++ .github/workflows/dockers-image-scan.yml | 285 +++------------------- 2 files changed, 73 insertions(+), 256 deletions(-) create mode 100644 .github/workflows/_docker-image_scan.yaml diff --git a/.github/workflows/_docker-image_scan.yaml b/.github/workflows/_docker-image_scan.yaml new file mode 100644 index 0000000000..a8140065ab --- /dev/null +++ b/.github/workflows/_docker-image_scan.yaml @@ -0,0 +1,44 @@ +name: "Docker image scanning" + +on: + workflow_call: + inputs: + target: + type: string + description: 'Build target. e.g. `agent-ngt`' + required: true + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true + - name: Build the Docker image + id: build_image + run: | + make docker/build/${{ inputs.target }} + imagename=`make docker/name/${{ inputs.target }}` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + # - name: Upload Trivy scan results to Security tab + # uses: github/codeql-action/upload-sarif@v2 + # with: + # sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index a8f063a178..4ef31263f4 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -18,6 +18,11 @@ on: schedule: - cron: "0 1 * * *" + # TODO: Delete it later. + push: + branches: + - "refactor/ci/update-action" + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} cancel-in-progress: true @@ -30,266 +35,34 @@ jobs: - uses: ./.github/actions/dump-context agent-ngt: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/agent-ngt - imagename=`make docker/name/agent-ngt` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-ngt agent-sidecar: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/agent-sidecar - imagename=`make docker/name/agent-sidecar` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-sidecar discoverer-k8s: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/discoverer-k8s - imagename=`make docker/name/discoverer-k8s` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: discoverer-k8s manager-index: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/manager-index - imagename=`make docker/name/manager-index` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: manager-index operator-helm: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/operator/helm - imagename=`make docker/name/operator/helm` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: operator/helm loadtest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/loadtest - imagename=`make docker/name/loadtest` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: loadtest gateway-lb: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/gateway-lb - imagename=`make docker/name/gateway-lb` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-lb gateway-filter: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Build the Docker image - id: build_image - run: | - make docker/build/gateway-filter - imagename=`make docker/name/gateway-filter` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - severity: "HIGH,CRITICAL" - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-filter From 4adca05bde6b8fa5c2b01731da9583ac3e10c322 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 18:10:13 +0900 Subject: [PATCH 013/139] bugfix wrong file specification Signed-off-by: hlts2 --- .github/workflows/dockers-image-scan.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 4ef31263f4..61b9069611 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -35,34 +35,34 @@ jobs: - uses: ./.github/actions/dump-context agent-ngt: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: agent-ngt agent-sidecar: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: agent-sidecar discoverer-k8s: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: discoverer-k8s manager-index: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: manager-index operator-helm: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: operator/helm loadtest: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: loadtest gateway-lb: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: gateway-lb gateway-filter: - uses: ./.github/workflows/_docker-image.yaml + uses: ./.github/workflows/_docker-image_scan.yaml with: target: gateway-filter From 653827888dd949e0010c08c47c046b40748bcef0 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 18:42:51 +0900 Subject: [PATCH 014/139] use local action Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 541ef1fcd8..b48e9e5e23 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -34,17 +34,9 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 10 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} + path: ${{ github.workspace }} + set-safe-directory: true + - uses: ./.github/actions/setup-go - name: Build and zip run: | make binary/build/zip From ccba2d01208ce3725d1e85f8604758d0b949fa2c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 18:43:39 +0900 Subject: [PATCH 015/139] commentout for debug Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index b48e9e5e23..0c85ee3676 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -40,11 +40,11 @@ jobs: - name: Build and zip run: | make binary/build/zip - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: artifacts-linux - path: ./artifacts/ + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # name: artifacts-linux + # path: ./artifacts/ # build-macos: ## or using cross-compiler? # runs-on: macos-latest # steps: From c5b55160936236bbd19cfea35a3a73df4401ba2b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 18:44:51 +0900 Subject: [PATCH 016/139] test execution for binary build Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 0c85ee3676..a3ea115b0e 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -19,6 +19,11 @@ on: types: - created + # TODO: Delete it later. + push: + branches: + - "refactor/ci/update-action" + jobs: dump_contexts_to_log: runs-on: ubuntu-latest From 1f3e8c59190937abf061a87ba001f94ea2e6ecc2 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 19:24:52 +0900 Subject: [PATCH 017/139] delete debug code Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index a3ea115b0e..b48e9e5e23 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -19,11 +19,6 @@ on: types: - created - # TODO: Delete it later. - push: - branches: - - "refactor/ci/update-action" - jobs: dump_contexts_to_log: runs-on: ubuntu-latest @@ -45,11 +40,11 @@ jobs: - name: Build and zip run: | make binary/build/zip - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # name: artifacts-linux - # path: ./artifacts/ + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: artifacts-linux + path: ./artifacts/ # build-macos: ## or using cross-compiler? # runs-on: macos-latest # steps: From 4d37cc8a6aa7ba81b6cba5b22e029ac4968b77f4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 19:30:13 +0900 Subject: [PATCH 018/139] update checkout setting Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 334c0dce58..c04db30137 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,6 +18,7 @@ on: push: branches: - main + - "refactor/ci/update-action" paths: - "apis/proto/**" - "versions/GO_VERSION" @@ -41,10 +42,8 @@ jobs: steps: - uses: actions/checkout@v3 with: - persist-credentials: false - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - name: build protobuf run: | make proto/all From c29ad4fafeaef6dd5d8d28d92d2e62360d82bdda Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 19:32:49 +0900 Subject: [PATCH 019/139] deleted unused branch rule Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index c04db30137..384c734600 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,7 +18,6 @@ on: push: branches: - main - - "refactor/ci/update-action" paths: - "apis/proto/**" - "versions/GO_VERSION" From 8effee52bb89066eda524b840d5944466fb1c6c1 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 19:34:51 +0900 Subject: [PATCH 020/139] rename workflow name Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 384c734600..3f78a65546 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: build protobuf +name: "Build protobuf" on: push: branches: From 809db0ba2132c486cada45595d29fe485060b131 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 19:42:06 +0900 Subject: [PATCH 021/139] deleted unused chatops command Signed-off-by: hlts2 --- .github/chatops_commands.md | 2 - .github/workflows/chatops.yml | 150 +--------------------------------- 2 files changed, 1 insertion(+), 151 deletions(-) diff --git a/.github/chatops_commands.md b/.github/chatops_commands.md index b06976e1ae..c08eb41a2d 100644 --- a/.github/chatops_commands.md +++ b/.github/chatops_commands.md @@ -1,7 +1,5 @@ - :ok_woman: `/approve` - approve -- :love_letter: `/changelog` - replace the PR body by changelog details - :bento: `/format` - format codes and add licenses - :white_check_mark: `/gen-test` - generate test codes - :label: `/label` - add labels -- :rewind: `/rebase` - rebase main - :end: :two: :end: `/label actions/e2e-deploy` - run E2E deploy & integration test diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index fe37956172..90b84556f2 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -81,156 +81,7 @@ jobs: ISSUE_URL: ${{ github.event.issue.labels_url }} USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} TRAILING_LINE: ${{ steps.check_comments_label.outputs.TRAILING_LINE }} - changelog: - name: Add draft comment for a new CHANGELOG entry - runs-on: ubuntu-latest - if: github.event.comment.user.login != 'vdaas-ci' - steps: - - name: check PR Comments - id: check_comments_changelog - uses: machine-learning-apps/actions-chatops@master - with: - TRIGGER_PHRASE: "/changelog" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq - if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - - name: check permissions - if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' - id: check_permissions - run: | - curl -o chatops_permissions.yaml https://raw.githubusercontent.com/vdaas/vald/main/.github/chatops_permissions.yaml - roles=`yq r chatops_permissions.yaml "users.${USERNAME}.role.*"` - if [ "$roles" = "" ]; then - roles=`yq r chatops_permissions.yaml "default-roles.*"` - fi - for role in `echo $roles` - do - if yq r chatops_permissions.yaml "roles.${role}.policies.*" | grep "^${REQUIRED_POLICY}$" > /dev/null 2>&1 ; then - echo "[OK] changelog requested by ${USERNAME}" - echo "EXECUTABLE=true" >> $GITHUB_OUTPUT - break - fi - done - env: - USERNAME: ${{ steps.check_comments_changelog.outputs.COMMENTER_USERNAME }} - REQUIRED_POLICY: changelog - - uses: actions/checkout@v3 - if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - with: - fetch-depth: 0 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: comment to PR - if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - run: | - git checkout main - PREVIOUS_VERSION=$(git describe --tags $(git rev-list --tags --max-count=1)) - BODY=`git log --pretty=format:'- %s' ${PREVIOUS_VERSION}..main | grep "#[[:digit:]]\+" | sed -e "s/\[\(patch\|minor\|major\)\] *//g" | sed -e "s%#\([[:digit:]]\+\)%[&](https://github.com/vdaas/vald/pull/\1)%g" | sed ':a;N;$!ba;s/\n/\\\\n/g'` - CURRENT="- ${PR_TITLE} ([#${PR_NUM}](https://github.com/vdaas/vald/pull/${PR_NUM}))" - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request PATCH \ - --data "{\"body\": \"${CURRENT}\\n${BODY}\"}" \ - https://api.github.com/repos/vdaas/vald/pulls/${PR_NUM} - - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request POST \ - --data "{\"body\": \"**[CHANGELOG]** The description of this PR is updated. Please edit it.\"}" \ - $API_URL - env: - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: ${{ github.event.issue.comments_url }} - PR_TITLE: ${{ github.event.issue.title }} - PR_NUM: ${{ github.event.issue.number }} - rebase: - name: Rebase - runs-on: ubuntu-latest - if: github.event.comment.user.login != 'vdaas-ci' - steps: - - name: check PR Comments - id: check_comments_rebase - uses: machine-learning-apps/actions-chatops@master - with: - TRIGGER_PHRASE: "/rebase" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq - if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - - name: check permissions - if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' - id: check_permissions - run: | - curl -o chatops_permissions.yaml https://raw.githubusercontent.com/vdaas/vald/main/.github/chatops_permissions.yaml - roles=`yq r chatops_permissions.yaml "users.${USERNAME}.role.*"` - if [ "$roles" = "" ]; then - roles=`yq r chatops_permissions.yaml "default-roles.*"` - fi - for role in `echo $roles` - do - if yq r chatops_permissions.yaml "roles.${role}.policies.*" | grep "^${REQUIRED_POLICY}$" > /dev/null 2>&1 ; then - echo "[OK] rebase requested by ${USERNAME}" - echo "EXECUTABLE=true" >> $GITHUB_OUTPUT - break - fi - done - env: - USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} - REQUIRED_POLICY: rebase - - name: comment back to PR - if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - run: | - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request POST \ - --data "{\"body\": \"**[REBASE]** Rebase triggered by ${USERNAME} for branch: ${BRANCH}\"}" \ - $API_URL - env: - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: ${{ github.event.issue.comments_url }} - BRANCH: ${{ steps.check_comments_rebase.outputs.BRANCH_NAME }} - USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} - - uses: actions/checkout@v3 - if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - with: - fetch-depth: 0 - token: ${{ secrets.DISPATCH_TOKEN }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Automatic Rebase - if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - uses: cirrus-actions/rebase@1.5 - env: - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - name: failure comment - if: failure() - run: | - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request POST \ - --data "{\"body\": \"**[REBASE]** Failed to rebase.\"}" \ - $API_URL - env: - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: ${{ github.event.issue.comments_url }} gentest: name: Add tests needs: @@ -501,6 +352,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} + approve: name: Approve needs: From cde3922864bbd2843598afeddc084db48e98fa6d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 20:00:09 +0900 Subject: [PATCH 022/139] refactoring codeQL workflow Signed-off-by: hlts2 --- .github/workflows/codeql-analysis.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1a35c9dd04..f6f01bafbe 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Code scanning - action" - +name: "Code scanning" on: push: branches: @@ -40,8 +39,10 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - CodeQL-Build: + codeQL: runs-on: ubuntu-latest + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - name: Checkout repository uses: actions/checkout@v3 @@ -49,14 +50,8 @@ jobs: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Install dependencies - run: | - sudo apt-get update && \ - sudo apt-get install -y --no-install-recommends libhdf5-dev - sudo make ngt/install + path: ${{ github.workspace }} + set-safe-directory: true - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: From c1ab71a1b0dd0041082f576e25cca2dd7458046b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 20:41:35 +0900 Subject: [PATCH 023/139] refacroing Signed-off-by: hlts2 --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/coverage.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f6f01bafbe..adab07b095 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - codeQL: + CodeQL-Build: runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8acca32c1c..4e02943ba7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Coverage" +name: "Coverage reporting" on: push: branches: @@ -38,7 +38,6 @@ jobs: - uses: ./.github/actions/dump-context coverage: - name: Coverage runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly From 35402a2f8a335316a3ab60efef37255c7f28de2f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 26 Jul 2023 21:12:22 +0900 Subject: [PATCH 024/139] tweak Signed-off-by: hlts2 --- .github/workflows/codeql-analysis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index adab07b095..2db2308d48 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,8 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - CodeQL-Build: + codeQL-build: + name: CodeQL runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly From 295db4dd176936489cd1c7f45cfab468ddbbba74 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 10:54:28 +0900 Subject: [PATCH 025/139] add cron for ci image build Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 0 .github/workflows/dockers-ci-container-image.yml | 2 ++ .github/workflows/dockers-image-scan.yml | 7 +++++++ 3 files changed, 9 insertions(+) create mode 100644 .github/actions/setup-e2e/action.yaml diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml index 3cd52484b7..d73065c7b4 100644 --- a/.github/workflows/dockers-ci-container-image.yml +++ b/.github/workflows/dockers-ci-container-image.yml @@ -44,6 +44,8 @@ on: - "Makefile.d/**" - "versions/GO_VERSION" - "versions/NGT_VERSION" + schedule: + - cron: "0 1 * * *" jobs: build: diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 61b9069611..20577ccb5c 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -38,30 +38,37 @@ jobs: uses: ./.github/workflows/_docker-image_scan.yaml with: target: agent-ngt + agent-sidecar: uses: ./.github/workflows/_docker-image_scan.yaml with: target: agent-sidecar + discoverer-k8s: uses: ./.github/workflows/_docker-image_scan.yaml with: target: discoverer-k8s + manager-index: uses: ./.github/workflows/_docker-image_scan.yaml with: target: manager-index + operator-helm: uses: ./.github/workflows/_docker-image_scan.yaml with: target: operator/helm + loadtest: uses: ./.github/workflows/_docker-image_scan.yaml with: target: loadtest + gateway-lb: uses: ./.github/workflows/_docker-image_scan.yaml with: target: gateway-lb + gateway-filter: uses: ./.github/workflows/_docker-image_scan.yaml with: From 76241e7d9442da71da55c22761d63c7fe18e38d4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 11:01:50 +0900 Subject: [PATCH 026/139] add new line Signed-off-by: hlts2 --- .github/workflows/detect-internal-config-changes.yml | 1 + .github/workflows/dockers-image-scan.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index ab6cc38c91..e4ca34a9da 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -14,6 +14,7 @@ # limitations under the License. # name: "Detect internal config changes" + on: pull_request: paths: diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 20577ccb5c..de6ec73ea2 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -14,6 +14,7 @@ # limitations under the License. # name: "Docker image scanning" + on: schedule: - cron: "0 1 * * *" From e2cc8574bfd517f3fff9ce8b9aa133d7c3714d6a Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 11:35:53 +0900 Subject: [PATCH 027/139] add action to setup E2E test Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 74 ++++++++++++++++++++++++++ .github/actions/setup-go/action.yaml | 6 ++- .github/actions/setup-helm/action.yaml | 28 ++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 .github/actions/setup-helm/action.yaml diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index e69de29bb2..e6f2c6697d 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -0,0 +1,74 @@ +name: "Setup E2E environment" +description: 'Setup the environment to run E2E test' + +inputs: + require_libhdf5: + description: "If libhdf5 is required, set this to true." + required: false + default: "false" + target_images: + description: "image names" + required: false + default: "vdaas/vald-agent-ngt \ + vdaas/vald-discoverer-k8s \ + vdaas/vald-lb-gateway \ + vdaas/vald-manager-index" + +outputs: + HELM_EXTRA_OPTIONS: + description: "helm extra options that specifies E2E target image tags" + value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + IMAGE_TAGS: + description: "specifies E2E target image tags" + value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} + +runs: + using: "composite" + steps: + - name: Install libhdf5 + if: ${{ inputs.require_libhdf5 == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev + + - name: Setup Go environment + uses: ./.github/actions/setup-go + + - name: Setup Helm environment + uses: ./.github/actions/setup-helm + + - name: Get PR number + id: get_pr_number + shell: bash + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + + - name: Wait for E2E target Docker images + if: startsWith( github.ref, 'refs/tags/') + uses: ./.github/actions/wait-for-docker-image + with: + images: ${{ inputs.target_images }} + + - name: Specify container versions + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + # TODO: Change it later + # tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + # images: ${{ inputs.target_images }} + tag_name: nightly + images: ${{ inputs.target_images }} + + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + options: "--image docker.io/rancher/k3s:latest" + + - name: Check Kubernetes cluster + shell: bash + run: | + kubectl cluster-info diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml index fdf491978e..76faa4f911 100644 --- a/.github/actions/setup-go/action.yaml +++ b/.github/actions/setup-go/action.yaml @@ -10,7 +10,7 @@ inputs: runs: using: "composite" steps: - - name: Fetch go version + - name: Fetch Go version id: go_version shell: bash run: | @@ -19,10 +19,12 @@ runs: echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT env: GO_VERSION: ${{ inputs.go_version }} + - uses: actions/setup-go@v3 with: go-version: ${{ steps.go_version.outputs.version }} - - name: Check go version + + - name: Check Go version shell: bash run: | go version diff --git a/.github/actions/setup-helm/action.yaml b/.github/actions/setup-helm/action.yaml new file mode 100644 index 0000000000..e12fea589c --- /dev/null +++ b/.github/actions/setup-helm/action.yaml @@ -0,0 +1,28 @@ +name: "Setup Helm environment" +description: "Setup Helm environment" + +inputs: + helm_version: + description: "The Helm version to use. The default version is `versions/HELM_VERSION`" + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Fetch Helm version + id: helm_version + shell: bash + run: | + HELM_VERSION=${HELM_VERSION:-`make version/helm`} + + echo "version=${HELM_VERSION}" >> $GITHUB_OUTPUT + env: + HELM_VERSION: ${{ inputs.helm_version }} + - uses: azure/setup-helm@v3 + with: + helm-version: ${{ steps.helm_version.outputs.version }} + - name: Check Helm version + shell: bash + run: | + helm version From 19ea468adc2535d4d5b4b6829eeeb66b1ca60da1 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 11:49:21 +0900 Subject: [PATCH 028/139] refactoring action to deploy vald and vald helm operator Signed-off-by: hlts2 --- .../e2e-deploy-vald-helm-operator/action.yaml | 33 +++---------------- .github/actions/e2e-deploy-vald/action.yaml | 33 +++---------------- 2 files changed, 8 insertions(+), 58 deletions(-) diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index ad5848cf23..c270d6e32e 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -15,6 +15,7 @@ # name: "Setting E2E test environment" description: "A workflow to set E2E test environment" + inputs: require_libhdf5: description: "If libhdf5 is required, set this to true." @@ -44,42 +45,16 @@ outputs: POD_NAME: description: "a pod name that waited for" value: ${{ steps.deploy_vald.outputs.POD_NAME }} + runs: using: "composite" steps: - - name: Install libhdf5 - id: install_libhdf5 - shell: bash - run: | - if [[ "${REQUIRE_LIBHDF5}" == "true" ]]; then - sudo apt-get update - sudo apt-get install -y libhdf5-dev - else - echo "skipped." - fi - env: - REQUIRE_LIBHDF5: ${{ inputs.require_libhdf5 }} - - name: check k3d - shell: bash - id: check_k3d - run: | - kubectl cluster-info - - name: Helm version - shell: bash - id: helm_version - run: | - helm version - name: deploy Minio id: deploy_minio shell: bash + if: ${{ inputs.require_minio == 'true' }} run: | - if [[ "${REQUIRE_MINIO}" == "true" ]]; then - make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - else - echo "skipped." - fi - env: - REQUIRE_MINIO: ${{ inputs.require_minio }} + make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - name: deploy vald helm operator from remote charts shell: bash id: deploy_vald_helm_operator_remote diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 92cb42245a..9fcf5b4b4c 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -15,6 +15,7 @@ # name: "Setting E2E test environment" description: "A workflow to set E2E test environment" + inputs: require_libhdf5: description: "If libhdf5 is required, set this to true." @@ -48,42 +49,16 @@ outputs: POD_NAME: description: "a pod name that waited for" value: ${{ steps.get_real_pod_name.outputs.POD_NAME }} + runs: using: "composite" steps: - - name: Install libhdf5 - id: install_libhdf5 - shell: bash - run: | - if [[ "${REQUIRE_LIBHDF5}" == "true" ]]; then - sudo apt-get update - sudo apt-get install -y libhdf5-dev - else - echo "skipped." - fi - env: - REQUIRE_LIBHDF5: ${{ inputs.require_libhdf5 }} - - name: check k3d - shell: bash - id: check_k3d - run: | - kubectl cluster-info - - name: Helm version - shell: bash - id: helm_version - run: | - helm version - name: deploy Minio id: deploy_minio shell: bash + if: ${{ inputs.require_minio == 'true' }} run: | - if [[ "${REQUIRE_MINIO}" == "true" ]]; then - make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - else - echo "skipped." - fi - env: - REQUIRE_MINIO: ${{ inputs.require_minio }} + make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - name: deploy vald from remote charts shell: bash id: deploy_vald_remote From 8b278efe90cb221bc433b1809a2ecefb548982d7 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 12:47:30 +0900 Subject: [PATCH 029/139] fix job name and deleted unnecessary Signed-off-by: hlts2 --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/coverage.yml | 1 + .github/workflows/detect-internal-config-changes.yml | 1 - .github/workflows/dockers-image-scan.yml | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2db2308d48..d767904e82 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - codeQL-build: + codeql-build: name: CodeQL runs-on: ubuntu-latest container: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 4e02943ba7..fbcd8e909d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -38,6 +38,7 @@ jobs: - uses: ./.github/actions/dump-context coverage: + name: Coverage runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index e4ca34a9da..ab6cc38c91 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -14,7 +14,6 @@ # limitations under the License. # name: "Detect internal config changes" - on: pull_request: paths: diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index de6ec73ea2..20577ccb5c 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -14,7 +14,6 @@ # limitations under the License. # name: "Docker image scanning" - on: schedule: - cron: "0 1 * * *" From be743b93f5c94fc97f2c87807a73cda8a7f77b3d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 16:57:43 +0900 Subject: [PATCH 030/139] use local e2e action for agent chaos test Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 3 +- .github/workflows/e2e-chaos.yaml | 110 +++++++++----------------- 2 files changed, 40 insertions(+), 73 deletions(-) diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index e6f2c6697d..378b747a86 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -39,13 +39,14 @@ runs: uses: ./.github/actions/setup-helm - name: Get PR number + # TODO: add if statement id: get_pr_number shell: bash run: | pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Wait for E2E target Docker images + - name: Wait for target Docker images if: startsWith( github.ref, 'refs/tags/') uses: ./.github/actions/wait-for-docker-image with: diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 3b81f488c3..c17c218159 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -16,6 +16,9 @@ name: "Run E2E chaos test" on: push: + # TODO: delete it later + branches: + - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" @@ -26,7 +29,7 @@ on: - "labeled" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -36,87 +39,50 @@ jobs: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + # TODO: delete commentout it later + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 3 - options: "--image docker.io/rancher/k3s:latest" - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - name: deploy Chaos Mesh - run: | - make kubectl/install - curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s - helm install \ - --set podChaos.failure.enabled=true \ - vald-chaos-test tests/chaos/chart - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 + + - name: Deploy Chaos Mesh + uses: ./.github/actions/deploy-chaos-mesh with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run Insert and Search jobs - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + helm_extra_options: "--set podChaos.kill.enabled=true" + + # - name: Run Insert and Search operations + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # go version + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_TIMEOUT=15m \ + # E2E_INSERT_COUNT=10000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/insert/search + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" runs-on: ubuntu-latest From 9005e8855577d376bb7e732c6d50c32e77759aaf Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 17:05:11 +0900 Subject: [PATCH 031/139] fix helm action warning Signed-off-by: hlts2 --- .github/actions/setup-helm/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-helm/action.yaml b/.github/actions/setup-helm/action.yaml index e12fea589c..f1bcacc896 100644 --- a/.github/actions/setup-helm/action.yaml +++ b/.github/actions/setup-helm/action.yaml @@ -21,7 +21,7 @@ runs: HELM_VERSION: ${{ inputs.helm_version }} - uses: azure/setup-helm@v3 with: - helm-version: ${{ steps.helm_version.outputs.version }} + version: ${{ steps.helm_version.outputs.version }} - name: Check Helm version shell: bash run: | From 657ca87bf7034e50fe2ced8b5fcf4f765d7f28eb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 17:47:57 +0900 Subject: [PATCH 032/139] use local e2e action for chaos testing Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 357 +++++++++++-------------------- 1 file changed, 119 insertions(+), 238 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index c17c218159..8b93bb2ca0 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -64,12 +64,11 @@ jobs: - name: Deploy Chaos Mesh uses: ./.github/actions/deploy-chaos-mesh with: - helm_extra_options: "--set podChaos.kill.enabled=true" + helm_extra_options: "--set podChaos.failure.enabled=true" # - name: Run Insert and Search operations # run: | # make hack/benchmark/assets/dataset/${DATASET} - # go version # make E2E_BIND_PORT=8081 \ # E2E_DATASET_NAME=${DATASET} \ # E2E_TIMEOUT=15m \ @@ -87,219 +86,116 @@ jobs: name: "E2E chaos test (random Pod failure: to test redundancy)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + # TODO: delete commentout it later + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 3 - options: "--image docker.io/rancher/k3s:latest" - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - name: deploy Chaos Mesh - run: | - make kubectl/install - curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s - helm install \ - --set podChaos.kill.enabled=true \ - vald-chaos-test tests/chaos/chart - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 + + - name: Deploy Chaos Mesh + uses: ./.github/actions/deploy-chaos-mesh with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run Insert and Search jobs - run: | - make hack/benchmark/assets/dataset/${DATASET} - podname=`kubectl get pods --selector=app=vald-lb-gateway | tail -1 | awk '{print $1}'` - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${podname} \ - E2E_TARGET_NAMESPACE=default \ - e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 + helm_extra_options: "--set podChaos.kill.enabled=true" + + # - name: Run Insert and Search operations + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_TIMEOUT=15m \ + # E2E_INSERT_COUNT=10000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/insert/search + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + # TODO: delete commentout it later + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 3 - options: "--image docker.io/rancher/k3s:latest" - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - name: deploy Chaos Mesh - run: | - make kubectl/install - curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s - helm install \ - --set networkChaos.partition.enabled=true \ - vald-chaos-test tests/chaos/chart - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 + + - name: Deploy Chaos Mesh + uses: ./.github/actions/deploy-chaos-mesh with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run Insert and Search jobs - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + helm_extra_options: "--set networkChaos.partition.enabled=true" + + # - name: Run Insert and Search operations + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_TIMEOUT=15m \ + # E2E_INSERT_COUNT=10000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/insert/search + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + # TODO: delete commentout it later + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 3 - options: "--image docker.io/rancher/k3s:latest" - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: @@ -307,69 +203,54 @@ jobs: helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - - name: deploy Chaos Mesh - run: | - make kubectl/install - curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s - helm install \ - --set networkChaos.bandwidth.enabled=true \ - vald-chaos-test tests/chaos/chart - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 + + - name: Deploy Chaos Mesh + uses: ./.github/actions/deploy-chaos-mesh with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run Insert jobs - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/insert - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - name: run Search jobs - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_SEARCH_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + helm_extra_options: "--set networkChaos.bandwidth.enabled=true" + + # - name: Run Insert operations + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_TIMEOUT=15m \ + # E2E_INSERT_COUNT=10000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/insert + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # + # - name: Run Search operations + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_TIMEOUT=15m \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/search + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + slack-notification: name: "Slack notification" + runs-on: ubuntu-latest needs: - agent-failure - random-pod-failure - agent-network-partition - clusterwide-network-bandwidth - runs-on: ubuntu-latest - if: startsWith( github.ref, 'refs/tags/') + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 + - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: - author_name: "E2E chaos test" - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + author_name: E2E Chaos test + slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} From 7437714992a7567d83c9fcc7d341a32a91d97b4a Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 18:27:39 +0900 Subject: [PATCH 033/139] refactoring and use locaal e2e setup action in the e2e deploy workflow Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 3 + .github/workflows/e2e-chaos.yaml | 2 +- .github/workflows/e2e-code-bench-agent.yaml | 26 +- .github/workflows/e2e-deploy.yml | 509 +++++++------------- 4 files changed, 183 insertions(+), 357 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index b48e9e5e23..9f607e10fb 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -36,10 +36,13 @@ jobs: fetch-depth: 10 path: ${{ github.workspace }} set-safe-directory: true + - uses: ./.github/actions/setup-go + - name: Build and zip run: | make binary/build/zip + - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 8b93bb2ca0..39d37987e6 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -200,7 +200,7 @@ jobs: uses: ./.github/actions/e2e-deploy-vald with: require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index 422a6c900a..c7f2256f90 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -55,13 +55,13 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - persist-credentials: false - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true + + - uses: ./.github/actions/setup-go + - name: Run grpc-sequential run: | make hack/benchmark/assets/dataset/${DATASET} @@ -69,6 +69,7 @@ jobs: env: DATASET: fashion-mnist-784-euclidean.hdf5 DATASET_ARGS: fashion-mnist + - name: Upload artifact uses: actions/upload-artifact@v3 with: @@ -87,11 +88,13 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true + + - uses: ./.github/actions/setup-go + - name: Run grpc-stream run: | make hack/benchmark/assets/dataset/${DATASET} @@ -99,6 +102,7 @@ jobs: env: DATASET: fashion-mnist-784-euclidean.hdf5 DATASET_ARGS: fashion-mnist + - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index a23c7f39f5..cbeb244659 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -16,6 +16,10 @@ name: "Run E2E deploy and integration test" on: push: + # TODO: delete it later + branches: + - "refactor/ci/update-action" + tags: - "*.*.*" - "v*.*.*" @@ -35,131 +39,69 @@ jobs: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # TODO: delete commentout + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_INSERT_COUNT=60000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-stream-crud-for-operator: name: "E2E test (Stream CRUD) for operator" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # TODO: delete commentout later + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: Merge image tag + require_libhdf5: "true" + + - name: Merge Docker image tag run: | - IMAGE_TAGS=(${{ steps.specify_container_versions.outputs.IMAGE_TAGS }}) + IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) for IMAGE_TAG in "${IMAGE_TAGS[@]}" do @@ -170,253 +112,143 @@ jobs: TAG="\"${ARR[1]}\"" yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml done - - name: deploy Vald + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald-helm-operator with: - require_libhdf5: "true" valdrelease: ./.github/valdrelease/valdrelease.yaml wait_for_selector: app=vald-lb-gateway - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_INSERT_COUNT=60000 \ + # E2E_SEARCH_COUNT=10000 \ + # E2E_SEARCH_BY_ID_COUNT=10000 \ + # E2E_GET_OBJECT_COUNT=100 \ + # E2E_UPDATE_COUNT=100 \ + # E2E_UPSERT_COUNT=100 \ + # E2E_REMOVE_COUNT=100 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-skip-exist-check: name: "E2E test (Stream CRUD: skip strict exist check)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # TODO: delete commentout + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_GET_OBJECT_COUNT=10 \ - E2E_UPDATE_COUNT=10 \ - E2E_UPSERT_COUNT=10 \ - E2E_REMOVE_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/skip - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_INSERT_COUNT=10 \ + # E2E_SEARCH_COUNT=10 \ + # E2E_SEARCH_BY_ID_COUNT=10 \ + # E2E_GET_OBJECT_COUNT=10 \ + # E2E_UPDATE_COUNT=10 \ + # E2E_UPSERT_COUNT=10 \ + # E2E_REMOVE_COUNT=10 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/skip + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-multiapis-crud: name: "E2E test (Multi-APIs CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # TODO: delete commentout + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=100 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/multi - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + # - name: Run E2E CRUD + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # go version + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_INSERT_COUNT=100 \ + # E2E_SEARCH_COUNT=10 \ + # E2E_SEARCH_BY_ID_COUNT=10 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/multi + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-agent-and-sidecar: name: "E2E Agent & Sidecar test" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + # TODO: delete commentout + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - with: - images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libhdf5-dev - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: @@ -426,29 +258,22 @@ jobs: values: .github/helm/values/values-agent-sidecar.yaml wait_for_selector: app=vald-agent-ngt wait_for_timeout: 29m - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E Agent & Sidecar - run: | - make hack/benchmark/assets/dataset/${DATASET} - go version - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=4000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/sidecar - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + # - name: Run E2E Agent & Sidecar + # run: | + # make hack/benchmark/assets/dataset/${DATASET} + # make E2E_BIND_PORT=8081 \ + # E2E_DATASET_NAME=${DATASET} \ + # E2E_INSERT_COUNT=10000 \ + # E2E_SEARCH_COUNT=4000 \ + # E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ + # E2E_TARGET_POD_NAME=${POD_NAME} \ + # E2E_TARGET_NAMESPACE=default \ + # e2e/sidecar + # env: + # DATASET: fashion-mnist-784-euclidean.hdf5 + # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + slack-notification: name: "Slack notification" needs: @@ -460,14 +285,8 @@ jobs: runs-on: ubuntu-latest if: startsWith( github.ref, 'refs/tags/') steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 + - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: author_name: "E2E deploy test" - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} From 215796fbeb7adc0a833e387a7e97e1948db50f70 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 18:57:31 +0900 Subject: [PATCH 034/139] use local e2e action for max dimension test Signed-off-by: hlts2 --- .github/workflows/e2e-max-dim.yml | 71 +++++++------------------------ 1 file changed, 15 insertions(+), 56 deletions(-) diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 0ab0c58bf9..207a7d1f1b 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -36,58 +36,22 @@ jobs: name: "E2E test (Max Dimension Insert: skip strict exist check)" runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' + # TODO: delete commentout + # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 1 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 - with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E MaxDimension + require_libhdf5: "true" + + - name: Run E2E MaxDimension run: | - go version go get github.com/vdaas/vald-client-go/v1/payload go get github.com/vdaas/vald-client-go/v1/vald BIT=18 @@ -145,6 +109,7 @@ jobs: WAIT_FOR_TIMEOUT: 29m VALUES: .github/helm/values/values-max-dim.yaml timeout-minutes: 60 + slack-notification: name: "Slack notification" needs: @@ -152,14 +117,8 @@ jobs: runs-on: ubuntu-latest if: startsWith( github.ref, 'refs/tags/') steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: 8398a7/action-slack@v3 + - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: author_name: "E2E max dim test" - status: ${{ env.WORKFLOW_CONCLUSION }} - only_mention_fail: channel - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} From 46bda313f3c9acbf80200533f9e2bbbad2b0260f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 27 Jul 2023 20:25:54 +0900 Subject: [PATCH 035/139] use local e2e action for profiling and refactoring format workflow Signed-off-by: hlts2 --- .github/workflows/e2e-profiling.yml | 71 ++++++++++------------------- .github/workflows/format.yml | 25 +++++----- 2 files changed, 34 insertions(+), 62 deletions(-) diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index 5edbfea183..71a0f5b389 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -39,66 +39,32 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' steps: - uses: actions/checkout@v3 - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - if: startsWith( github.ref, 'refs/tags/') - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - name: Get PR number - id: get_pr_number - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Specify container versions - if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - uses: rinx/setup-k3d@v0.0.4 with: - version: latest - name: vald - agents: 3 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 + path: ${{ github.workspace }} + set-safe-directory: true + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - version: ${{ steps.version.outputs.helm }} - - name: Helm version - run: | - helm version - - name: deploy Vald + require_libhdf5: "true" + + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-profile.yaml wait_for_selector: app=vald-lb-gateway - - name: deploy profefe + + - name: Deploy profefe run: | make k8s/metrics/profefe/deploy kubectl patch cronjob kprofefe -p '{"spec": {"schedule": "*/1 * * * *"}}' kubectl wait --for=condition=ready pod -l app=profefe --timeout=300s - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD + + - name: Run E2E CRUD continue-on-error: true run: | make hack/benchmark/assets/dataset/${DATASET} @@ -118,6 +84,7 @@ jobs: env: DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Get profiles run: | mkdir -p profiles @@ -131,20 +98,24 @@ jobs: --output profiles/${svc}-${t}.pb done done + - uses: actions/upload-artifact@v3 with: name: profiles path: profiles/* + - uses: actions/cache@v2 id: cache with: path: ./profiles-main key: ${{ runner.os }}-profiles-main-${{ github.sha }} restore-keys: ${{ runner.os }}-profiles-main- + - name: Install dependencies run: | sudo apt update sudo apt install -y graphviz + - name: Generate graphs run: | tag=$(cat profiles-main/VALD_VERSION || echo "unknown") @@ -167,16 +138,19 @@ jobs: fi done done + - uses: actions/upload-artifact@v3 with: name: graphs path: graphs/* + - name: Update cache if: startsWith( github.ref, 'refs/tags/') run: | mkdir -p profiles-main cp -f profiles/* profiles-main/ cp -f versions/VALD_VERSION profiles-main/ + - name: Upload to vald-ci-images repository if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' run: | @@ -192,6 +166,7 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: Comment if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' run: | diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index ca2ba48adc..aca06bd428 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -32,20 +32,20 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 + path: ${{ github.workspace }} + set-safe-directory: true token: ${{ secrets.DISPATCH_TOKEN }} - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - name: switch new branch + + - name: Switch new branch id: switch_to_new_branch run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) @@ -53,19 +53,16 @@ jobs: git checkout main git checkout -b ${BRANCH_NAME} echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} + + - name: Setup Go environment + uses: ./.github/actions/setup-go + - name: Run formatter and license.go run: | make deps/install make format git checkout go.mod go.sum + - name: Check and Push to main branch continue-on-error: true run: | From e286ecf4dc7d5ebb29b6cf11df82701621960b4b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 28 Jul 2023 15:58:44 +0900 Subject: [PATCH 036/139] add action to scan docker image Signed-off-by: hlts2 --- .github/actions/scan-docker-image/action.yaml | 36 +++++++++++++++++++ .github/workflows/_docker-image.yaml | 29 +++++++-------- .github/workflows/_docker-image_scan.yaml | 20 +++-------- 3 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 .github/actions/scan-docker-image/action.yaml diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml new file mode 100644 index 0000000000..891f4411cd --- /dev/null +++ b/.github/actions/scan-docker-image/action.yaml @@ -0,0 +1,36 @@ +name: "Scan the Docker image" +description: "Scan the Docker image" + +inputs: + image_ref: + description: "Docker image reference" + required: true + default: "" + severity: + description: "severities of vulnerabilities to be displayed" + required: false + default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" + +runs: + using: "composite" + steps: + - name: Run vulnerability scanner (table) + uses: aquasecurity/trivy-action@master + with: + image-ref: {{ inputs.image_ref }} + format: "table" + severity: {{ inputs.severity }} + + - name: Run vulnerability scanner (sarif) + uses: aquasecurity/trivy-action@master + with: + image-ref: {{ inputs.image_ref }} + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: {{ inputs.severity }} + + # - name: Upload Trivy scan results to Security tab + # uses: github/codeql-action/upload-sarif@v2 + # with: + # sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 2d9e880e87..b6a3dd7e2b 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -38,32 +38,39 @@ jobs: else echo ref=${{ github.sha }} >> $GITHUB_OUTPUT fi + - uses: actions/checkout@v3 with: ref: ${{ steps.ref.outputs.ref }} + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU uses: docker/setup-qemu-action@v2 with: platforms: all + - name: Setup Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 with: buildkitd-flags: "--debug" + - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ secrets.PACKAGE_USER }} password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish id: build_and_publish uses: ./.github/actions/docker-build @@ -71,28 +78,16 @@ jobs: target: ${{ inputs.target }} platforms: ${{ inputs.platforms }} builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL if: startsWith( github.ref, 'refs/tags/') uses: github/codeql-action/init@v2 - - name: Run vulnerability scanner (table) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "table" - - name: Run vulnerability scanner (sarif) - if: startsWith( github.ref, 'refs/tags/') - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" - - name: Upload Trivy scan results to Security tab + + - name: Scan the Docker image if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/upload-sarif@v2 + uses: ./.github/actions/scan-docker-image with: - sarif_file: "trivy-results.sarif" + image_ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" slack: runs-on: ubuntu-latest diff --git a/.github/workflows/_docker-image_scan.yaml b/.github/workflows/_docker-image_scan.yaml index a8140065ab..7dcb028962 100644 --- a/.github/workflows/_docker-image_scan.yaml +++ b/.github/workflows/_docker-image_scan.yaml @@ -16,6 +16,7 @@ jobs: with: path: ${{ github.workspace }} set-safe-directory: true + - name: Build the Docker image id: build_image run: | @@ -25,20 +26,9 @@ jobs: echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT env: DOCKER_BUILDKIT: 1 - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "table" - - name: Run vulnerability scanner - uses: aquasecurity/trivy-action@master + + - name: Scan the Docker image + uses: ./.github/actions/scan-docker-image with: - image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - format: "template" - template: "@/contrib/sarif.tpl" - output: "trivy-results.sarif" + image_ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" severity: "HIGH,CRITICAL" - # - name: Upload Trivy scan results to Security tab - # uses: github/codeql-action/upload-sarif@v2 - # with: - # sarif_file: "trivy-results.sarif" From 5e15954ada59c0a92b663b6b0cf53bad1db853af Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 28 Jul 2023 16:09:55 +0900 Subject: [PATCH 037/139] refactor: add new line Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 3 +- .github/workflows/chatops.yml | 28 +++++++++++++++++++ .github/workflows/codeql-analysis.yml | 3 ++ .github/workflows/coverage.yml | 3 ++ .github/workflows/e2e-chaos.yaml | 1 + .github/workflows/e2e-deploy.yml | 1 + .github/workflows/e2e-max-dim.yml | 1 + .github/workflows/fossa.yml | 3 ++ .github/workflows/helm-lint.yml | 19 +++++++++++++ .github/workflows/helm.yml | 28 ++++++++++++++++++- .github/workflows/labeler.yml | 4 +++ .github/workflows/reviewdog-hadolint.yml | 2 ++ .github/workflows/reviewdog-k8s.yml | 8 ++++++ .github/workflows/reviewdog-markdown.yml | 8 ++++++ .github/workflows/reviewdog.yml | 3 ++ .github/workflows/semver.yml | 6 ++++ .github/workflows/test-hack.yml | 4 +++ .github/workflows/test.yml | 11 ++++++++ .github/workflows/update-protobuf.yml | 3 ++ ...update_pull_request_and_issue_template.yml | 3 ++ 20 files changed, 140 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 3f78a65546..c4ba39059b 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -43,6 +43,7 @@ jobs: with: path: ${{ github.workspace }} set-safe-directory: true - - name: build protobuf + + - name: Build protobuf run: | make proto/all diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index 90b84556f2..313a384a51 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -40,11 +40,13 @@ jobs: TRIGGER_PHRASE: "/label" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: install yq if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' run: | sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -65,6 +67,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: label + - name: add label if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | @@ -96,11 +99,13 @@ jobs: TRIGGER_PHRASE: "/gen-test" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: install yq if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' run: | sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -121,6 +126,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_gen_test.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: gen-test + - name: check executable if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | @@ -136,29 +142,35 @@ jobs: USERNAME: ${{ steps.check_comments_gen_test.outputs.COMMENTER_USERNAME }} PR_AUTHOR: ${{ github.event.issue.user.login }} API_URL: ${{ github.event.issue.comments_url }} + - uses: actions/checkout@v3 if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: fetch-depth: 0 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Fetch golang version if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: go-version: ${{ steps.golang_version.outputs.version }} + - name: Generate tests and push id: gen_test if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' @@ -193,6 +205,7 @@ jobs: PR_INFO_URL: ${{ github.event.issue.pull_request.url }} PR_AUTHOR: ${{ github.event.issue.user.login }} PR_NUM: ${{ github.event.issue.number }} + - name: failure comment if: failure() run: | @@ -207,6 +220,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} + format: name: Add license and run golines + gofumpt + goimports needs: @@ -222,11 +236,13 @@ jobs: TRIGGER_PHRASE: "/format" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: install yq if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' run: | sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -247,6 +263,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_format.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: format + - name: check executable if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | @@ -261,30 +278,36 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} USERNAME: ${{ steps.check_comments_format.outputs.COMMENTER_USERNAME }} API_URL: ${{ github.event.issue.comments_url }} + - uses: actions/checkout@v3 if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Fetch golang version if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: go-version: ${{ steps.golang_version.outputs.version }} + - name: update and push id: format_push if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' @@ -326,6 +349,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} PR_INFO_URL: ${{ github.event.issue.pull_request.url }} PR_NUM: ${{ github.event.issue.number }} + - name: no changes if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' && steps.format_push.outputs.UPDATED == 'false' run: | @@ -339,6 +363,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} + - name: failure comment if: failure() run: | @@ -368,11 +393,13 @@ jobs: TRIGGER_PHRASE: "/approve" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: install yq if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' run: | sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -393,6 +420,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_approve.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: approve + - name: approve if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d767904e82..b26b35ffb7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -53,12 +53,15 @@ jobs: fetch-depth: 2 path: ${{ github.workspace }} set-safe-directory: true + - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: go config-file: ./.github/codeql/codeql-config.yaml + - name: Autobuild uses: github/codeql-action/autobuild@v2 + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fbcd8e909d..e57234aacc 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -48,15 +48,18 @@ jobs: fetch-depth: 10 path: ${{ github.workspace }} set-safe-directory: true + - name: Run coverage continue-on-error: true run: | make coverage + - name: Upload coverage report to Codecov uses: codecov/codecov-action@v3 with: token: ${{secrets.CODECOV_TOKEN}} file: ./coverage.out + - name: Upload coverage report to deepsource run: | mv ./coverage.out ./cover.out diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 39d37987e6..2c3d803a5c 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -250,6 +250,7 @@ jobs: if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: author_name: E2E Chaos test diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index cbeb244659..8ceb8b3e3e 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -286,6 +286,7 @@ jobs: if: startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: author_name: "E2E deploy test" diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 207a7d1f1b..7da12afa95 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -118,6 +118,7 @@ jobs: if: startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/notify-slack with: author_name: "E2E max dim test" diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 86c715c277..89a4aff1b4 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -40,9 +40,11 @@ jobs: - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: "Install fossa-cli" run: | curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install.sh | bash + - name: "Run for main branch" if: github.ref == 'refs/heads/main' run: | @@ -51,6 +53,7 @@ jobs: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_SHA: ${{ github.sha }} + - name: "Run for PRs" if: github.event_name == 'pull_request' run: | diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 6276e8c52b..dcc40049e4 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -34,17 +34,21 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Helm version run: | helm version + - name: Fetch golang version run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 with: go-version: ${{ steps.golang_version.outputs.version }} @@ -55,6 +59,7 @@ jobs: - name: Run lint for charts/vald run: | helm lint charts/vald + lint-vald-helm-operator-chart: name: lint for vald-helm-operator chart runs-on: ubuntu-latest @@ -63,17 +68,21 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Helm version run: | helm version + - name: Fetch golang version run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 with: go-version: ${{ steps.golang_version.outputs.version }} @@ -84,39 +93,49 @@ jobs: - name: Run lint for charts/vald-helm-operator run: | helm lint charts/vald-helm-operator + lint-values-schema: name: lint for values schema runs-on: ubuntu-latest steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Helm version run: | helm version + - name: install YQ run: | sudo make yq/install + - name: Fetch golang version run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 with: go-version: ${{ steps.golang_version.outputs.version }} + - name: Generate ValdRelease schema run: | make helm/schema/crd/vald + - uses: rinx/setup-k3d@v0.0.4 with: version: latest name: vald + - name: check k3d run: | kubectl cluster-info + - name: Test to apply run: | kubectl create -f charts/vald-helm-operator/crds/valdrelease.yaml diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 369d6cd2d2..f78c3638fb 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -39,17 +39,21 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Helm version run: | helm version + - name: switch new branch id: switch_to_new_branch run: | @@ -58,14 +62,17 @@ jobs: git checkout main git checkout -b ${BRANCH_NAME} echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT + - name: Update k8s-dir run: | make k8s/manifest/update make k8s/manifest/helm-operator/update + - name: Run formatter and license.go run: | make format/yaml make license + - name: Push to main continue-on-error: true run: | @@ -87,6 +94,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: https://api.github.com/repos/vdaas/vald/pulls BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} + update-helm-chart: name: Update Helm chart runs-on: ubuntu-latest @@ -98,21 +106,26 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Helm version run: | helm version + - name: Packaging run: | make helm/package/vald make helm/package/vald-helm-operator + - name: Push to gh-pages run: | git checkout . @@ -128,42 +141,52 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + crud-on-remote-helm-chart: name: CRUD test on remote Helm chart runs-on: ubuntu-latest timeout-minutes: 60 steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images id: wait_for_docker_images uses: ./.github/actions/wait-for-docker-image + - name: Specify container versions id: specify_container_versions uses: ./.github/actions/detect-docker-image-tags with: tag_name: ${{ github.ref_name }} + - name: Fetch Helm version run: | HELM_VERSION=`make version/helm` echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT id: version + - uses: rinx/setup-k3d@v0.0.4 with: version: latest name: vald agents: 3 + - name: check k3d run: | kubectl cluster-info + - uses: azure/setup-helm@v3 with: version: ${{ steps.version.outputs.helm }} + - name: Helm version run: | helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald @@ -173,15 +196,18 @@ jobs: values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway use_local_charts: false + - name: Fetch golang version run: | GO_VERSION=`make version/go` echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT id: golang_version + - uses: actions/setup-go@v3 with: go-version: ${{ steps.golang_version.outputs.version }} - - name: run E2E CRUD + + - name: Run E2E CRUD run: | make hack/benchmark/assets/dataset/${DATASET} go version diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index fb2ab40a20..438b784421 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -32,14 +32,17 @@ jobs: - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Checkout PR branch uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} fetch-depth: 0 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Add labels run: | pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` @@ -93,6 +96,7 @@ jobs: env: REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + remove-duplication: name: Remove duplication needs: diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index a725a1aaf3..1d03ae268d 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -32,9 +32,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: hadolint uses: reviewdog/action-hadolint@v1 with: diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index ae87d86852..873663ec14 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -35,9 +35,11 @@ jobs: image: mumoshu/conflint:latest steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: kubeval-conflint run: | set -vx @@ -47,6 +49,7 @@ jobs: CONFLINT_LOG: DEBUG REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPORTER: github-pr-review + kubelinter-k8s-manifests: name: runner / kubelinter runs-on: ubuntu-latest @@ -54,9 +57,11 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: kubelinter for k8s directory run: | kube-linter lint \ @@ -65,6 +70,7 @@ jobs: k8s/discoverer \ k8s/gateway \ k8s/manager + kubelinter-vald-chart: name: runner / kubelinter for vald chart runs-on: ubuntu-latest @@ -72,9 +78,11 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: kubelinter for generated manifests run: | kube-linter lint \ diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index c21b926444..fff7db1aae 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -33,9 +33,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: LanguageTool uses: reviewdog/action-languagetool@v1 with: @@ -51,9 +53,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: LanguageTool uses: reviewdog/action-languagetool@v1 with: @@ -70,16 +74,20 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup node/npm uses: actions/setup-node@v3 with: node-version: "16" + - name: Install textlint run: | make textlint/ci/install + - name: textlint-github-pr-review uses: tsuyoshicho/action-textlint@v3 with: diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index ee16a468c7..7013fb91c5 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -36,12 +36,15 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: go build run: | go build -mod=readonly ./... + - name: Run golangci-lint run: | golangci-lint run --config .golangci.yml \ diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 2f960ddfd4..75a063f504 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -37,14 +37,17 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Get latest gitwerk run: | curl -fsSLO \ @@ -53,6 +56,7 @@ jobs: unzip gitwerk-linux-amd64.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run gitwerk semver-auto id: semver run: | @@ -134,6 +138,7 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: Create release if: steps.semver.outputs.RELEASE == 'true' id: create_release @@ -147,6 +152,7 @@ jobs: See [CHANGELOG.md](https://github.com/vdaas/vald/blob/${{ steps.semver.outputs.VERSION }}/CHANGELOG.md) for details. draft: false prerelease: false + - name: Add the version to goproxy run: | curl "https://proxy.golang.org/github.com/vdaas/vald/@v/${{ steps.semver.outputs.VERSION }}.info" diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index e396b37f69..09be58c2e0 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -57,18 +57,22 @@ jobs: with: fetch-depth: 1 path: src/github.com/vdaas/vald + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald working-directory: src/github.com/vdaas/vald + - name: dependencies run: | GOPATH=${GITHUB_WORKSPACE} make proto/deps working-directory: src/github.com/vdaas/vald + - name: Run tests for hack packages / gotestfmt run: | GOPATH=${GITHUB_WORKSPACE} TEST_RESULT_DIR=${GITHUB_WORKSPACE}/src/github.com/vdaas/vald make test/hack/gotestfmt working-directory: src/github.com/vdaas/vald + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald/test-hack-gotestfmt-result.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e2c7347a5..481b08f587 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,15 +49,19 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Run tests for cmd packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/cmd/gotestfmt + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-cmd-gotestfmt-result.json" + test-internal: name: Run tests for internal packages runs-on: ubuntu-latest @@ -66,15 +70,19 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Run tests for internal packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/internal/gotestfmt + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-internal-gotestfmt-result.json" + test-pkg: name: Run tests for pkg packages runs-on: ubuntu-latest @@ -83,12 +91,15 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Run tests for pkg packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/pkg/gotestfmt + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-pkg-gotestfmt-result.json" diff --git a/.github/workflows/update-protobuf.yml b/.github/workflows/update-protobuf.yml index 121eaa6d06..6dc324fc26 100644 --- a/.github/workflows/update-protobuf.yml +++ b/.github/workflows/update-protobuf.yml @@ -40,6 +40,7 @@ jobs: REPO: vdaas/vald-client-go USER: ${{ secrets.DISPATCH_USER }} TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-java run: | curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' @@ -47,6 +48,7 @@ jobs: REPO: vdaas/vald-client-java USER: ${{ secrets.DISPATCH_USER }} TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-python run: | curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' @@ -54,6 +56,7 @@ jobs: REPO: vdaas/vald-client-python USER: ${{ secrets.DISPATCH_USER }} TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-node run: | curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update_pull_request_and_issue_template.yml index adc62b940d..0b319ab398 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -38,14 +38,17 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true + - name: Push to main run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) From eb97c251d01a500ad11cc29b9ecf5b824c468e24 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 28 Jul 2023 16:21:59 +0900 Subject: [PATCH 038/139] fix workflow execution error Signed-off-by: hlts2 --- .github/actions/scan-docker-image/action.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index 891f4411cd..586270cf9f 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -17,18 +17,18 @@ runs: - name: Run vulnerability scanner (table) uses: aquasecurity/trivy-action@master with: - image-ref: {{ inputs.image_ref }} + image-ref: ${{ inputs.image_ref }} format: "table" - severity: {{ inputs.severity }} + severity: ${{ inputs.severity }} - name: Run vulnerability scanner (sarif) uses: aquasecurity/trivy-action@master with: - image-ref: {{ inputs.image_ref }} + image-ref: ${{ inputs.image_ref }} format: "template" template: "@/contrib/sarif.tpl" output: "trivy-results.sarif" - severity: {{ inputs.severity }} + severity: ${{ inputs.severity }} # - name: Upload Trivy scan results to Security tab # uses: github/codeql-action/upload-sarif@v2 From a99135464d52c6520488e0c79c92766ca34884f8 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 28 Jul 2023 16:44:55 +0900 Subject: [PATCH 039/139] fix deprecate Signed-off-by: hlts2 --- .github/actions/scan-docker-image/action.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index 586270cf9f..caf0cdea6d 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -25,8 +25,7 @@ runs: uses: aquasecurity/trivy-action@master with: image-ref: ${{ inputs.image_ref }} - format: "template" - template: "@/contrib/sarif.tpl" + format: "sarif" output: "trivy-results.sarif" severity: ${{ inputs.severity }} From 75db80e89629d66eecad35b55f7275a06ac19f51 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 10:24:55 +0900 Subject: [PATCH 040/139] deleted unnecessary step and add debug Signed-off-by: hlts2 --- ...update_pull_request_and_issue_template.yml | 90 +++++++++---------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update_pull_request_and_issue_template.yml index 0b319ab398..a88e47dec7 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -18,6 +18,7 @@ on: push: branches: - main + - refactor/ci/update-action paths: - "versions/GO_VERSION" - "versions/NGT_VERSION" @@ -34,14 +35,11 @@ jobs: name: Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE runs-on: ubuntu-latest steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - uses: crazy-max/ghaction-import-gpg@v4 with: @@ -49,43 +47,43 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Push to main - run: | - TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) - BRANCH_NAME="documentation/pull_request_and_issue_template/update_versions_${TIMESTAMP}" - git checkout main - git checkout -b ${BRANCH_NAME} - - GO_VERSION=$(make version/go) - NGT_VERSION=$(make version/ngt) - KUBECTL_VERSION=$(make version/k8s) - - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - - git add .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/security_issue_report.md - git commit -S --signoff -m ":robot: Automatically update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE" - - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push -u origin ${BRANCH_NAME} - - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request POST \ - --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ - $API_URL - env: - GITHUB_USER: ${{ secrets.DISPATCH_USER }} - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: https://api.github.com/repos/vdaas/vald/pulls + # - name: Push to main + # run: | + # TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) + # BRANCH_NAME="documentation/pull_request_and_issue_template/update_versions_${TIMESTAMP}" + # git checkout main + # git checkout -b ${BRANCH_NAME} + # + # GO_VERSION=$(make version/go) + # NGT_VERSION=$(make version/ngt) + # KUBECTL_VERSION=$(make version/k8s) + # + # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + # + # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + # + # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + # + # git add .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/security_issue_report.md + # git commit -S --signoff -m ":robot: Automatically update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE" + # + # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + # git push -u origin ${BRANCH_NAME} + # + # curl --include --verbose --fail \ + # -H "Accept: application/json" \ + # -H "Content-Type:application/json" \ + # -H "Authorization: token ${GITHUB_TOKEN}" \ + # --request POST \ + # --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ + # $API_URL + # env: + # GITHUB_USER: ${{ secrets.DISPATCH_USER }} + # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + # API_URL: https://api.github.com/repos/vdaas/vald/pulls From 9d2462795d64723a254ef1a52c2b2d11f888d92f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 10:25:48 +0900 Subject: [PATCH 041/139] deleted debug comment Signed-off-by: hlts2 --- ...update_pull_request_and_issue_template.yml | 81 +++++++++---------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update_pull_request_and_issue_template.yml index a88e47dec7..c88676d4e9 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -18,7 +18,6 @@ on: push: branches: - main - - refactor/ci/update-action paths: - "versions/GO_VERSION" - "versions/NGT_VERSION" @@ -47,43 +46,43 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - # - name: Push to main - # run: | - # TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) - # BRANCH_NAME="documentation/pull_request_and_issue_template/update_versions_${TIMESTAMP}" - # git checkout main - # git checkout -b ${BRANCH_NAME} - # - # GO_VERSION=$(make version/go) - # NGT_VERSION=$(make version/ngt) - # KUBECTL_VERSION=$(make version/k8s) - # - # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - # sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - # - # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - # sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - # - # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - # sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - # - # git add .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/security_issue_report.md - # git commit -S --signoff -m ":robot: Automatically update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE" - # - # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - # git push -u origin ${BRANCH_NAME} - # - # curl --include --verbose --fail \ - # -H "Accept: application/json" \ - # -H "Content-Type:application/json" \ - # -H "Authorization: token ${GITHUB_TOKEN}" \ - # --request POST \ - # --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ - # $API_URL - # env: - # GITHUB_USER: ${{ secrets.DISPATCH_USER }} - # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - # API_URL: https://api.github.com/repos/vdaas/vald/pulls + - name: Push to main + run: | + TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) + BRANCH_NAME="documentation/pull_request_and_issue_template/update_versions_${TIMESTAMP}" + git checkout main + git checkout -b ${BRANCH_NAME} + + GO_VERSION=$(make version/go) + NGT_VERSION=$(make version/ngt) + KUBECTL_VERSION=$(make version/k8s) + + sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + + sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + + sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + + git add .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/security_issue_report.md + git commit -S --signoff -m ":robot: Automatically update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE" + + git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push -u origin ${BRANCH_NAME} + + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request POST \ + --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ + $API_URL + env: + GITHUB_USER: ${{ secrets.DISPATCH_USER }} + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: https://api.github.com/repos/vdaas/vald/pulls From ae9cadb2273d80095aadcf27034896a1fc7b6844 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 10:37:38 +0900 Subject: [PATCH 042/139] deleted unnecessary step Signed-off-by: hlts2 --- .github/workflows/test.yml | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 481b08f587..80699e61bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,12 +47,10 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: Run tests for cmd packages / gotestfmt run: | @@ -68,12 +66,10 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: Run tests for internal packages / gotestfmt run: | @@ -89,12 +85,10 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: Run tests for pkg packages / gotestfmt run: | From ac5eae8664fb0ee8b2188667770c055ed19ab309 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 10:48:52 +0900 Subject: [PATCH 043/139] refactor existing workflow step name Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 15 +++++---------- .github/workflows/test.yml | 6 +++--- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 09be58c2e0..40d2187d87 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -52,18 +52,13 @@ jobs: run: working-directory: src/github.com/vdaas/vald steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 1 - path: src/github.com/vdaas/vald - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald - working-directory: src/github.com/vdaas/vald + path: ${{ github.workspace }} + set-safe-directory: true - - name: dependencies + - name: Install Proto dependencies run: | GOPATH=${GITHUB_WORKSPACE} make proto/deps working-directory: src/github.com/vdaas/vald @@ -73,6 +68,6 @@ jobs: GOPATH=${GITHUB_WORKSPACE} TEST_RESULT_DIR=${GITHUB_WORKSPACE}/src/github.com/vdaas/vald make test/hack/gotestfmt working-directory: src/github.com/vdaas/vald - - name: print tparse result + - name: Print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald/test-hack-gotestfmt-result.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80699e61bf..ec97a13694 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,7 @@ jobs: run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/cmd/gotestfmt - - name: print tparse result + - name: Print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-cmd-gotestfmt-result.json" @@ -75,7 +75,7 @@ jobs: run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/internal/gotestfmt - - name: print tparse result + - name: Print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-internal-gotestfmt-result.json" @@ -94,6 +94,6 @@ jobs: run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/pkg/gotestfmt - - name: print tparse result + - name: Print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-pkg-gotestfmt-result.json" From 284dd86461e57b04f3cca9b99d58e6db9209a8f4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 11:57:17 +0900 Subject: [PATCH 044/139] update checkout setting Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 1 - .github/workflows/labeler.yml | 9 +++----- .github/workflows/reviewdog-hadolint.yml | 7 +++--- .github/workflows/reviewdog-k8s.yml | 21 ++++++++--------- .github/workflows/reviewdog-markdown.yml | 29 ++++++++++++------------ .github/workflows/reviewdog.yml | 9 +++----- .github/workflows/semver.yml | 6 ++--- 7 files changed, 34 insertions(+), 48 deletions(-) diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 378b747a86..64c2fe19c6 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -58,7 +58,6 @@ runs: with: # TODO: Change it later # tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - # images: ${{ inputs.target_images }} tag_name: nightly images: ${{ inputs.target_images }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 438b784421..db3311f683 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -33,15 +33,12 @@ jobs: with: repo-token: "${{ secrets.GITHUB_TOKEN }}" - - name: Checkout PR branch - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} fetch-depth: 0 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - name: Add labels run: | diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index 1d03ae268d..c53fe3741e 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -32,10 +32,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: hadolint uses: reviewdog/action-hadolint@v1 diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index 873663ec14..3454400141 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -35,10 +35,9 @@ jobs: image: mumoshu/conflint:latest steps: - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: kubeval-conflint run: | @@ -57,10 +56,9 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: kubelinter for k8s directory run: | @@ -78,10 +76,9 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: kubelinter for generated manifests run: | diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index fff7db1aae..a68b76be55 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -32,11 +32,11 @@ jobs: name: runner / LanguageTool / apis & charts runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Check out code. + uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: LanguageTool uses: reviewdog/action-languagetool@v1 @@ -48,15 +48,15 @@ jobs: language: en-US disabled_rules: "DOUBLE_PUNCTUATION,WORD_CONTAINS_UNDERSCORE,ARROWS,CURRENCY,DASH_RULE,EN_QUOTES" disabled_categories: "TYPOS,TYPOGRAPHY,STYLE,CASING" + languagetool_docs: name: runner / LanguageTool / Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: LanguageTool uses: reviewdog/action-languagetool@v1 @@ -68,16 +68,15 @@ jobs: language: en-US disabled_rules: "DOUBLE_PUNCTUATION,WORD_CONTAINS_UNDERSCORE,ARROWS,CURRENCY,DASH_RULE,EN_QUOTES" disabled_categories: "TYPOS,TYPOGRAPHY,STYLE" + textlint: name: runner / textlint runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: actions/checkout@v3 + with: + path: ${{ github.workspace }} + set-safe-directory: true - name: Setup node/npm uses: actions/setup-node@v3 diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 7013fb91c5..b47506892c 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -35,13 +35,10 @@ jobs: steps: - uses: actions/checkout@v3 with: - persist-credentials: false + path: ${{ github.workspace }} + set-safe-directory: true - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: go build + - name: Go build run: | go build -mod=readonly ./... diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 75a063f504..5e818f896e 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -37,10 +37,8 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - uses: crazy-max/ghaction-import-gpg@v4 with: From 724ae26ae35c59cc44132ff7f564549c954f1cc0 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 12:01:25 +0900 Subject: [PATCH 045/139] fix dump context job name Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 2 +- .github/workflows/build-binaries.yml | 2 +- .github/workflows/build-protobuf.yml | 2 +- .github/workflows/chatops-help.yml | 2 +- .github/workflows/chatops.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/detect-internal-config-changes.yml | 2 +- .github/workflows/dockers-image-scan.yml | 2 +- .github/workflows/e2e-code-bench-agent.yaml | 2 +- .github/workflows/e2e-deploy.yml | 2 +- .github/workflows/e2e-max-dim.yml | 2 +- .github/workflows/e2e-profiling.yml | 2 +- .github/workflows/format.yml | 2 +- .github/workflows/fossa.yml | 2 +- .github/workflows/helm-lint.yml | 2 +- .github/workflows/helm.yml | 2 +- .github/workflows/labeler.yml | 2 +- .github/workflows/reviewdog-hadolint.yml | 2 +- .github/workflows/reviewdog-k8s.yml | 2 +- .github/workflows/reviewdog-markdown.yml | 2 +- .github/workflows/reviewdog.yml | 2 +- .github/workflows/semver.yml | 2 +- .github/workflows/test-hack.yml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/update-protobuf.yml | 2 +- .github/workflows/update_pull_request_and_issue_template.yml | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index b6a3dd7e2b..0f444d05ab 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: true jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 9f607e10fb..63a6e80215 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -20,7 +20,7 @@ on: - created jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index c4ba39059b..f2a3add4a5 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -27,7 +27,7 @@ on: - "versions/GO_VERSION" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index af699cbdac..78161b6e8d 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -19,7 +19,7 @@ on: types: [opened] jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index 313a384a51..27839ca146 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -22,7 +22,7 @@ env: YQ_VERSION: 3.2.1 jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b26b35ffb7..4a8166dff8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,7 +33,7 @@ on: - cron: "0 1 * * *" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e57234aacc..fd982eadd0 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -31,7 +31,7 @@ on: - "cmd/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index ab6cc38c91..dcfe248bd4 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -21,7 +21,7 @@ on: - "!internal/config/**/*_test.go" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 20577ccb5c..0db872da78 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: true jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index c7f2256f90..90f68cba54 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -43,7 +43,7 @@ on: - "versions/NGT_VERSION" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 8ceb8b3e3e..83cfd56757 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -29,7 +29,7 @@ on: types: - "labeled" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 7da12afa95..dcb3028e7d 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -26,7 +26,7 @@ on: - "labeled" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index 71a0f5b389..bbdd49ce2b 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -26,7 +26,7 @@ on: - "labeled" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index aca06bd428..682cd28c7b 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -20,7 +20,7 @@ on: - main jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 89a4aff1b4..c75e68ae64 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -23,7 +23,7 @@ on: - "labeled" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index dcc40049e4..d46995bdc0 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -20,7 +20,7 @@ on: - "charts/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index f78c3638fb..af14ab98fa 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -23,7 +23,7 @@ on: - "v*.*.*-*" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index db3311f683..df3a1f75f1 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -18,7 +18,7 @@ on: - pull_request jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index c53fe3741e..ecf4f592bb 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -21,7 +21,7 @@ on: - "dockers/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index 3454400141..1af5a48696 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -22,7 +22,7 @@ on: - "k8s/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index a68b76be55..4974693308 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -22,7 +22,7 @@ on: - "CHANGELOG.md" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index b47506892c..270129bc21 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -21,7 +21,7 @@ on: - "**.go" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 5e818f896e..858e03adf1 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -20,7 +20,7 @@ on: - main jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 40d2187d87..4f255e4b09 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -37,7 +37,7 @@ on: - "hack/tools/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec97a13694..c057f3d266 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ on: - "cmd/**" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/update-protobuf.yml b/.github/workflows/update-protobuf.yml index 6dc324fc26..5fb97156f8 100644 --- a/.github/workflows/update-protobuf.yml +++ b/.github/workflows/update-protobuf.yml @@ -23,7 +23,7 @@ on: - "v*.*.*-*" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update_pull_request_and_issue_template.yml index c88676d4e9..b3a6bc5f04 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -24,7 +24,7 @@ on: - "versions/KUBECTL_VERSION" jobs: - dump_contexts_to_log: + dump-contexts-to-log: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From c938886c817fb84ae5c1deb7f897e5c52ee25b49 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 14:27:03 +0900 Subject: [PATCH 046/139] add action to create k3d cluster Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 4 +- .github/actions/setup-k3d/action.yaml | 63 +++++++++++++++++++++++++++ .github/workflows/helm-lint.yml | 5 +-- .github/workflows/helm.yml | 4 +- 4 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 .github/actions/setup-k3d/action.yaml diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 64c2fe19c6..f2f7a00438 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -61,10 +61,8 @@ runs: tag_name: nightly images: ${{ inputs.target_images }} - - uses: rinx/setup-k3d@v0.0.4 + - uses: ./.github/actions/setup-k3d with: - version: latest - name: vald agents: 3 options: "--image docker.io/rancher/k3s:latest" diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml new file mode 100644 index 0000000000..fc5ff1de1f --- /dev/null +++ b/.github/actions/setup-k3d/action.yaml @@ -0,0 +1,63 @@ +name: "Setup k3d environment" +description: "GitHub Action for setting up k3d (k3s in Docker). It's lighter than KinD (Kubernetes in Docker)" + +inputs: + version: + description: 'k3d version' + required: false + default: 'latest' + name: + description: 'cluster name' + required: false + default: 'vald' + agents: + description: 'number of agents' + required: false + default: '3' + options: + description: 'options for k3d cluster create command' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Detect k3d version + id: k3d_version + shell: bash + run: | + if [ "${K3D_VERSION}" != "latest" ]; then + TAG="TAG=v${K3D_VERSION}" + fi + echo "tag=${TAG}" >> $GITHUB_OUTPUT + env: + K3D_VERSION: ${{ inputs.version }} + + - name: Install k3d + shell: bash + run: | + curl -s ${REPO_URL} | ${{ steps.k3d_version.outputs.tag }} bash + env: + REPO_URL: "https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh" + + - name: Check k3d version + shell: bash + run: | + k3d version + + - name: Setup k3d options + shell: bash + id: k3d_options + run: | + if [ "${AGENTS}" != 0 ]; then + OPTIONS="${OPTIONS} --agents ${AGENTS}" + fi + echo "options=${OPTIONS}" >> $GITHUB_OUTPUT + env: + AGENTS: ${{ inputs.agents }} + OPTIONS: ${{ inputs.options }} + + - name: Create k8s cluster + shell: bash + run: | + k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index d46995bdc0..ce387821b4 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -127,10 +127,7 @@ jobs: run: | make helm/schema/crd/vald - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald + - uses: ./.github/actions/setup-k3d - name: check k3d run: | diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index af14ab98fa..93df1a9e1c 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -169,10 +169,8 @@ jobs: echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT id: version - - uses: rinx/setup-k3d@v0.0.4 + - uses: ./.github/actions/setup-k3d with: - version: latest - name: vald agents: 3 - name: check k3d From ac144866fa1a46ea41ab5fadd2c638c4530f9e55 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 15:20:23 +0900 Subject: [PATCH 047/139] deleted unnecessary job and step Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 32 ++-------------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 63a6e80215..c749175246 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -48,45 +48,17 @@ jobs: with: name: artifacts-linux path: ./artifacts/ - # build-macos: ## or using cross-compiler? - # runs-on: macos-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # fetch-depth: 10 - # - name: Fetch golang version - # run: | - # GO_VERSION=`make version/go` - # echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - # id: golang_version - # - uses: actions/setup-go@v3 - # with: - # go-version: ${{ steps.golang_version.outputs.version }} - # - name: Build and zip - # run: | - # export PATH=$PATH:$(go env GOPATH)/bin - # brew install llvm libomp protobuf ngt - # make CXXFLAGS="-I/usr/local/opt/llvm/include -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl" binary/build/zip - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # name: artifacts-macos - # path: ./artifacts + publish: runs-on: ubuntu-latest needs: - build-linux - # - build-macos - # - build-windows steps: - uses: actions/download-artifact@v2 with: name: artifacts-linux path: tmp/linux - # - uses: actions/download-artifact@v2 - # with: - # name: artifacts-macos - # path: tmp/macos + - uses: shogo82148/actions-upload-release-asset@v1 with: upload_url: ${{ github.event.release.upload_url }} From 4a63e209bd1bc332463dd9c084556cb80116b40b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 15:50:18 +0900 Subject: [PATCH 048/139] bugfix fails hack test workflow Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 4f255e4b09..327c4fcaf3 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -60,14 +60,12 @@ jobs: - name: Install Proto dependencies run: | - GOPATH=${GITHUB_WORKSPACE} make proto/deps - working-directory: src/github.com/vdaas/vald + make proto/deps - name: Run tests for hack packages / gotestfmt run: | - GOPATH=${GITHUB_WORKSPACE} TEST_RESULT_DIR=${GITHUB_WORKSPACE}/src/github.com/vdaas/vald make test/hack/gotestfmt - working-directory: src/github.com/vdaas/vald + TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/hack/gotestfmt - name: Print tparse result run: | - tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald/test-hack-gotestfmt-result.json + tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/test-hack-gotestfmt-result.json From 5dea6da86a274f42248847e1cc173223a475f524 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 31 Jul 2023 16:08:35 +0900 Subject: [PATCH 049/139] deleted work space settings Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 327c4fcaf3..6baf22abaa 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -48,9 +48,6 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly - defaults: - run: - working-directory: src/github.com/vdaas/vald steps: - uses: actions/checkout@v3 with: From d6ca9bb06419560a1d13980c4931e755b92e0cb4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 11:28:15 +0900 Subject: [PATCH 050/139] delete git config settings Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 0f444d05ab..f6af6559be 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -42,10 +42,8 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ steps.ref.outputs.ref }} - - - name: set git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} + path: ${{ github.workspace }} + set-safe-directory: true - name: Setup QEMU uses: docker/setup-qemu-action@v2 From 4039920cacd720bc836f68cf809af532bf09dc1a Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 11:49:59 +0900 Subject: [PATCH 051/139] add duplicate workflow running rule and add indent Signed-off-by: hlts2 --- .github/actions/deploy-chaos-mesh/action.yaml | 3 +++ .github/actions/detect-docker-image-tags/action.yaml | 2 ++ .github/actions/docker-build/action.yaml | 6 ++++++ .github/actions/dump-context/action.yaml | 5 +++++ .../actions/e2e-deploy-vald-helm-operator/action.yaml | 11 +++++++---- .github/actions/e2e-deploy-vald/action.yaml | 11 +++++++---- .github/actions/notify-slack/action.yaml | 1 + .github/actions/setup-helm/action.yaml | 2 ++ .github/workflows/_docker-image.yaml | 2 +- 9 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/actions/deploy-chaos-mesh/action.yaml b/.github/actions/deploy-chaos-mesh/action.yaml index 27ae0b7d4a..8fdbcda2da 100644 --- a/.github/actions/deploy-chaos-mesh/action.yaml +++ b/.github/actions/deploy-chaos-mesh/action.yaml @@ -23,15 +23,18 @@ runs: echo "version=${CHAOS_MESH_VERSION}" >> $GITHUB_OUTPUT env: CHAOS_MESH_VERSION: ${{ inputs.chaos_mesh_version }} + - name: Check k3d shell: bash id: check_k3d run: | kubectl cluster-info + - name: Check Helm shell: bash run: | helm version + - name: Deploy Chaos Mesh shell: bash run: | diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index 75e278d2c7..4e56a56e5d 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -15,6 +15,7 @@ # name: "Detect Docker image tags" description: "Detect Docker image tags" + inputs: tag_name: description: "tag name to check whether exists or not" @@ -34,6 +35,7 @@ outputs: IMAGE_TAGS: description: "specifies image tags" value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} + runs: using: "composite" steps: diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index b97b4b49af..6d7b57f82d 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -15,6 +15,7 @@ # name: "Build Docker images" description: "Build Docker images and publish them" + inputs: target: description: "build target" @@ -44,6 +45,7 @@ outputs: EXTRA_TAGS: description: "extra tags" value: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }} + runs: using: "composite" steps: @@ -60,6 +62,7 @@ runs: echo "ALTER_IMAGE_NAME=${alter_image_name}" >> $GITHUB_OUTPUT env: TARGET: ${{ inputs.target }} + - name: Determine tag name shell: bash id: determine_tag_name @@ -87,6 +90,7 @@ runs: fi echo "PRIMARY_TAG is determined: ${primary_tag}" echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT + - name: Determine platforms shell: bash id: determine_platforms @@ -106,6 +110,7 @@ runs: echo "PLATFORMS=${platforms}" >> $GITHUB_OUTPUT env: TARGET_PLATFORMS: ${{ inputs.platforms }} + - name: Add extra tags shell: bash id: add_extra_tags @@ -121,6 +126,7 @@ runs: IMAGE_NAME: ${{ steps.image_name.outputs.IMAGE_NAME }} ALTER_IMAGE_NAME: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }} PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} + - name: Build and Push shell: bash id: build_and_push diff --git a/.github/actions/dump-context/action.yaml b/.github/actions/dump-context/action.yaml index 442e2821a4..4ab39331d5 100644 --- a/.github/actions/dump-context/action.yaml +++ b/.github/actions/dump-context/action.yaml @@ -9,26 +9,31 @@ runs: run: echo $JSON env: JSON: ${{toJSON(github)}} + - name: "Dump job context" shell: bash run: echo $JSON env: JSON: ${{ toJSON(job) }} + - name: "Dump steps context" shell: bash run: echo $JSON env: JSON: ${{toJSON(steps)}} + - name: "Dump runner context" shell: bash run: echo $JSON env: JSON: ${{toJSON(runner)}} + - name: "Dump strategy context" shell: bash run: echo $JSON env: JSON: ${{toJSON(strategy)}} + - name: "Dump matrix context" shell: bash run: echo $JSON diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index c270d6e32e..c9e6f00b1c 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -49,13 +49,14 @@ outputs: runs: using: "composite" steps: - - name: deploy Minio + - name: Deploy Minio id: deploy_minio shell: bash if: ${{ inputs.require_minio == 'true' }} run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - name: deploy vald helm operator from remote charts + + - name: Deploy vald helm operator from remote charts shell: bash id: deploy_vald_helm_operator_remote if: ${{ inputs.use_local_charts == 'false' }} @@ -65,14 +66,16 @@ runs: charts/vald-helm-operator/. sleep 3 - - name: deploy vald helm operator from local charts + + - name: Deploy vald helm operator from local charts shell: bash id: deploy_vald_helm_operator_local if: ${{ inputs.use_local_charts == 'true' }} run: | make k8s/vald-helm-operator/deploy sleep 3 - - name: deploy vald + + - name: Deploy vald shell: bash id: deploy_vald run: | diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 9fcf5b4b4c..1950213dbf 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -53,13 +53,14 @@ outputs: runs: using: "composite" steps: - - name: deploy Minio + - name: Deploy Minio id: deploy_minio shell: bash if: ${{ inputs.require_minio == 'true' }} run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - name: deploy vald from remote charts + + - name: Deploy vald from remote charts shell: bash id: deploy_vald_remote if: ${{ inputs.use_local_charts == 'false' }} @@ -83,7 +84,8 @@ runs: HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} - - name: deploy vald from local charts + + - name: Deploy vald from local charts shell: bash id: deploy_vald_local if: ${{ inputs.use_local_charts == 'true' }} @@ -103,7 +105,8 @@ runs: HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} - - name: get real pod name + + - name: Get real pod name shell: bash id: get_real_pod_name env: diff --git a/.github/actions/notify-slack/action.yaml b/.github/actions/notify-slack/action.yaml index fc776d119c..fa74f423e2 100644 --- a/.github/actions/notify-slack/action.yaml +++ b/.github/actions/notify-slack/action.yaml @@ -22,6 +22,7 @@ runs: - uses: technote-space/workflow-conclusion-action@v2 with: GITHUB_TOKEN: ${{ inputs.token }} + - uses: 8398a7/action-slack@v3 with: author_name: ${{ inputs.author_name }} diff --git a/.github/actions/setup-helm/action.yaml b/.github/actions/setup-helm/action.yaml index f1bcacc896..0d0890d4f4 100644 --- a/.github/actions/setup-helm/action.yaml +++ b/.github/actions/setup-helm/action.yaml @@ -19,9 +19,11 @@ runs: echo "version=${HELM_VERSION}" >> $GITHUB_OUTPUT env: HELM_VERSION: ${{ inputs.helm_version }} + - uses: azure/setup-helm@v3 with: version: ${{ steps.helm_version.outputs.version }} + - name: Check Helm version shell: bash run: | diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index f6af6559be..2a91078b39 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -14,7 +14,7 @@ on: default: "" concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}-${{ inputs.target }} cancel-in-progress: true jobs: From 7dde8f2e11b25995ff9418cb884169102977bc77 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 12:03:45 +0900 Subject: [PATCH 052/139] proto build for debug Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index f2a3add4a5..271c93e60a 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,9 +18,10 @@ on: push: branches: - main - paths: - - "apis/proto/**" - - "versions/GO_VERSION" + - refactor/ci/update-action + # paths: + # - "apis/proto/**" + # - "versions/GO_VERSION" pull_request: paths: - "apis/proto/**" From 3d3cd3bd46a3a9be8cf826a00eb5494122c1b18d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 12:09:29 +0900 Subject: [PATCH 053/139] add safe directory settings Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 271c93e60a..7c2c7032cb 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -41,9 +41,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Build protobuf run: | From 933c83cf12c98068eea0830fd4ca2087e7a50f12 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 12:15:33 +0900 Subject: [PATCH 054/139] delete debug code because build-protobuf workflow was successful Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 7c2c7032cb..cbccca10b5 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,10 +18,9 @@ on: push: branches: - main - - refactor/ci/update-action - # paths: - # - "apis/proto/**" - # - "versions/GO_VERSION" + paths: + - "apis/proto/**" + - "versions/GO_VERSION" pull_request: paths: - "apis/proto/**" From fef946b01cafb23ae856019c5f0e36e77e339a1c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 13:18:58 +0900 Subject: [PATCH 055/139] deleted unnecessary max-parallel configuration for matrics job Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 2a91078b39..d768f882f4 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -26,8 +26,6 @@ jobs: build: runs-on: ubuntu-latest - strategy: - max-parallel: 4 if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} steps: - name: Get ref From a094f44cadd33bed26e68a484a590f3437593e52 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 13:44:43 +0900 Subject: [PATCH 056/139] set safe directory to docker image scan workflow Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 7 ++++--- .github/workflows/_docker-image_scan.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index d768f882f4..c3662fe11b 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -1,5 +1,4 @@ name: "Build docker image" - on: workflow_call: inputs: @@ -40,8 +39,10 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ steps.ref.outputs.ref }} - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/_docker-image_scan.yaml b/.github/workflows/_docker-image_scan.yaml index 7dcb028962..fe1caa39e9 100644 --- a/.github/workflows/_docker-image_scan.yaml +++ b/.github/workflows/_docker-image_scan.yaml @@ -1,5 +1,4 @@ name: "Docker image scanning" - on: workflow_call: inputs: @@ -13,9 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Build the Docker image id: build_image From bace21d75680ccd04b2fbbab09b920da776cdcf9 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 13:59:25 +0900 Subject: [PATCH 057/139] set safe directory to build-binary workflow and refactoring docker image scan workflow Signed-off-by: hlts2 --- .github/workflows/_docker-image_scan.yaml | 5 +++-- .github/workflows/build-binaries.yml | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_docker-image_scan.yaml b/.github/workflows/_docker-image_scan.yaml index fe1caa39e9..1e20a00a42 100644 --- a/.github/workflows/_docker-image_scan.yaml +++ b/.github/workflows/_docker-image_scan.yaml @@ -20,12 +20,13 @@ jobs: - name: Build the Docker image id: build_image run: | - make docker/build/${{ inputs.target }} - imagename=`make docker/name/${{ inputs.target }}` + make docker/build/${TARGET} + imagename=`make docker/name/${TARGET}` docker tag ${imagename} ${imagename}:${{ github.sha }} echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT env: DOCKER_BUILDKIT: 1 + TARGET: ${{ inputs.target }} - name: Scan the Docker image uses: ./.github/actions/scan-docker-image diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index c749175246..dab4837d72 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -34,8 +34,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 10 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: ./.github/actions/setup-go From dba5aece6b3b7b0acc91abb88682fccf9d51bc8b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 15:49:44 +0900 Subject: [PATCH 058/139] debug build binary workflow Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 43 ++++++++++++++++------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index dab4837d72..7a311ded29 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -19,6 +19,11 @@ on: types: - created + # TODO: Delete it later. + push: + branches: + - "refactor/ci/update-action" + jobs: dump-contexts-to-log: runs-on: ubuntu-latest @@ -45,23 +50,23 @@ jobs: run: | make binary/build/zip - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: artifacts-linux - path: ./artifacts/ + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # name: artifacts-linux + # path: ./artifacts/ - publish: - runs-on: ubuntu-latest - needs: - - build-linux - steps: - - uses: actions/download-artifact@v2 - with: - name: artifacts-linux - path: tmp/linux - - - uses: shogo82148/actions-upload-release-asset@v1 - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: tmp/linux/vald-*.zip + # publish: + # runs-on: ubuntu-latest + # needs: + # - build-linux + # steps: + # - uses: actions/download-artifact@v2 + # with: + # name: artifacts-linux + # path: tmp/linux + # + # - uses: shogo82148/actions-upload-release-asset@v1 + # with: + # upload_url: ${{ github.event.release.upload_url }} + # asset_path: tmp/linux/vald-*.zip From 931349939b19826c989e1d4695b2d58b4e7336e6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 16:06:11 +0900 Subject: [PATCH 059/139] Revert "debug build binary workflow" This reverts commit 35e5d14a5dc5c853c9d0db1a3ced3d37d3387886. --- .github/workflows/build-binaries.yml | 43 ++++++++++++---------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 7a311ded29..dab4837d72 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -19,11 +19,6 @@ on: types: - created - # TODO: Delete it later. - push: - branches: - - "refactor/ci/update-action" - jobs: dump-contexts-to-log: runs-on: ubuntu-latest @@ -50,23 +45,23 @@ jobs: run: | make binary/build/zip - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # name: artifacts-linux - # path: ./artifacts/ + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: artifacts-linux + path: ./artifacts/ - # publish: - # runs-on: ubuntu-latest - # needs: - # - build-linux - # steps: - # - uses: actions/download-artifact@v2 - # with: - # name: artifacts-linux - # path: tmp/linux - # - # - uses: shogo82148/actions-upload-release-asset@v1 - # with: - # upload_url: ${{ github.event.release.upload_url }} - # asset_path: tmp/linux/vald-*.zip + publish: + runs-on: ubuntu-latest + needs: + - build-linux + steps: + - uses: actions/download-artifact@v2 + with: + name: artifacts-linux + path: tmp/linux + + - uses: shogo82148/actions-upload-release-asset@v1 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: tmp/linux/vald-*.zip From a835552f07cb98e6a159b1b934a603899f0f7500 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 16:09:35 +0900 Subject: [PATCH 060/139] debug build protobuf workflow Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index cbccca10b5..d530f6ea78 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,13 +18,14 @@ on: push: branches: - main - paths: - - "apis/proto/**" - - "versions/GO_VERSION" - pull_request: - paths: - - "apis/proto/**" - - "versions/GO_VERSION" + - refactor/ci/update-action + # paths: + # - "apis/proto/**" + # - "versions/GO_VERSION" + # pull_request: + # paths: + # - "apis/proto/**" + # - "versions/GO_VERSION" jobs: dump-contexts-to-log: @@ -34,7 +35,6 @@ jobs: - uses: ./.github/actions/dump-context build: - name: build all runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly From e6643b0d14fc4fd9fcf857f6dc5d4c0ff6e7c5ca Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 16:32:28 +0900 Subject: [PATCH 061/139] delete debug workflow rule Signed-off-by: hlts2 --- .github/workflows/build-protobuf.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index d530f6ea78..6aab18aa03 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -18,14 +18,13 @@ on: push: branches: - main - - refactor/ci/update-action - # paths: - # - "apis/proto/**" - # - "versions/GO_VERSION" - # pull_request: - # paths: - # - "apis/proto/**" - # - "versions/GO_VERSION" + paths: + - "apis/proto/**" + - "versions/GO_VERSION" + pull_request: + paths: + - "apis/proto/**" + - "versions/GO_VERSION" jobs: dump-contexts-to-log: From 90869c1901cbabbae6172694de76c6d9351cc96e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 16:52:27 +0900 Subject: [PATCH 062/139] add debug rule to chatops help workflow and refactoring Signed-off-by: hlts2 --- .github/workflows/chatops-help.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index 78161b6e8d..a3707c4cdb 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -15,6 +15,10 @@ # name: "ChatOps help" on: + push: + branches: + - refactor/ci/update-action + pull_request: types: [opened] @@ -25,11 +29,10 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - print-help: - name: print help + print: runs-on: ubuntu-latest steps: - - name: comment to PR + - name: Comment to PR run: | curl -o chatops_commands.md https://raw.githubusercontent.com/vdaas/vald/main/.github/chatops_commands.md BODY=`cat chatops_commands.md | sed ':a;N;$!ba;s/\n/\\\\n/g'` From e398ac30c689f31cefa86aba03bf05cb268f681a Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:01:19 +0900 Subject: [PATCH 063/139] add debug url for comment api Signed-off-by: hlts2 --- .github/workflows/chatops-help.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index a3707c4cdb..aff41d5a7c 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -45,5 +45,6 @@ jobs: $API_URL env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: ${{ github.event.pull_request.comments_url }} + # API_URL: ${{ github.event.pull_request.comments_url }} + API_URL: "https://api.github.com/repos/vdaas/vald/issues/2141/comments" PR_NUM: ${{ github.event.issue.number }} From 4c20ca6e7f07bbb569b9b390f911cf77e094f34c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:02:30 +0900 Subject: [PATCH 064/139] delete debug workflow rule for chaotops help Signed-off-by: hlts2 --- .github/workflows/chatops-help.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index aff41d5a7c..7ad564d8be 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -15,10 +15,6 @@ # name: "ChatOps help" on: - push: - branches: - - refactor/ci/update-action - pull_request: types: [opened] @@ -45,6 +41,5 @@ jobs: $API_URL env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - # API_URL: ${{ github.event.pull_request.comments_url }} - API_URL: "https://api.github.com/repos/vdaas/vald/issues/2141/comments" + API_URL: ${{ github.event.pull_request.comments_url }} PR_NUM: ${{ github.event.issue.number }} From 7247e71fd8c3ad5c7f382088a2a3854a7f6e0642 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:12:52 +0900 Subject: [PATCH 065/139] set safe directory to codeql workflow Signed-off-by: hlts2 --- .github/workflows/codeql-analysis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a8166dff8..9904976f29 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -51,8 +51,10 @@ jobs: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Initialize CodeQL uses: github/codeql-action/init@v2 From 5ea7fe7e2ff92b8d73d46bcb599ab3bba4a2c135 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:34:28 +0900 Subject: [PATCH 066/139] set safe directory to coverage workflow Signed-off-by: hlts2 --- .github/workflows/coverage.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fd982eadd0..7f007ff8bf 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -38,7 +38,6 @@ jobs: - uses: ./.github/actions/dump-context coverage: - name: Coverage runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly @@ -46,8 +45,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 10 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Run coverage continue-on-error: true From 5f011eef664d130b5a35ea713eb36947c596d711 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:55:26 +0900 Subject: [PATCH 067/139] delete unnecessary job name Signed-off-by: hlts2 --- .github/workflows/detect-internal-config-changes.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index dcfe248bd4..935268ee70 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -28,7 +28,6 @@ jobs: - uses: ./.github/actions/dump-context warning: - name: warn internal config changes runs-on: ubuntu-latest steps: - name: Warn From 2b271336055b1111f56840db2c99416e4d46cabe Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:56:54 +0900 Subject: [PATCH 068/139] debug for internal config changes workflow Signed-off-by: hlts2 --- internal/config/backoff.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/config/backoff.go b/internal/config/backoff.go index 2e3dd95d3d..58f1d97122 100644 --- a/internal/config/backoff.go +++ b/internal/config/backoff.go @@ -30,6 +30,7 @@ type Backoff struct { EnableErrorLog bool `json:"enable_error_log" yaml:"enable_error_log"` } +// Bind binds the actual data from the Backoff receiver fields. // Bind binds the actual data from the Backoff receiver fields. func (b *Backoff) Bind() *Backoff { b.InitialDuration = GetActualValue(b.InitialDuration) From 50b81ae9febe36f1115397e6849ab075fbdc3c6d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 17:58:17 +0900 Subject: [PATCH 069/139] Revert "debug for internal config changes workflow" This reverts commit 5d68cd3e6c962bd5e4a2d4ed46fa4f69b0ab6bb1. --- internal/config/backoff.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/config/backoff.go b/internal/config/backoff.go index 58f1d97122..2e3dd95d3d 100644 --- a/internal/config/backoff.go +++ b/internal/config/backoff.go @@ -30,7 +30,6 @@ type Backoff struct { EnableErrorLog bool `json:"enable_error_log" yaml:"enable_error_log"` } -// Bind binds the actual data from the Backoff receiver fields. // Bind binds the actual data from the Backoff receiver fields. func (b *Backoff) Bind() *Backoff { b.InitialDuration = GetActualValue(b.InitialDuration) From 2e826cc15ddcaa10ba71079db49e369dd3c845bf Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 18:17:51 +0900 Subject: [PATCH 070/139] fix docker image build rule Signed-off-by: hlts2 --- .github/actions/scan-docker-image/action.yaml | 8 ++++---- .github/workflows/_docker-image.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index caf0cdea6d..49fe889ab8 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -29,7 +29,7 @@ runs: output: "trivy-results.sarif" severity: ${{ inputs.severity }} - # - name: Upload Trivy scan results to Security tab - # uses: github/codeql-action/upload-sarif@v2 - # with: - # sarif_file: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index c3662fe11b..625751a8e9 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -13,7 +13,7 @@ on: default: "" concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}-${{ inputs.target }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} cancel-in-progress: true jobs: From 368348d49fc65b7713c8eb013fffa7bdea2e3383 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 18:30:36 +0900 Subject: [PATCH 071/139] fix concurrency group name for reusable docker image workflow Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 625751a8e9..c3662fe11b 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -13,7 +13,7 @@ on: default: "" concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}-${{ inputs.target }} cancel-in-progress: true jobs: From a5c069f960d449dde4c449e8548d535ceb7eb18b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 1 Aug 2023 18:59:38 +0900 Subject: [PATCH 072/139] deleted debug rule of docker image scan workflow Signed-off-by: hlts2 --- .github/workflows/dockers-image-scan.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 0db872da78..ee00db8e1f 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -18,11 +18,6 @@ on: schedule: - cron: "0 1 * * *" - # TODO: Delete it later. - push: - branches: - - "refactor/ci/update-action" - concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} cancel-in-progress: true From 46de3fb7aebc006c3961d0517163b7ec5c410c3b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 11:24:03 +0900 Subject: [PATCH 073/139] add action to setup yq Signed-off-by: hlts2 --- .github/actions/setup-yq/action.yaml | 34 ++++++++++++++++++++++++++++ .github/workflows/chatops.yml | 20 ++++------------ Makefile | 4 ++++ 3 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 .github/actions/setup-yq/action.yaml diff --git a/.github/actions/setup-yq/action.yaml b/.github/actions/setup-yq/action.yaml new file mode 100644 index 0000000000..7831683151 --- /dev/null +++ b/.github/actions/setup-yq/action.yaml @@ -0,0 +1,34 @@ +name: "Setup yq environment" +description: "GitHub Action for setting up yq" + +inputs: + yq_version: + description: "The yq version to use. The default version is `versions/YQ_VERSION`" + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Fetch yq version + id: yq_version + shell: bash + run: | + YQ_VERSION=${YQ_VERSION:-`make version/yq`} + + echo "version=${YQ_VERSION}" >> $GITHUB_OUTPUT + env: + YQ_VERSION: ${{ inputs.yq_version }} + + - name: Install yq + shell: bash + run: | + curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + chmod a+x /usr/local/bin/yq + env: + YQ_VERSION: ${{ steps.yq_version.outputs.version }} + + - name: Check yq version + shell: bash + run: | + yq --version diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index 27839ca146..d4ca42a0fe 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -41,11 +41,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq + - uses: ./.github/actions/setup-yq if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' @@ -100,11 +97,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq + - uses: ./.github/actions/setup-yq if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' @@ -237,11 +231,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq + - uses: ./.github/actions/setup-yq if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' @@ -394,11 +385,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: install yq + - uses: ./.github/actions/setup-yq if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' - run: | - sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' diff --git a/Makefile b/Makefile index ec122ea034..50ca6a7874 100644 --- a/Makefile +++ b/Makefile @@ -504,6 +504,10 @@ version/kind: version/helm: @echo $(HELM_VERSION) +.PHONY: version/yq +version/yq: + @echo $(YQ_VERSION) + .PHONY: version/valdcli version/valdcli: @echo $(VALDCLI_VERSION) From 31df6eaa0a2621cc17447eed0556c4c549e1ce91 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 11:45:40 +0900 Subject: [PATCH 074/139] deleted unnecessary environment Signed-off-by: hlts2 --- .github/workflows/chatops.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index d4ca42a0fe..b27c2b01d1 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -18,9 +18,6 @@ on: issue_comment: types: [created] -env: - YQ_VERSION: 3.2.1 - jobs: dump-contexts-to-log: runs-on: ubuntu-latest From 37b11363574d586b690f90cb16c5a7a48c335ee6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 12:39:15 +0900 Subject: [PATCH 075/139] add new workflow rule for docker build Signed-off-by: hlts2 --- .github/workflows/dockers-agent-ngt-image.yml | 3 +++ .github/workflows/dockers-agent-sidecar-image.yml | 3 +++ .github/workflows/dockers-ci-container-image.yml | 3 +++ .github/workflows/dockers-dev-container-image.yml | 3 +++ .github/workflows/dockers-discoverer-k8s-image.yml | 3 +++ .github/workflows/dockers-gateway-filter-image.yml | 3 +++ .github/workflows/dockers-gateway-lb-image.yml | 3 +++ .github/workflows/dockers-helm-operator-image.yml | 3 +++ 8 files changed, 24 insertions(+) diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index ef163c654b..568ff361e6 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" @@ -41,6 +42,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" @@ -57,6 +59,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml index 375a89e281..2bf26f8c29 100644 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ b/.github/workflows/dockers-agent-sidecar-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" @@ -41,6 +42,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" @@ -57,6 +59,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml index d73065c7b4..baf5c21378 100644 --- a/.github/workflows/dockers-ci-container-image.yml +++ b/.github/workflows/dockers-ci-container-image.yml @@ -20,6 +20,7 @@ on: - main paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" @@ -29,6 +30,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" @@ -38,6 +40,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml index 0f9fbb6d03..7994d1f7d8 100644 --- a/.github/workflows/dockers-dev-container-image.yml +++ b/.github/workflows/dockers-dev-container-image.yml @@ -20,18 +20,21 @@ on: - main paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml index dff4b8339b..d0d12c260d 100644 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ b/.github/workflows/dockers-discoverer-k8s-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" @@ -39,6 +40,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" @@ -53,6 +55,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" diff --git a/.github/workflows/dockers-gateway-filter-image.yml b/.github/workflows/dockers-gateway-filter-image.yml index 537ff837fb..a8a1f56027 100644 --- a/.github/workflows/dockers-gateway-filter-image.yml +++ b/.github/workflows/dockers-gateway-filter-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" @@ -42,6 +43,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" @@ -59,6 +61,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" diff --git a/.github/workflows/dockers-gateway-lb-image.yml b/.github/workflows/dockers-gateway-lb-image.yml index 43a02279f9..69e80cb921 100644 --- a/.github/workflows/dockers-gateway-lb-image.yml +++ b/.github/workflows/dockers-gateway-lb-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" @@ -42,6 +43,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" @@ -59,6 +61,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index 3cb4f5ed8e..48b68fca3f 100644 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" @@ -37,6 +38,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" @@ -49,6 +51,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" From d0d231677f3e50a4b568103e4e164fa86d51cfd0 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 14:11:06 +0900 Subject: [PATCH 076/139] rename docekr image scan workflow Signed-off-by: hlts2 --- ...r-image_scan.yaml => _docker-image-scan.yaml} | 0 .github/workflows/dockers-image-scan.yml | 16 ++++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{_docker-image_scan.yaml => _docker-image-scan.yaml} (100%) diff --git a/.github/workflows/_docker-image_scan.yaml b/.github/workflows/_docker-image-scan.yaml similarity index 100% rename from .github/workflows/_docker-image_scan.yaml rename to .github/workflows/_docker-image-scan.yaml diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index ee00db8e1f..61835d6a5c 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -30,41 +30,41 @@ jobs: - uses: ./.github/actions/dump-context agent-ngt: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: agent-ngt agent-sidecar: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: agent-sidecar discoverer-k8s: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: discoverer-k8s manager-index: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: manager-index operator-helm: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: operator/helm loadtest: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: loadtest gateway-lb: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: gateway-lb gateway-filter: - uses: ./.github/workflows/_docker-image_scan.yaml + uses: ./.github/workflows/_docker-image-scan.yaml with: target: gateway-filter From 19af0bf211448a058e5a29e6ddc046c61d4aa6e6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 15:25:29 +0900 Subject: [PATCH 077/139] add workflow to dispatch vald client Signed-off-by: hlts2 deleted unnecessary workflow Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 2 +- .../workflows/_protobuf-update-dispatch.yaml | 43 ++++++++++++ .../workflows/update-protobuf-client-go.yaml | 16 +++++ .../update-protobuf-client-java.yaml | 16 +++++ .../update-protobuf-client-node.yaml | 16 +++++ .../update-protobuf-client-python.yaml | 16 +++++ .github/workflows/update-protobuf.yml | 66 ------------------- 7 files changed, 108 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/_protobuf-update-dispatch.yaml create mode 100644 .github/workflows/update-protobuf-client-go.yaml create mode 100644 .github/workflows/update-protobuf-client-java.yaml create mode 100644 .github/workflows/update-protobuf-client-node.yaml create mode 100644 .github/workflows/update-protobuf-client-python.yaml delete mode 100644 .github/workflows/update-protobuf.yml diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index c3662fe11b..05ab1e650c 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -4,7 +4,7 @@ on: inputs: target: type: string - description: 'Build target. e.g. `agent-ngt`' + description: "Build target. e.g. `agent-ngt`" required: true platforms: type: string diff --git a/.github/workflows/_protobuf-update-dispatch.yaml b/.github/workflows/_protobuf-update-dispatch.yaml new file mode 100644 index 0000000000..61395bb2c8 --- /dev/null +++ b/.github/workflows/_protobuf-update-dispatch.yaml @@ -0,0 +1,43 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Protobuf update dispatch" +on: + workflow_call: + inputs: + repo: + type: string + description: "Dispatch target repository" + required: true + default: "" + +jobs: + dump-contexts-to-log: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + + dispatch: + runs-on: ubuntu-latest + steps: + - name: Dispatch + run: | + curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + env: + REPO: ${{ inputs.repo }} + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} + diff --git a/.github/workflows/update-protobuf-client-go.yaml b/.github/workflows/update-protobuf-client-go.yaml new file mode 100644 index 0000000000..6ee0cab213 --- /dev/null +++ b/.github/workflows/update-protobuf-client-go.yaml @@ -0,0 +1,16 @@ +name: "Update protobuf: client-go" +on: + workflow_dispatch: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + client-go: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vdaas/vald-client-go + secrets: inherit diff --git a/.github/workflows/update-protobuf-client-java.yaml b/.github/workflows/update-protobuf-client-java.yaml new file mode 100644 index 0000000000..8e626a53a7 --- /dev/null +++ b/.github/workflows/update-protobuf-client-java.yaml @@ -0,0 +1,16 @@ +name: "Update protobuf: client-java" +on: + workflow_dispatch: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + client-jave: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vdaas/vald-client-java + secrets: inherit diff --git a/.github/workflows/update-protobuf-client-node.yaml b/.github/workflows/update-protobuf-client-node.yaml new file mode 100644 index 0000000000..c349a3c77d --- /dev/null +++ b/.github/workflows/update-protobuf-client-node.yaml @@ -0,0 +1,16 @@ +name: "Update protobuf: client-node" +on: + workflow_dispatch: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + client-node: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vdaas/vald-client-node + secrets: inherit diff --git a/.github/workflows/update-protobuf-client-python.yaml b/.github/workflows/update-protobuf-client-python.yaml new file mode 100644 index 0000000000..f9b9b7f706 --- /dev/null +++ b/.github/workflows/update-protobuf-client-python.yaml @@ -0,0 +1,16 @@ +name: "Update protobuf: client-python" +on: + workflow_dispatch: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + client-python: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vdaas/vald-client-python + secrets: inherit diff --git a/.github/workflows/update-protobuf.yml b/.github/workflows/update-protobuf.yml deleted file mode 100644 index 5fb97156f8..0000000000 --- a/.github/workflows/update-protobuf.yml +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: update protobuf -on: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - dump-contexts-to-log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - dispatches: - name: Dispatches - runs-on: ubuntu-latest - steps: - - name: client-go - run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' - env: - REPO: vdaas/vald-client-go - USER: ${{ secrets.DISPATCH_USER }} - TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - - name: client-java - run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' - env: - REPO: vdaas/vald-client-java - USER: ${{ secrets.DISPATCH_USER }} - TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - - name: client-python - run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' - env: - REPO: vdaas/vald-client-python - USER: ${{ secrets.DISPATCH_USER }} - TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - - name: client-node - run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' - env: - REPO: vdaas/vald-client-node - USER: ${{ secrets.DISPATCH_USER }} - TOKEN: ${{ secrets.DISPATCH_TOKEN }} From 58200725e5f40a4084f3af67295aec9b7f0469e5 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 17:07:04 +0900 Subject: [PATCH 078/139] deleted unnecessary step Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 05ab1e650c..9de800c8dc 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -76,10 +76,6 @@ jobs: platforms: ${{ inputs.platforms }} builder: ${{ steps.buildx.outputs.name }} - - name: Initialize CodeQL - if: startsWith( github.ref, 'refs/tags/') - uses: github/codeql-action/init@v2 - - name: Scan the Docker image if: startsWith( github.ref, 'refs/tags/') uses: ./.github/actions/scan-docker-image From 394f5535cb5cdc2bf91923b40e02c34baf87db96 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 17:14:11 +0900 Subject: [PATCH 079/139] debug for scan docker image Signed-off-by: hlts2 --- .github/actions/scan-docker-image/action.yaml | 8 ++++---- .github/workflows/_docker-image.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index 49fe889ab8..caf0cdea6d 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -29,7 +29,7 @@ runs: output: "trivy-results.sarif" severity: ${{ inputs.severity }} - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" + # - name: Upload Trivy scan results to Security tab + # uses: github/codeql-action/upload-sarif@v2 + # with: + # sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 9de800c8dc..5a8a0517ac 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -77,7 +77,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} - name: Scan the Docker image - if: startsWith( github.ref, 'refs/tags/') + # if: startsWith( github.ref, 'refs/tags/') uses: ./.github/actions/scan-docker-image with: image_ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" From 44d3bc45e007a075db31db6ad26367c70026521d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 17:16:36 +0900 Subject: [PATCH 080/139] Revert "debug for scan docker image" This reverts commit f6ee24145b6213457f08cbf19b6025c5aadb5e53. --- .github/actions/scan-docker-image/action.yaml | 8 ++++---- .github/workflows/_docker-image.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index caf0cdea6d..49fe889ab8 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -29,7 +29,7 @@ runs: output: "trivy-results.sarif" severity: ${{ inputs.severity }} - # - name: Upload Trivy scan results to Security tab - # uses: github/codeql-action/upload-sarif@v2 - # with: - # sarif_file: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 5a8a0517ac..9de800c8dc 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -77,7 +77,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} - name: Scan the Docker image - # if: startsWith( github.ref, 'refs/tags/') + if: startsWith( github.ref, 'refs/tags/') uses: ./.github/actions/scan-docker-image with: image_ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" From d2ed93d160f02f7cad8f910eaf82620bd9d31cbb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 17:22:08 +0900 Subject: [PATCH 081/139] add safe directory settings to test/hack-test workflow Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 8 ++++---- .github/workflows/test.yml | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 6baf22abaa..821612ad5c 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -50,10 +50,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - fetch-depth: 1 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Install Proto dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c057f3d266..ffb46c3d7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,9 +48,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Run tests for cmd packages / gotestfmt run: | @@ -67,9 +68,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Run tests for internal packages / gotestfmt run: | @@ -86,9 +88,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Run tests for pkg packages / gotestfmt run: | From 7656ad70c737904903355972d92dcb108ce4bde2 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 17:48:48 +0900 Subject: [PATCH 082/139] fix fails hack test Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 821612ad5c..2175eac65f 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -36,6 +36,10 @@ on: - "hack/license/**" - "hack/tools/**" +env: + GOPATH: ${{ github.workspace }} + PROJECT_ROOT_DIR: src/github.com/vdaas/vald + jobs: dump-contexts-to-log: runs-on: ubuntu-latest @@ -50,6 +54,8 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 + with: + path: ${{ env.PROJECT_ROOT_DIR }} - name: Set Git config run: | From fab095adfb3159b884195fd154026b9e6750024e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 18:14:30 +0900 Subject: [PATCH 083/139] refactor existing workflow Signed-off-by: hlts2 --- .../workflows/_protobuf-update-dispatch.yaml | 8 +-- .../workflows/protobuf-update-dispatch.yaml | 55 +++++++++++++++++++ .../workflows/update-protobuf-client-go.yaml | 16 ------ .../update-protobuf-client-java.yaml | 16 ------ .../update-protobuf-client-node.yaml | 16 ------ .../update-protobuf-client-python.yaml | 16 ------ 6 files changed, 56 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/protobuf-update-dispatch.yaml delete mode 100644 .github/workflows/update-protobuf-client-go.yaml delete mode 100644 .github/workflows/update-protobuf-client-java.yaml delete mode 100644 .github/workflows/update-protobuf-client-node.yaml delete mode 100644 .github/workflows/update-protobuf-client-python.yaml diff --git a/.github/workflows/_protobuf-update-dispatch.yaml b/.github/workflows/_protobuf-update-dispatch.yaml index 61395bb2c8..edd15b81e0 100644 --- a/.github/workflows/_protobuf-update-dispatch.yaml +++ b/.github/workflows/_protobuf-update-dispatch.yaml @@ -24,18 +24,12 @@ on: default: "" jobs: - dump-contexts-to-log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - dispatch: runs-on: ubuntu-latest steps: - name: Dispatch run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + curl --fail -u "${USER}:${TOKEN}" -X POST https://api.github.com/repos/vdaas/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' env: REPO: ${{ inputs.repo }} USER: ${{ secrets.DISPATCH_USER }} diff --git a/.github/workflows/protobuf-update-dispatch.yaml b/.github/workflows/protobuf-update-dispatch.yaml new file mode 100644 index 0000000000..d9f9961a70 --- /dev/null +++ b/.github/workflows/protobuf-update-dispatch.yaml @@ -0,0 +1,55 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Update protobuf" +on: + workflow_dispatch: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + dump-contexts-to-log: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/dump-context + + client-go: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vald-client-go + secrets: inherit + + client-jave: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vald-client-java + secrets: inherit + + client-node: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vald-client-node + secrets: inherit + + client-python: + uses: ./.github/workflows/_protobuf-update-dispatch.yaml + with: + repo: vald-client-python + secrets: inherit diff --git a/.github/workflows/update-protobuf-client-go.yaml b/.github/workflows/update-protobuf-client-go.yaml deleted file mode 100644 index 6ee0cab213..0000000000 --- a/.github/workflows/update-protobuf-client-go.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: "Update protobuf: client-go" -on: - workflow_dispatch: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - client-go: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml - with: - repo: vdaas/vald-client-go - secrets: inherit diff --git a/.github/workflows/update-protobuf-client-java.yaml b/.github/workflows/update-protobuf-client-java.yaml deleted file mode 100644 index 8e626a53a7..0000000000 --- a/.github/workflows/update-protobuf-client-java.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: "Update protobuf: client-java" -on: - workflow_dispatch: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - client-jave: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml - with: - repo: vdaas/vald-client-java - secrets: inherit diff --git a/.github/workflows/update-protobuf-client-node.yaml b/.github/workflows/update-protobuf-client-node.yaml deleted file mode 100644 index c349a3c77d..0000000000 --- a/.github/workflows/update-protobuf-client-node.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: "Update protobuf: client-node" -on: - workflow_dispatch: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - client-node: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml - with: - repo: vdaas/vald-client-node - secrets: inherit diff --git a/.github/workflows/update-protobuf-client-python.yaml b/.github/workflows/update-protobuf-client-python.yaml deleted file mode 100644 index f9b9b7f706..0000000000 --- a/.github/workflows/update-protobuf-client-python.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: "Update protobuf: client-python" -on: - workflow_dispatch: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - client-python: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml - with: - repo: vdaas/vald-client-python - secrets: inherit From 7a279d9a5b5a25f08a70a4c08c464eb9799b84c3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Wed, 2 Aug 2023 18:31:31 +0900 Subject: [PATCH 084/139] rename workflow name for proto update Signed-off-by: hlts2 --- ...rotobuf-update-dispatch.yaml => _update-protobuf.yaml} | 2 +- ...protobuf-update-dispatch.yaml => update-protobuf.yaml} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{_protobuf-update-dispatch.yaml => _update-protobuf.yaml} (97%) rename .github/workflows/{protobuf-update-dispatch.yaml => update-protobuf.yaml} (83%) diff --git a/.github/workflows/_protobuf-update-dispatch.yaml b/.github/workflows/_update-protobuf.yaml similarity index 97% rename from .github/workflows/_protobuf-update-dispatch.yaml rename to .github/workflows/_update-protobuf.yaml index edd15b81e0..b6b28a0fa8 100644 --- a/.github/workflows/_protobuf-update-dispatch.yaml +++ b/.github/workflows/_update-protobuf.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Protobuf update dispatch" +name: "Update protobuf" on: workflow_call: inputs: diff --git a/.github/workflows/protobuf-update-dispatch.yaml b/.github/workflows/update-protobuf.yaml similarity index 83% rename from .github/workflows/protobuf-update-dispatch.yaml rename to .github/workflows/update-protobuf.yaml index d9f9961a70..afe48500ed 100644 --- a/.github/workflows/protobuf-update-dispatch.yaml +++ b/.github/workflows/update-protobuf.yaml @@ -31,25 +31,25 @@ jobs: - uses: ./.github/actions/dump-context client-go: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml + uses: ./.github/workflows/_update-protobuf.yaml with: repo: vald-client-go secrets: inherit client-jave: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml + uses: ./.github/workflows/_update-protobuf.yaml with: repo: vald-client-java secrets: inherit client-node: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml + uses: ./.github/workflows/_update-protobuf.yaml with: repo: vald-client-node secrets: inherit client-python: - uses: ./.github/workflows/_protobuf-update-dispatch.yaml + uses: ./.github/workflows/_update-protobuf.yaml with: repo: vald-client-python secrets: inherit From 89733d3d6d5cf09aabec6453fc0183f489abeee7 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:00:48 +0900 Subject: [PATCH 085/139] add working directory for hack test Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 2175eac65f..51dd32c141 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -52,6 +52,9 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly + defaults: + run: + working-directory: ${{ env.GOPATH }}/${{ env.PROJECT_ROOT_DIR }} steps: - uses: actions/checkout@v3 with: From 59495890900197975d11be8995282fd3696a84eb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:12:21 +0900 Subject: [PATCH 086/139] set safe directory and rename workflow file Signed-off-by: hlts2 --- ...plate.yml => update-pull-request-and-issue-template.yml} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename .github/workflows/{update_pull_request_and_issue_template.yml => update-pull-request-and-issue-template.yml} (97%) diff --git a/.github/workflows/update_pull_request_and_issue_template.yml b/.github/workflows/update-pull-request-and-issue-template.yml similarity index 97% rename from .github/workflows/update_pull_request_and_issue_template.yml rename to .github/workflows/update-pull-request-and-issue-template.yml index b3a6bc5f04..4273b17b63 100644 --- a/.github/workflows/update_pull_request_and_issue_template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -37,8 +37,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: crazy-max/ghaction-import-gpg@v4 with: From 74e4d56225b46476847a98108a77c4d1fdda325d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:15:52 +0900 Subject: [PATCH 087/139] add debug rule for update pull request template workflow Signed-off-by: hlts2 --- .../workflows/update-pull-request-and-issue-template.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update-pull-request-and-issue-template.yml index 4273b17b63..bf92368a6c 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -18,6 +18,7 @@ on: push: branches: - main + - refactor/ci/update-action paths: - "versions/GO_VERSION" - "versions/NGT_VERSION" @@ -84,7 +85,7 @@ jobs: --request POST \ --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ $API_URL - env: - GITHUB_USER: ${{ secrets.DISPATCH_USER }} - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: https://api.github.com/repos/vdaas/vald/pulls + # env: + # GITHUB_USER: ${{ secrets.DISPATCH_USER }} + # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + # API_URL: https://api.github.com/repos/vdaas/vald/pulls From dcdcbe04f07d48571520652a4d6b45b824f43397 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:17:23 +0900 Subject: [PATCH 088/139] Revert "add debug rule for update pull request template workflow" This reverts commit 967dff9b1cae03eeb5f1a8f59bf8a4e77854cca6. --- .../workflows/update-pull-request-and-issue-template.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update-pull-request-and-issue-template.yml index bf92368a6c..4273b17b63 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -18,7 +18,6 @@ on: push: branches: - main - - refactor/ci/update-action paths: - "versions/GO_VERSION" - "versions/NGT_VERSION" @@ -85,7 +84,7 @@ jobs: --request POST \ --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ $API_URL - # env: - # GITHUB_USER: ${{ secrets.DISPATCH_USER }} - # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - # API_URL: https://api.github.com/repos/vdaas/vald/pulls + env: + GITHUB_USER: ${{ secrets.DISPATCH_USER }} + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: https://api.github.com/repos/vdaas/vald/pulls From 81496a23fcee88b8373536502e11eed6ba9c6d36 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:18:03 +0900 Subject: [PATCH 089/139] add debug rule for update pull request template workflow Signed-off-by: hlts2 --- .../update-pull-request-and-issue-template.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update-pull-request-and-issue-template.yml index 4273b17b63..d050247239 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -18,10 +18,11 @@ on: push: branches: - main - paths: - - "versions/GO_VERSION" - - "versions/NGT_VERSION" - - "versions/KUBECTL_VERSION" + - refactor/ci/update-action + # paths: + # - "versions/GO_VERSION" + # - "versions/NGT_VERSION" + # - "versions/KUBECTL_VERSION" jobs: dump-contexts-to-log: @@ -84,7 +85,7 @@ jobs: --request POST \ --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ $API_URL - env: - GITHUB_USER: ${{ secrets.DISPATCH_USER }} - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: https://api.github.com/repos/vdaas/vald/pulls + # env: + # GITHUB_USER: ${{ secrets.DISPATCH_USER }} + # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + # API_URL: https://api.github.com/repos/vdaas/vald/pulls From 1e8bcbb1af72b6c81cad5ee4658b378cdd382dc6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:22:19 +0900 Subject: [PATCH 090/139] Revert "add debug rule for update pull request template workflow" This reverts commit 4ef0388f8845da28ef3f4e2e7f9d9c1cf5efe7dc. --- .../update-pull-request-and-issue-template.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update-pull-request-and-issue-template.yml index d050247239..4273b17b63 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -18,11 +18,10 @@ on: push: branches: - main - - refactor/ci/update-action - # paths: - # - "versions/GO_VERSION" - # - "versions/NGT_VERSION" - # - "versions/KUBECTL_VERSION" + paths: + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + - "versions/KUBECTL_VERSION" jobs: dump-contexts-to-log: @@ -85,7 +84,7 @@ jobs: --request POST \ --data "{\"title\": \"Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE are updated.\", \"maintainer_can_modify\": true}" \ $API_URL - # env: - # GITHUB_USER: ${{ secrets.DISPATCH_USER }} - # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - # API_URL: https://api.github.com/repos/vdaas/vald/pulls + env: + GITHUB_USER: ${{ secrets.DISPATCH_USER }} + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: https://api.github.com/repos/vdaas/vald/pulls From d30ddec9f3bc24ae5433fbe87da1dd4a9a1a75ae Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 11:27:55 +0900 Subject: [PATCH 091/139] set safe directory and small refactor for semver workflow Signed-off-by: hlts2 --- .github/workflows/semver.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 858e03adf1..0ea3150ab8 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -32,13 +32,14 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: crazy-max/ghaction-import-gpg@v4 with: From 6a23f425f7d0614d52a2e9cf918106d7ae7ce248 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 12:18:24 +0900 Subject: [PATCH 092/139] update configuration for reviewdog workflow Signed-off-by: hlts2 --- .github/workflows/reviewdog-hadolint.yml | 11 +++++---- .github/workflows/reviewdog-k8s.yml | 29 +++++++++++++----------- .github/workflows/reviewdog-markdown.yml | 2 +- .github/workflows/reviewdog.yml | 9 ++++---- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index ecf4f592bb..94cb0179d2 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: "reviewdog - Dockerfiles" +name: "Reviewdog - Dockerfiles" on: pull_request: paths: @@ -32,11 +32,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true - - name: hadolint + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Run hadolint uses: reviewdog/action-hadolint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index 1af5a48696..12f90a9164 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: "reviewdog - K8s manifests" +name: "Reviewdog - K8s manifests" on: pull_request: paths: @@ -35,11 +35,12 @@ jobs: image: mumoshu/conflint:latest steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true - - name: kubeval-conflint + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Run kubeval-conflint run: | set -vx conflint run -c .github/conflint.yaml -efm "%f:%l:%c: %m" || true @@ -56,11 +57,12 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true - - name: kubelinter for k8s directory + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Run kubelinter for k8s directory run: | kube-linter lint \ --config .github/kubelinter.yaml \ @@ -76,11 +78,12 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true - - name: kubelinter for generated manifests + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Run kubelinter for generated manifests run: | kube-linter lint \ --config .github/kubelinter.yaml \ diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index 4974693308..658299fc42 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: "reviewdog - Markdown" +name: "Reviewdog - Markdown" on: pull_request: paths-ignore: diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 270129bc21..ea4c4132b9 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: reviewdog +name: "Reviewdog - Go" on: pull_request: paths: @@ -34,9 +34,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Go build run: | From d32569cbd869736983a1c3684e3bd11ba4919a15 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 15:30:33 +0900 Subject: [PATCH 093/139] add action to determine docekr image tag name and fix helm workflow Signed-off-by: hlts2 --- .../determine-docker-image-tag/action.yaml | 38 ++++++++++ .github/actions/docker-build/action.yaml | 28 +------ .github/actions/setup-e2e/action.yaml | 16 ++-- .github/workflows/helm.yml | 74 ++++--------------- 4 files changed, 61 insertions(+), 95 deletions(-) create mode 100644 .github/actions/determine-docker-image-tag/action.yaml diff --git a/.github/actions/determine-docker-image-tag/action.yaml b/.github/actions/determine-docker-image-tag/action.yaml new file mode 100644 index 0000000000..64964417b5 --- /dev/null +++ b/.github/actions/determine-docker-image-tag/action.yaml @@ -0,0 +1,38 @@ +name: "Determine Docker image tag" +description: "Determine Docker image tag" + +outputs: + PRIMARY_TAG: + description: "primary tag" + value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} + +runs: + using: "composite" + steps: + - name: Determine tag name + shell: bash + id: determine_tag_name + run: | + echo "GITHUB_REF $GITHUB_REF" + echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH" + echo "GITHUB_EVENT_NAME ${{ github.event_name }}" + echo "GITHUB_EVENT_NUMBER ${{ github.event.number }}" + if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then + tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` + primary_tag="${tag_name}" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION + primary_tag="pr-${pr_num}" + elif [ "${{ github.event_name }}" = "pull_request_target" ]; then + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION + primary_tag="pr-${pr_num}" + elif [ "$GITHUB_REF" = "refs/heads/main" ]; then + echo "nightly" > versions/VALD_VERSION + primary_tag="nightly" + else + primary_tag="unknown" + fi + echo "PRIMARY_TAG is determined: ${primary_tag}" + echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 6d7b57f82d..ff189e509c 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -63,33 +63,9 @@ runs: env: TARGET: ${{ inputs.target }} - - name: Determine tag name - shell: bash + - name: Determine Docker image tag id: determine_tag_name - run: | - echo "GITHUB_REF $GITHUB_REF" - echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH" - echo "GITHUB_EVENT_NAME ${{ github.event_name }}" - echo "GITHUB_EVENT_NUMBER ${{ github.event.number }}" - if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then - tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` - primary_tag="${tag_name}" - elif [ "${{ github.event_name }}" = "pull_request" ]; then - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR-${pr_num}" > versions/VALD_VERSION - primary_tag="pr-${pr_num}" - elif [ "${{ github.event_name }}" = "pull_request_target" ]; then - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR-${pr_num}" > versions/VALD_VERSION - primary_tag="pr-${pr_num}" - elif [ "$GITHUB_REF" = "refs/heads/main" ]; then - echo "nightly" > versions/VALD_VERSION - primary_tag="nightly" - else - primary_tag="unknown" - fi - echo "PRIMARY_TAG is determined: ${primary_tag}" - echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT + uses: ./.github/actions/determine-docker-image-tag - name: Determine platforms shell: bash diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index f2f7a00438..1613d98b40 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -38,27 +38,21 @@ runs: - name: Setup Helm environment uses: ./.github/actions/setup-helm - - name: Get PR number - # TODO: add if statement - id: get_pr_number - shell: bash - run: | - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT - - name: Wait for target Docker images if: startsWith( github.ref, 'refs/tags/') uses: ./.github/actions/wait-for-docker-image with: images: ${{ inputs.target_images }} + - name: Determine Docker image tag + id: determine_tag_name + uses: ./.github/actions/determine-docker-image-tag + - name: Specify container versions id: specify_container_versions uses: ./.github/actions/detect-docker-image-tags with: - # TODO: Change it later - # tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} - tag_name: nightly + tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} images: ${{ inputs.target_images }} - uses: ./.github/actions/setup-k3d diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 93df1a9e1c..d8394a45fb 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -35,12 +35,11 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: set git config + - name: Set Git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} @@ -50,11 +49,10 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Helm version - run: | - helm version + - name: Setup Helm environment + uses: ./.github/actions/setup-helm - - name: switch new branch + - name: Switch new branch id: switch_to_new_branch run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) @@ -101,13 +99,12 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - name: Check out code. - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - name: set git config + - name: Set Git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} @@ -117,9 +114,8 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Helm version - run: | - helm version + - name: Setup Helm environment + uses: ./.github/actions/setup-helm - name: Packaging run: | @@ -149,66 +145,28 @@ jobs: steps: - uses: actions/checkout@v3 - - name: set git config + - name: Set Git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Wait for Docker images - id: wait_for_docker_images - uses: ./.github/actions/wait-for-docker-image - - - name: Specify container versions - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e with: - tag_name: ${{ github.ref_name }} - - - name: Fetch Helm version - run: | - HELM_VERSION=`make version/helm` - echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT - id: version - - - uses: ./.github/actions/setup-k3d - with: - agents: 3 - - - name: check k3d - run: | - kubectl cluster-info - - - uses: azure/setup-helm@v3 - with: - version: ${{ steps.version.outputs.helm }} - - - name: Helm version - run: | - helm version + require_libhdf5: "true" - - name: deploy Vald + - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway use_local_charts: false - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} - - name: Run E2E CRUD run: | make hack/benchmark/assets/dataset/${DATASET} - go version make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${DATASET} \ E2E_INSERT_COUNT=60000 \ From d7967f3ef1846881552feb21adf40d2c8fd2ac4e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 15:42:41 +0900 Subject: [PATCH 094/139] deleted unnecessary commentout Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 15 ++++----------- .github/workflows/e2e-deploy.yml | 19 +++++-------------- .github/workflows/e2e-max-dim.yml | 3 +-- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 2c3d803a5c..c69dddf634 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -16,9 +16,6 @@ name: "Run E2E chaos test" on: push: - # TODO: delete it later - branches: - - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" @@ -39,8 +36,7 @@ jobs: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout it later - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 with: @@ -86,8 +82,7 @@ jobs: name: "E2E chaos test (random Pod failure: to test redundancy)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout it later - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 with: @@ -134,8 +129,7 @@ jobs: name: "E2E chaos test (agent network partition: to test retries)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout it later - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 with: @@ -181,8 +175,7 @@ jobs: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout it later - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 83cfd56757..e101adcd06 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -16,10 +16,6 @@ name: "Run E2E deploy and integration test" on: push: - # TODO: delete it later - branches: - - "refactor/ci/update-action" - tags: - "*.*.*" - "v*.*.*" @@ -39,8 +35,7 @@ jobs: name: "E2E test (Stream CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 with: @@ -85,8 +80,7 @@ jobs: name: "E2E test (Stream CRUD) for operator" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout later - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 with: @@ -144,8 +138,7 @@ jobs: name: "E2E test (Stream CRUD: skip strict exist check)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 with: @@ -191,8 +184,7 @@ jobs: name: "E2E test (Multi-APIs CRUD)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 with: @@ -234,8 +226,7 @@ jobs: name: "E2E Agent & Sidecar test" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index dcb3028e7d..8d6710c0ed 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -36,8 +36,7 @@ jobs: name: "E2E test (Max Dimension Insert: skip strict exist check)" runs-on: ubuntu-latest timeout-minutes: 60 - # TODO: delete commentout - # if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' steps: - uses: actions/checkout@v3 with: From a1e1876b80652bd8273c93c1993e03ad48e91a25 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 3 Aug 2023 16:31:41 +0900 Subject: [PATCH 095/139] add safe directory settings to e2e and formater workflow Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 28 ++++++++++------- .github/workflows/e2e-code-bench-agent.yaml | 14 +++++---- .github/workflows/e2e-deploy.yml | 35 ++++++++++++--------- .github/workflows/e2e-max-dim.yml | 7 +++-- .github/workflows/e2e-profiling.yml | 7 +++-- .github/workflows/format.yml | 6 ++-- .github/workflows/labeler.yml | 6 ++-- .github/workflows/reviewdog-markdown.yml | 24 +++++++------- 8 files changed, 73 insertions(+), 54 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index c69dddf634..31b9fe074c 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -39,9 +39,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -85,9 +86,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -132,9 +134,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -178,9 +181,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index 90f68cba54..8bcbe8c205 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -56,9 +56,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: ./.github/actions/setup-go @@ -89,9 +90,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: ./.github/actions/setup-go diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index e101adcd06..77bced5dd1 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -38,9 +38,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -83,9 +84,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -141,9 +143,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -187,9 +190,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e @@ -229,9 +233,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 8d6710c0ed..d8a9d9e973 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -39,9 +39,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index bbdd49ce2b..5cc6187a87 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -39,9 +39,10 @@ jobs: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup E2E environment id: setup_e2e diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 682cd28c7b..66a6dda040 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -35,10 +35,12 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - path: ${{ github.workspace }} - set-safe-directory: true token: ${{ secrets.DISPATCH_TOKEN }} + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index df3a1f75f1..616e7284e2 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -37,8 +37,10 @@ jobs: with: ref: ${{ github.head_ref }} fetch-depth: 0 - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Add labels run: | diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index 658299fc42..4f29c5b419 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -32,11 +32,11 @@ jobs: name: runner / LanguageTool / apis & charts runs-on: ubuntu-latest steps: - - name: Check out code. - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + - uses: actions/checkout@v3 + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: LanguageTool uses: reviewdog/action-languagetool@v1 @@ -54,9 +54,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: LanguageTool uses: reviewdog/action-languagetool@v1 @@ -74,9 +75,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - with: - path: ${{ github.workspace }} - set-safe-directory: true + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Setup node/npm uses: actions/setup-node@v3 From 1e34b01128c6542e27d8ccfba3b022f6b8874d50 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 12:40:29 +0900 Subject: [PATCH 096/139] bugfix e2e agent test error Signed-off-by: hlts2 --- tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go index e4c5e7d832..eda3bf6570 100644 --- a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go +++ b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go @@ -66,9 +66,9 @@ func registerVector(ctx context.Context, n service.NGT) error { for i := int64(0); i < maxIDNum; i++ { uuid := strconv.FormatInt(i, 10) - _, err := n.GetObject(uuid) - if err != nil { - return err + vec, _, err := n.GetObject(uuid) + if err != nil || len(vec) == 0 { + return errors.ErrObjectNotFound(err, uuid) } } return nil @@ -150,9 +150,9 @@ func Test_ngt_parallel_delete_and_insert(t *testing.T) { for i := int64(0); i < maxIDNum; i++ { uuid := strconv.FormatInt(i, 10) - _, err := n.GetObject(uuid) - if err != nil { - t.Error(err) + vec, _, err := n.GetObject(uuid) + if err != nil || len(vec) == 0 { + t.Error(errors.ErrObjectNotFound(err, uuid)) } err = n.Insert(uuid, []float32{1, 2}) if err == nil { From 1c4fcc78aac7d1ae9f13d92440bc5346cd5aabcb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 15:53:44 +0900 Subject: [PATCH 097/139] use local setup go action Signed-off-by: hlts2 --- .github/workflows/chatops.yml | 24 +++-------------- .github/workflows/helm-lint.yml | 46 +++++---------------------------- 2 files changed, 10 insertions(+), 60 deletions(-) diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index b27c2b01d1..762acde0fb 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -150,17 +150,8 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Fetch golang version + - uses: ./.github/actions/setup-go if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - - uses: actions/setup-go@v3 - if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - with: - go-version: ${{ steps.golang_version.outputs.version }} - name: Generate tests and push id: gen_test @@ -284,17 +275,8 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Fetch golang version - if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version - - - uses: actions/setup-go@v3 - if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - with: - go-version: ${{ steps.golang_version.outputs.version }} + - uses: ./.github/actions/setup-go + if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - name: update and push id: format_push diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index ce387821b4..24cc346f57 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -39,19 +39,9 @@ jobs: run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Helm version - run: | - helm version - - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version + - uses: ./.github/actions/setup-go - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} + - uses: ./.github/actions/setup-helm - name: Generate schema for charts/vald run: | @@ -73,19 +63,9 @@ jobs: run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Helm version - run: | - helm version - - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version + - uses: ./.github/actions/setup-go - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} + - uses: ./.github/actions/setup-helm - name: Generate schema for charts/vald-helm-operator run: | @@ -105,23 +85,11 @@ jobs: run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Helm version - run: | - helm version + - uses: ./.github/actions/setup-yq - - name: install YQ - run: | - sudo make yq/install - - - name: Fetch golang version - run: | - GO_VERSION=`make version/go` - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - id: golang_version + - uses: ./.github/actions/setup-go - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.golang_version.outputs.version }} + - uses: ./.github/actions/setup-helm - name: Generate ValdRelease schema run: | From 072d0abaf73af6093b4d8b60d912b0549d8793ac Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 15:58:40 +0900 Subject: [PATCH 098/139] deleted unnecessary step Signed-off-by: hlts2 --- .github/workflows/helm-lint.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 24cc346f57..32ecdfd685 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -91,15 +91,11 @@ jobs: - uses: ./.github/actions/setup-helm - - name: Generate ValdRelease schema - run: | - make helm/schema/crd/vald - - uses: ./.github/actions/setup-k3d - - name: check k3d + - name: Generate ValdRelease schema run: | - kubectl cluster-info + make helm/schema/crd/vald - name: Test to apply run: | From 535a011b05b951b9b1716e163763366c3914643f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:14:38 +0900 Subject: [PATCH 099/139] add debug rule for helm lint workflow Signed-off-by: hlts2 --- .github/workflows/helm-lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 32ecdfd685..4f6964bfa5 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -15,6 +15,9 @@ # name: "Run Helm lint" on: + push: + branches: + - "refactor/ci/update-action" pull_request: paths: - "charts/**" From 4e099f6e931521e3801ad0a154bb26f14c4d4c75 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:15:03 +0900 Subject: [PATCH 100/139] Revert "add debug rule for helm lint workflow" This reverts commit 7fd16dae958ad006ea31662253d73bd297201ed1. --- .github/workflows/helm-lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 4f6964bfa5..32ecdfd685 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -15,9 +15,6 @@ # name: "Run Helm lint" on: - push: - branches: - - "refactor/ci/update-action" pull_request: paths: - "charts/**" From 30e2062cef974222d50c72e9aa2a46df7e7c59d3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:16:06 +0900 Subject: [PATCH 101/139] add new line between steps Signed-off-by: hlts2 --- .github/workflows/helm-lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 32ecdfd685..fd4bf9e2bd 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -46,6 +46,7 @@ jobs: - name: Generate schema for charts/vald run: | make helm/schema/vald + - name: Run lint for charts/vald run: | helm lint charts/vald @@ -70,6 +71,7 @@ jobs: - name: Generate schema for charts/vald-helm-operator run: | make helm/schema/vald-helm-operator + - name: Run lint for charts/vald-helm-operator run: | helm lint charts/vald-helm-operator From 96a85f7ee535d1a47da60e85c743a264c2e2d6ef Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:26:01 +0900 Subject: [PATCH 102/139] add debug rule for helm workflow Signed-off-by: hlts2 --- .github/workflows/helm.yml | 74 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index d8394a45fb..3eafe4efd1 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -16,6 +16,8 @@ name: "Update Helm charts" on: push: + branches: + - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" @@ -71,27 +73,27 @@ jobs: make format/yaml make license - - name: Push to main - continue-on-error: true - run: | - git add k8s - git commit -S --signoff -m ":robot: Automatically update k8s manifests" - - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push -u origin ${BRANCH_NAME} - - curl --include --verbose --fail \ - -H "Accept: application/json" \ - -H "Content-Type:application/json" \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - --request POST \ - --data "{\"title\": \"Update K8s manifests\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"K8s manifests are updated.\", \"maintainer_can_modify\": true}" \ - $API_URL - env: - GITHUB_USER: ${{ secrets.DISPATCH_USER }} - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - API_URL: https://api.github.com/repos/vdaas/vald/pulls - BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} + # - name: Push to main + # continue-on-error: true + # run: | + # git add k8s + # git commit -S --signoff -m ":robot: Automatically update k8s manifests" + # + # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + # git push -u origin ${BRANCH_NAME} + # + # curl --include --verbose --fail \ + # -H "Accept: application/json" \ + # -H "Content-Type:application/json" \ + # -H "Authorization: token ${GITHUB_TOKEN}" \ + # --request POST \ + # --data "{\"title\": \"Update K8s manifests\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"K8s manifests are updated.\", \"maintainer_can_modify\": true}" \ + # $API_URL + # env: + # GITHUB_USER: ${{ secrets.DISPATCH_USER }} + # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + # API_URL: https://api.github.com/repos/vdaas/vald/pulls + # BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} update-helm-chart: name: Update Helm chart @@ -122,21 +124,21 @@ jobs: make helm/package/vald make helm/package/vald-helm-operator - - name: Push to gh-pages - run: | - git checkout . - git checkout gh-pages - mv -f vald-*.tgz charts/ - helm repo index --url https://vald.vdaas.org/charts charts/ - - git add charts - git commit -S --signoff -m ":robot: Automatically update Helm charts" - - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin gh-pages - env: - GITHUB_USER: ${{ secrets.DISPATCH_USER }} - GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + # - name: Push to gh-pages + # run: | + # git checkout . + # git checkout gh-pages + # mv -f vald-*.tgz charts/ + # helm repo index --url https://vald.vdaas.org/charts charts/ + # + # git add charts + # git commit -S --signoff -m ":robot: Automatically update Helm charts" + # + # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + # git push origin gh-pages + # env: + # GITHUB_USER: ${{ secrets.DISPATCH_USER }} + # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} crud-on-remote-helm-chart: name: CRUD test on remote Helm chart From 197f937adce5072929ed62ff4480fddc44613a8c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:57:21 +0900 Subject: [PATCH 103/139] fix workflow execution rule for debug Signed-off-by: hlts2 --- .github/workflows/helm.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 3eafe4efd1..5750fdf01a 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -15,9 +15,10 @@ # name: "Update Helm charts" on: + pull_request: + paths: + - ".github/workflows/helm.yml" push: - branches: - - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" From 6a85cefb01d89841c2ad9e5bac99cc87425c13ae Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:57:45 +0900 Subject: [PATCH 104/139] Revert "fix workflow execution rule for debug" This reverts commit 8cc7e3059db9c70176acc25a62fe79b7d094e552. --- .github/workflows/helm.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 5750fdf01a..3eafe4efd1 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -15,10 +15,9 @@ # name: "Update Helm charts" on: - pull_request: - paths: - - ".github/workflows/helm.yml" push: + branches: + - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" From 6a937dd1f74f971f3b7ead5a2c3fd5364f32032e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 16:58:19 +0900 Subject: [PATCH 105/139] Revert "add debug rule for helm workflow" This reverts commit 77a2c7e1c5c64747fe8ca79bb679910e842ce1bc. --- .github/workflows/helm.yml | 74 +++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 3eafe4efd1..d8394a45fb 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -16,8 +16,6 @@ name: "Update Helm charts" on: push: - branches: - - "refactor/ci/update-action" tags: - "*.*.*" - "v*.*.*" @@ -73,27 +71,27 @@ jobs: make format/yaml make license - # - name: Push to main - # continue-on-error: true - # run: | - # git add k8s - # git commit -S --signoff -m ":robot: Automatically update k8s manifests" - # - # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - # git push -u origin ${BRANCH_NAME} - # - # curl --include --verbose --fail \ - # -H "Accept: application/json" \ - # -H "Content-Type:application/json" \ - # -H "Authorization: token ${GITHUB_TOKEN}" \ - # --request POST \ - # --data "{\"title\": \"Update K8s manifests\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"K8s manifests are updated.\", \"maintainer_can_modify\": true}" \ - # $API_URL - # env: - # GITHUB_USER: ${{ secrets.DISPATCH_USER }} - # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - # API_URL: https://api.github.com/repos/vdaas/vald/pulls - # BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} + - name: Push to main + continue-on-error: true + run: | + git add k8s + git commit -S --signoff -m ":robot: Automatically update k8s manifests" + + git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push -u origin ${BRANCH_NAME} + + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request POST \ + --data "{\"title\": \"Update K8s manifests\", \"head\": \"${BRANCH_NAME}\", \"base\": \"main\", \"body\": \"K8s manifests are updated.\", \"maintainer_can_modify\": true}" \ + $API_URL + env: + GITHUB_USER: ${{ secrets.DISPATCH_USER }} + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: https://api.github.com/repos/vdaas/vald/pulls + BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} update-helm-chart: name: Update Helm chart @@ -124,21 +122,21 @@ jobs: make helm/package/vald make helm/package/vald-helm-operator - # - name: Push to gh-pages - # run: | - # git checkout . - # git checkout gh-pages - # mv -f vald-*.tgz charts/ - # helm repo index --url https://vald.vdaas.org/charts charts/ - # - # git add charts - # git commit -S --signoff -m ":robot: Automatically update Helm charts" - # - # git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - # git push origin gh-pages - # env: - # GITHUB_USER: ${{ secrets.DISPATCH_USER }} - # GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: Push to gh-pages + run: | + git checkout . + git checkout gh-pages + mv -f vald-*.tgz charts/ + helm repo index --url https://vald.vdaas.org/charts charts/ + + git add charts + git commit -S --signoff -m ":robot: Automatically update Helm charts" + + git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push origin gh-pages + env: + GITHUB_USER: ${{ secrets.DISPATCH_USER }} + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} crud-on-remote-helm-chart: name: CRUD test on remote Helm chart From 819c1cdcf3923cbe659c7e0af0bc264b7dbcc8f3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 17:03:36 +0900 Subject: [PATCH 106/139] update step name for fossa workflow Signed-off-by: hlts2 --- .github/workflows/fossa.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index c75e68ae64..48aa6650ad 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -37,7 +37,8 @@ jobs: if: github.ref == 'refs/heads/main' || github.event.action == 'labeled' && github.event.label.name == 'actions/fossa' steps: - uses: actions/checkout@v3 - - name: set git config + + - name: Set Git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} From c89b3a06fdbc04eb0ea8a1d5814ffd6408a848c8 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 19:04:33 +0900 Subject: [PATCH 107/139] add workflow execution rule Signed-off-by: hlts2 --- .github/workflows/dockers-loadtest-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index 7c29f5c6b8..21a8df8ba5 100644 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" @@ -40,6 +41,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" @@ -55,6 +57,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" From 41986f11791597e7c1d062508b364f7753bd0b7d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 19:06:22 +0900 Subject: [PATCH 108/139] enable e2e workflow Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 148 +++++++++++++++---------------- .github/workflows/e2e-deploy.yml | 114 ++++++++++++------------ 2 files changed, 131 insertions(+), 131 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 31b9fe074c..a8dfee5e60 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -63,21 +63,21 @@ jobs: with: helm_extra_options: "--set podChaos.failure.enabled=true" - # - name: Run Insert and Search operations - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_TIMEOUT=15m \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/insert/search - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run Insert and Search operations + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/insert/search + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" @@ -111,21 +111,21 @@ jobs: with: helm_extra_options: "--set podChaos.kill.enabled=true" - # - name: Run Insert and Search operations - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_TIMEOUT=15m \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/insert/search - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run Insert and Search operations + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/insert/search + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" @@ -158,21 +158,21 @@ jobs: with: helm_extra_options: "--set networkChaos.partition.enabled=true" - # - name: Run Insert and Search operations - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_TIMEOUT=15m \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/insert/search - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run Insert and Search operations + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/insert/search + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" @@ -206,35 +206,35 @@ jobs: with: helm_extra_options: "--set networkChaos.bandwidth.enabled=true" - # - name: Run Insert operations - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_TIMEOUT=15m \ - # E2E_INSERT_COUNT=10000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/insert - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # - # - name: Run Search operations - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_TIMEOUT=15m \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/search - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run Insert operations + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_INSERT_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/insert + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + - name: Run Search operations + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/search + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} slack-notification: name: "Slack notification" diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 77bced5dd1..7f85ea40f0 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -57,25 +57,25 @@ jobs: values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_INSERT_COUNT=60000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=60000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-for-operator: name: "E2E test (Stream CRUD) for operator" @@ -116,25 +116,25 @@ jobs: valdrelease: ./.github/valdrelease/valdrelease.yaml wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_INSERT_COUNT=60000 \ - # E2E_SEARCH_COUNT=10000 \ - # E2E_SEARCH_BY_ID_COUNT=10000 \ - # E2E_GET_OBJECT_COUNT=100 \ - # E2E_UPDATE_COUNT=100 \ - # E2E_UPSERT_COUNT=100 \ - # E2E_REMOVE_COUNT=100 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=60000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-skip-exist-check: name: "E2E test (Stream CRUD: skip strict exist check)" @@ -163,25 +163,25 @@ jobs: values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_INSERT_COUNT=10 \ - # E2E_SEARCH_COUNT=10 \ - # E2E_SEARCH_BY_ID_COUNT=10 \ - # E2E_GET_OBJECT_COUNT=10 \ - # E2E_UPDATE_COUNT=10 \ - # E2E_UPSERT_COUNT=10 \ - # E2E_REMOVE_COUNT=10 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/skip - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_GET_OBJECT_COUNT=10 \ + E2E_UPDATE_COUNT=10 \ + E2E_UPSERT_COUNT=10 \ + E2E_REMOVE_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/skip + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-multiapis-crud: name: "E2E test (Multi-APIs CRUD)" From 16d035d7bd47b138c242ddbcdf5c24c1571432b5 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 19:34:41 +0900 Subject: [PATCH 109/139] fix job execution rule for e2e Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 11 ++++++----- .github/workflows/e2e-deploy.yml | 13 +++++++------ .github/workflows/e2e-max-dim.yml | 7 ++++--- .github/workflows/e2e-profiling.yml | 3 ++- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index a8dfee5e60..5ce93c9529 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -27,6 +27,7 @@ on: jobs: dump-contexts-to-log: + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -34,9 +35,9 @@ jobs: agent-failure: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 @@ -81,9 +82,9 @@ jobs: random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 @@ -129,9 +130,9 @@ jobs: agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 @@ -176,9 +177,9 @@ jobs: clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 @@ -238,13 +239,13 @@ jobs: slack-notification: name: "Slack notification" + if: startsWith( github.ref, 'refs/tags/') runs-on: ubuntu-latest needs: - agent-failure - random-pod-failure - agent-network-partition - clusterwide-network-bandwidth - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 7f85ea40f0..d2c5a8bb09 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -26,6 +26,7 @@ on: - "labeled" jobs: dump-contexts-to-log: + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -33,9 +34,9 @@ jobs: e2e-stream-crud: name: "E2E test (Stream CRUD)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 @@ -79,9 +80,9 @@ jobs: e2e-stream-crud-for-operator: name: "E2E test (Stream CRUD) for operator" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 @@ -138,9 +139,9 @@ jobs: e2e-stream-crud-skip-exist-check: name: "E2E test (Stream CRUD: skip strict exist check)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 @@ -185,9 +186,9 @@ jobs: e2e-multiapis-crud: name: "E2E test (Multi-APIs CRUD)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 @@ -228,9 +229,9 @@ jobs: e2e-agent-and-sidecar: name: "E2E Agent & Sidecar test" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v3 @@ -272,6 +273,7 @@ jobs: slack-notification: name: "Slack notification" + if: startsWith( github.ref, 'refs/tags/') needs: - e2e-agent-and-sidecar - e2e-multiapis-crud @@ -279,7 +281,6 @@ jobs: - e2e-stream-crud-for-operator - e2e-stream-crud-skip-exist-check runs-on: ubuntu-latest - if: startsWith( github.ref, 'refs/tags/') steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index d8a9d9e973..3b7ce1261b 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -27,6 +27,7 @@ on: jobs: dump-contexts-to-log: + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -34,9 +35,10 @@ jobs: e2e-max-dimension-insert: name: "E2E test (Max Dimension Insert: skip strict exist check)" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' + steps: - uses: actions/checkout@v3 @@ -112,8 +114,7 @@ jobs: slack-notification: name: "Slack notification" - needs: - - e2e-max-dimension-insert + needs: [e2e-max-dimension-insert] runs-on: ubuntu-latest if: startsWith( github.ref, 'refs/tags/') steps: diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index 5cc6187a87..f9f49f4544 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -27,6 +27,7 @@ on: jobs: dump-contexts-to-log: + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -34,9 +35,9 @@ jobs: e2e-profiling: name: "E2E profiling" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' steps: - uses: actions/checkout@v3 From 01b617d0abdc2f8835c5df16ad7901580444eec4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Fri, 4 Aug 2023 19:35:48 +0900 Subject: [PATCH 110/139] enable e2e deploy step Signed-off-by: hlts2 --- .github/workflows/e2e-deploy.yml | 60 ++++++++++++++++---------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index d2c5a8bb09..66796d6c3f 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -210,22 +210,22 @@ jobs: values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - # - name: Run E2E CRUD - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # go version - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_INSERT_COUNT=100 \ - # E2E_SEARCH_COUNT=10 \ - # E2E_SEARCH_BY_ID_COUNT=10 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/multi - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=100 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/multi + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-agent-and-sidecar: name: "E2E Agent & Sidecar test" @@ -256,20 +256,20 @@ jobs: wait_for_selector: app=vald-agent-ngt wait_for_timeout: 29m - # - name: Run E2E Agent & Sidecar - # run: | - # make hack/benchmark/assets/dataset/${DATASET} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${DATASET} \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=4000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/sidecar - # env: - # DATASET: fashion-mnist-784-euclidean.hdf5 - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: Run E2E Agent & Sidecar + run: | + make hack/benchmark/assets/dataset/${DATASET} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=4000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/sidecar + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} slack-notification: name: "Slack notification" From 5fcc6f74fc6bd956e8ecfcd794d817e769a85191 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 10:47:50 +0900 Subject: [PATCH 111/139] add new rob execution rule for fossa Signed-off-by: hlts2 --- .github/workflows/fossa.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 48aa6650ad..7f2a7f227a 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -24,6 +24,7 @@ on: jobs: dump-contexts-to-log: + if: github.ref == 'refs/heads/main' || github.event.action == 'labeled' && github.event.label.name == 'actions/fossa' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -31,10 +32,10 @@ jobs: fossa-scan: name: "trigger FOSSA scan" + needs: [dump-contexts-to-log] runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly - if: github.ref == 'refs/heads/main' || github.event.action == 'labeled' && github.event.label.name == 'actions/fossa' steps: - uses: actions/checkout@v3 From a4bb67b2cc929d01a3a1011c8318ce017967a3fa Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 11:34:38 +0900 Subject: [PATCH 112/139] deleted unnecessary actions parameter Signed-off-by: hlts2 --- .github/actions/e2e-deploy-vald/action.yaml | 4 ---- .github/workflows/e2e-chaos.yaml | 2 -- .github/workflows/e2e-deploy.yml | 2 -- .github/workflows/e2e-profiling.yml | 1 - 4 files changed, 9 deletions(-) diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 1950213dbf..d39cb7d9b5 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -17,10 +17,6 @@ name: "Setting E2E test environment" description: "A workflow to set E2E test environment" inputs: - require_libhdf5: - description: "If libhdf5 is required, set this to true." - required: false - default: "false" require_minio: description: "If Minio is required, set this to true." required: false diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index 5ce93c9529..fb74835f6b 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -102,7 +102,6 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway @@ -197,7 +196,6 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 66796d6c3f..b9f547c5c7 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -159,7 +159,6 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway @@ -249,7 +248,6 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" require_minio: "true" helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-agent-sidecar.yaml diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index f9f49f4544..a4769428fe 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -55,7 +55,6 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - require_libhdf5: "true" helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-profile.yaml wait_for_selector: app=vald-lb-gateway From 622636391d9c41bf76d6957da4a8bb28022891ed Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 11:36:45 +0900 Subject: [PATCH 113/139] deleted unnecessary actions parameter of helm operator deploy Signed-off-by: hlts2 --- .github/actions/e2e-deploy-vald-helm-operator/action.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index c9e6f00b1c..fb84bab934 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -17,10 +17,6 @@ name: "Setting E2E test environment" description: "A workflow to set E2E test environment" inputs: - require_libhdf5: - description: "If libhdf5 is required, set this to true." - required: false - default: "false" require_minio: description: "If Minio is required, set this to true." required: false From 69213a4bece514cb9003276196386e74fb60537f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 11:48:02 +0900 Subject: [PATCH 114/139] deleted unnecessary log Signed-off-by: hlts2 --- .github/workflows/e2e-deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index b9f547c5c7..eb5cafe90c 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -212,7 +212,6 @@ jobs: - name: Run E2E CRUD run: | make hack/benchmark/assets/dataset/${DATASET} - go version make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${DATASET} \ E2E_INSERT_COUNT=100 \ From f3eec26c2e5824a0cdfb4c945deabef207c4a3e8 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 14:47:39 +0900 Subject: [PATCH 115/139] output values or valdrelease.yaml Signed-off-by: hlts2 --- .github/actions/e2e-deploy-vald-helm-operator/action.yaml | 5 +++++ .github/actions/e2e-deploy-vald/action.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index fb84bab934..6034cea9f4 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -52,6 +52,11 @@ runs: run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy + - name: Output valdrelease + shell: bash + run: | + cat ${{ inputs.valdrelease }} + - name: Deploy vald helm operator from remote charts shell: bash id: deploy_vald_helm_operator_remote diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index d39cb7d9b5..b957560c4f 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -56,6 +56,11 @@ runs: run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy + - name: Output values + shell: bash + run: | + cat ${{ inputs.values }} + - name: Deploy vald from remote charts shell: bash id: deploy_vald_remote From 54a3db33fe0a0c37df9af57a5ad236af7767a810 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 7 Aug 2023 16:16:31 +0900 Subject: [PATCH 116/139] update insert vector count of e2e deploy workflow Signed-off-by: hlts2 --- .github/workflows/e2e-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index eb5cafe90c..f92f193ac3 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -63,7 +63,7 @@ jobs: make hack/benchmark/assets/dataset/${DATASET} make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=60000 \ + E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=10000 \ E2E_SEARCH_BY_ID_COUNT=10000 \ E2E_GET_OBJECT_COUNT=100 \ @@ -122,7 +122,7 @@ jobs: make hack/benchmark/assets/dataset/${DATASET} make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=60000 \ + E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=10000 \ E2E_SEARCH_BY_ID_COUNT=10000 \ E2E_GET_OBJECT_COUNT=100 \ From fc8befdf66917a0e5343a3be58d6cf44cfe143a6 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 01:22:31 +0900 Subject: [PATCH 117/139] change default value to install libhdf5 Signed-off-by: hlts2 --- .github/actions/setup-e2e/action.yaml | 4 ++-- .github/workflows/e2e-chaos.yaml | 8 -------- .github/workflows/e2e-deploy.yml | 10 ---------- .github/workflows/e2e-max-dim.yml | 2 -- .github/workflows/e2e-profiling.yml | 2 -- .github/workflows/helm.yml | 2 -- 6 files changed, 2 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 1613d98b40..cf6d846ae2 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -3,9 +3,9 @@ description: 'Setup the environment to run E2E test' inputs: require_libhdf5: - description: "If libhdf5 is required, set this to true." + description: "If libhdf5 is not required, set this to false" required: false - default: "false" + default: "true" target_images: description: "image names" required: false diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index fb74835f6b..d66614b4b0 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -48,8 +48,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -95,8 +93,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -142,8 +138,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -189,8 +183,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index f92f193ac3..0b5db94845 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -47,8 +47,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -93,8 +91,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Merge Docker image tag run: | @@ -152,8 +148,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -198,8 +192,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald @@ -240,8 +232,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index 3b7ce1261b..de8f5f8c61 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -49,8 +49,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Run E2E MaxDimension run: | diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index a4769428fe..43c9de812c 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -48,8 +48,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index d8394a45fb..0698e52b74 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -152,8 +152,6 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e - with: - require_libhdf5: "true" - name: Deploy Vald id: deploy_vald From ab778e02847394f9faab469a5199b53d0747c5ac Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 10:44:13 +0900 Subject: [PATCH 118/139] bugfix operator deploy using specified image tag Signed-off-by: hlts2 --- .../actions/detect-docker-image-tags/action.yaml | 1 + .../e2e-deploy-vald-helm-operator/action.yaml | 13 +++++++++++-- .github/actions/setup-e2e/action.yaml | 15 +++++++++++++++ .github/workflows/e2e-deploy.yml | 11 +++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index 4e56a56e5d..df6e365d50 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -49,6 +49,7 @@ runs: ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" ["vdaas/vald-manager-index"]="manager.index.image.tag" + ["vdaas/vald-helm-operator"]="image.tag" ) for image in ${IMAGES} diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index 6034cea9f4..617a6d3bbe 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -21,8 +21,12 @@ inputs: description: "If Minio is required, set this to true." required: false default: "false" + helm_extra_options: + description: "extra options that passed to Helm command to deploy vald helm operator" + required: false + default: "" valdrelease: - description: "path to the valdrelease.yaml that apply to cluster." + description: "path to the valdrelease.yaml that apply to cluster" required: true default: "true" wait_for_selector: @@ -64,17 +68,22 @@ runs: run: | helm install vald-helm-operator \ --set image.tag=nightly \ + ${HELM_EXTRA_OPTIONS} \ charts/vald-helm-operator/. sleep 3 + env: + HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} - name: Deploy vald helm operator from local charts shell: bash id: deploy_vald_helm_operator_local if: ${{ inputs.use_local_charts == 'true' }} run: | - make k8s/vald-helm-operator/deploy + make k8s/vald-helm-operator/deploy VERSION=nightly HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS} sleep 3 + env: + HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} - name: Deploy vald shell: bash diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index cf6d846ae2..af50ebaa3c 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -6,6 +6,18 @@ inputs: description: "If libhdf5 is not required, set this to false" required: false default: "true" + require_go: + description: "If go is not required, set this to false" + required: false + default: "true" + require_helm: + description: "If helm is not required, set this to false" + required: false + default: "true" + require_k3d: + description: "If k3d is not required, set this to false" + required: false + default: "true" target_images: description: "image names" required: false @@ -33,9 +45,11 @@ runs: sudo apt-get install -y libhdf5-dev - name: Setup Go environment + if: ${{ inputs.require_go == 'true' }} uses: ./.github/actions/setup-go - name: Setup Helm environment + if: ${{ inputs.require_helm == 'true' }} uses: ./.github/actions/setup-helm - name: Wait for target Docker images @@ -56,6 +70,7 @@ runs: images: ${{ inputs.target_images }} - uses: ./.github/actions/setup-k3d + if: ${{ inputs.require_k3d == 'true' }} with: agents: 3 options: "--image docker.io/rancher/k3s:latest" diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 0b5db94845..d14f10584e 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -92,6 +92,16 @@ jobs: id: setup_e2e uses: ./.github/actions/setup-e2e + - name: Create Helm options for vald-helm-operator + uses: ./.github/actions/setup-e2e + id: vald_helm_operator + with: + target_images: "vdaas/vald-helm-operator" + require_libhdf5: false + require_go: false + require_helm: false + require_k3d: false + - name: Merge Docker image tag run: | IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) @@ -110,6 +120,7 @@ jobs: id: deploy_vald uses: ./.github/actions/e2e-deploy-vald-helm-operator with: + helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} valdrelease: ./.github/valdrelease/valdrelease.yaml wait_for_selector: app=vald-lb-gateway From 2992fb8fe3165122c689ef49b5176e40510ccd8b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 10:53:00 +0900 Subject: [PATCH 119/139] add HELM_EXTRA_OPTIONS for helm operator deploy Signed-off-by: hlts2 --- Makefile.d/k8s.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 18b29ed43a..515137f54a 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -109,6 +109,7 @@ k8s/vald-helm-operator/deploy: helm template \ --output-dir $(TEMP_DIR) \ --set image.tag=$(VERSION) \ + $(HELM_EXTRA_OPTIONS) \ --include-crds \ charts/vald-helm-operator kubectl create -f $(TEMP_DIR)/vald-helm-operator/crds/valdrelease.yaml From e12ef1bfb5b40237016316ec87a7fbcf3bea76a5 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 11:11:57 +0900 Subject: [PATCH 120/139] fix step name Signed-off-by: hlts2 --- .github/workflows/e2e-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index d14f10584e..2bce627399 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -116,7 +116,7 @@ jobs: yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml done - - name: Deploy Vald + - name: Deploy Vald using Vald Helm Operator id: deploy_vald uses: ./.github/actions/e2e-deploy-vald-helm-operator with: From 4487dadb31e9faa0b6c247f02b24e89c9463cf18 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 12:05:03 +0900 Subject: [PATCH 121/139] bugfix passing of helm option Signed-off-by: hlts2 --- .github/actions/e2e-deploy-vald-helm-operator/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index 617a6d3bbe..025cde14b5 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -80,7 +80,7 @@ runs: id: deploy_vald_helm_operator_local if: ${{ inputs.use_local_charts == 'true' }} run: | - make k8s/vald-helm-operator/deploy VERSION=nightly HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS} + make k8s/vald-helm-operator/deploy VERSION=nightly HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" sleep 3 env: HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} From 83b88b226d15d73fde1e56ca4e14f2d1019f03eb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 12:39:16 +0900 Subject: [PATCH 122/139] bugfix use default nightly image Signed-off-by: hlts2 --- .github/actions/e2e-deploy-vald/action.yaml | 2 +- .github/workflows/e2e-deploy.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index b957560c4f..2dfb490414 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -91,7 +91,7 @@ runs: id: deploy_vald_local if: ${{ inputs.use_local_charts == 'true' }} run: | - make k8s/vald/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + make k8s/vald/deploy VERSION=nightly HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" sleep 3 diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 2bce627399..9d1688b8c2 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -243,6 +243,8 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e + with: + target_images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar - name: Deploy Vald id: deploy_vald From 0ed9143aebea68fd90ca0e223f654f6e97cbb9c4 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 14:11:05 +0900 Subject: [PATCH 123/139] bugfix assign of previous step result Signed-off-by: hlts2 --- .github/workflows/e2e-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 9d1688b8c2..46dd6b9379 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -251,7 +251,7 @@ jobs: uses: ./.github/actions/e2e-deploy-vald with: require_minio: "true" - helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-agent-sidecar.yaml wait_for_selector: app=vald-agent-ngt wait_for_timeout: 29m From 816f840554ba827eb58fca7a69f72253fc1393bf Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 15:25:02 +0900 Subject: [PATCH 124/139] set nightly version as default to valdrelease Signed-off-by: hlts2 --- .github/valdrelease/valdrelease.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/valdrelease/valdrelease.yaml b/.github/valdrelease/valdrelease.yaml index 47f120110b..94274292b4 100644 --- a/.github/valdrelease/valdrelease.yaml +++ b/.github/valdrelease/valdrelease.yaml @@ -20,6 +20,8 @@ metadata: name: vald-cluster spec: defaults: + image: + tag: nightly logging: level: info networkPolicy: From 236f7b4815085ca71f0b9578f7ad5c808316651d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 8 Aug 2023 17:12:15 +0900 Subject: [PATCH 125/139] fix typo Signed-off-by: hlts2 --- .github/workflows/update-protobuf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-protobuf.yaml b/.github/workflows/update-protobuf.yaml index afe48500ed..41aa2fda1d 100644 --- a/.github/workflows/update-protobuf.yaml +++ b/.github/workflows/update-protobuf.yaml @@ -36,7 +36,7 @@ jobs: repo: vald-client-go secrets: inherit - client-jave: + client-java: uses: ./.github/workflows/_update-protobuf.yaml with: repo: vald-client-java From 656a0718e9191c1c743a1a899ef5486e67be22b1 Mon Sep 17 00:00:00 2001 From: Hiroto Funakoshi Date: Mon, 14 Aug 2023 14:13:30 +0900 Subject: [PATCH 126/139] [CI]: Improve Chaos E2E test (#2151) * e2e test Signed-off-by: hlts2 * reduce inserted vector size Signed-off-by: hlts2 * reduce search by id count Signed-off-by: hlts2 * enable ingress when running e2e-chaos test Signed-off-by: hlts2 * style: Format code with prettier and gofumpt * change discovery duration for chaos test Signed-off-by: hlts2 * enable persistent volume for e2e chaos testing Signed-off-by: hlts2 * fix pv access mode to prevent unsupported error Signed-off-by: hlts2 * use make e2e/insert/search command Signed-off-by: hlts2 * set workflow environment variable Signed-off-by: hlts2 * add new environment variable for refactor Signed-off-by: hlts2 * fix index replicas Signed-off-by: hlts2 * update helth check duration for grpc client Signed-off-by: hlts2 --------- Signed-off-by: hlts2 Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> --- .github/actions/setup-e2e/action.yaml | 7 +- .github/actions/setup-k3d/action.yaml | 24 +++-- .github/helm/values/values-chaos.yaml | 21 +++- .github/workflows/e2e-chaos.yaml | 104 +++++++++----------- .github/workflows/e2e-code-bench-agent.yaml | 18 ++-- .github/workflows/e2e-deploy.yml | 40 ++++---- .github/workflows/e2e-profiling.yml | 8 +- 7 files changed, 118 insertions(+), 104 deletions(-) diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index af50ebaa3c..914f0a7cab 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -1,5 +1,5 @@ name: "Setup E2E environment" -description: 'Setup the environment to run E2E test' +description: "Setup the environment to run E2E test" inputs: require_libhdf5: @@ -18,6 +18,10 @@ inputs: description: "If k3d is not required, set this to false" required: false default: "true" + ingress_port: + description: 'if it is not "0", ingress will be exposed to the specified port' + required: false + default: "0" target_images: description: "image names" required: false @@ -73,6 +77,7 @@ runs: if: ${{ inputs.require_k3d == 'true' }} with: agents: 3 + ingress_port: ${{ inputs.ingress_port }} options: "--image docker.io/rancher/k3s:latest" - name: Check Kubernetes cluster diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index fc5ff1de1f..7df94954f7 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -3,21 +3,25 @@ description: "GitHub Action for setting up k3d (k3s in Docker). It's lighter tha inputs: version: - description: 'k3d version' + description: "k3d version" required: false - default: 'latest' + default: "latest" name: - description: 'cluster name' + description: "cluster name" required: false - default: 'vald' + default: "vald" + ingress_port: + description: 'if it is not "0", ingress will be exposed to the specified port' + required: false + default: "0" agents: - description: 'number of agents' + description: "number of agents" required: false - default: '3' + default: "3" options: - description: 'options for k3d cluster create command' + description: "options for k3d cluster create command" required: false - default: '' + default: "" runs: using: "composite" @@ -52,9 +56,13 @@ runs: if [ "${AGENTS}" != 0 ]; then OPTIONS="${OPTIONS} --agents ${AGENTS}" fi + if [ "${INGRESS_PORT}" != 0 ]; then + OPTIONS="${OPTIONS} -p ${INGRESS_PORT}:80@loadbalancer" + fi echo "options=${OPTIONS}" >> $GITHUB_OUTPUT env: AGENTS: ${{ inputs.agents }} + INGRESS_PORT: ${{ inputs.ingress_port }} OPTIONS: ${{ inputs.options }} - name: Create k8s cluster diff --git a/.github/helm/values/values-chaos.yaml b/.github/helm/values/values-chaos.yaml index 8ca30303ce..853c82faf0 100644 --- a/.github/helm/values/values-chaos.yaml +++ b/.github/helm/values/values-chaos.yaml @@ -17,6 +17,7 @@ defaults: grpc: client: + health_check_duration: 50ms dial_option: enable_backoff: true server_config: @@ -33,6 +34,13 @@ defaults: gateway: lb: enabled: true + ingress: + enabled: true + host: "localhost" + service: + # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service + annotations: + traefik.ingress.kubernetes.io/service.serversscheme: h2c minReplicas: 2 hpa: enabled: false @@ -41,12 +49,19 @@ gateway: cpu: 100m memory: 50Mi gateway_config: - index_replica: 2 + index_replica: 3 + discoverer: + duration: 50ms agent: minReplicas: 5 maxReplicas: 10 podManagementPolicy: Parallel + persistentVolume: + enabled: true + storageClass: local-path + accessMode: ReadWriteOnce + size: 200Mi hpa: enabled: false resources: @@ -54,6 +69,8 @@ agent: cpu: 100m memory: 50Mi ngt: + index_path: "/var/ngt/index" + enable_in_memory_mode: false auto_index_duration_limit: 3m auto_index_check_duration: 1m auto_index_length: 1000 @@ -68,7 +85,7 @@ discoverer: cpu: 100m memory: 50Mi discoverer: - discovery_duration: 300ms + discovery_duration: 50ms manager: index: diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index d66614b4b0..ebee729e56 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -25,6 +25,11 @@ on: types: - "labeled" +env: + VALUES: .github/helm/values/values-chaos.yaml + DATASET: fashion-mnist-784-euclidean.hdf5 + INGRESS_PORT: 8081 + jobs: dump-contexts-to-log: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' @@ -48,13 +53,15 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e + with: + ingress_port: ${{ env.INGRESS_PORT }} - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-chaos.yaml + values: ${{ env.VALUES }} wait_for_selector: app=vald-lb-gateway - name: Deploy Chaos Mesh @@ -64,19 +71,16 @@ jobs: - name: Run Insert and Search operations run: | - make hack/benchmark/assets/dataset/${DATASET} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_HOST=localhost \ + E2E_BIND_PORT=8081 \ + E2E_PORTFORWARD_ENABLED=false \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ + E2E_INSERT_COUNT=5000 \ + E2E_SEARCH_COUNT=5000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" @@ -93,13 +97,15 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e + with: + ingress_port: ${{ env.INGRESS_PORT }} - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-chaos.yaml + values: ${{ env.VALUES }} wait_for_selector: app=vald-lb-gateway - name: Deploy Chaos Mesh @@ -109,19 +115,16 @@ jobs: - name: Run Insert and Search operations run: | - make hack/benchmark/assets/dataset/${DATASET} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_HOST=localhost \ + E2E_BIND_PORT=8081 \ + E2E_PORTFORWARD_ENABLED=false \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ + E2E_INSERT_COUNT=5000 \ + E2E_SEARCH_COUNT=5000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" @@ -138,13 +141,15 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e + with: + ingress_port: ${{ env.INGRESS_PORT }} - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-chaos.yaml + values: ${{ env.VALUES }} wait_for_selector: app=vald-lb-gateway - name: Deploy Chaos Mesh @@ -154,19 +159,16 @@ jobs: - name: Run Insert and Search operations run: | - make hack/benchmark/assets/dataset/${DATASET} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_HOST=localhost \ + E2E_BIND_PORT=8081 \ + E2E_PORTFORWARD_ENABLED=false \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_TIMEOUT=15m \ E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=10000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ e2e/insert/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" @@ -183,13 +185,15 @@ jobs: - name: Setup E2E environment id: setup_e2e uses: ./.github/actions/setup-e2e + with: + ingress_port: ${{ env.INGRESS_PORT }} - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml + values: ${{ env.VALUES }} wait_for_selector: app=vald-lb-gateway - name: Deploy Chaos Mesh @@ -197,35 +201,17 @@ jobs: with: helm_extra_options: "--set networkChaos.bandwidth.enabled=true" - - name: Run Insert operations - run: | - make hack/benchmark/assets/dataset/${DATASET} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/insert - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - - name: Run Search operations + - name: Run Insert and Search operations run: | - make hack/benchmark/assets/dataset/${DATASET} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_HOST=localhost \ + E2E_BIND_PORT=8081 \ + E2E_PORTFORWARD_ENABLED=false \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_TIMEOUT=15m \ - E2E_SEARCH_COUNT=10000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/search - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + E2E_INSERT_COUNT=5000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + e2e/insert/search slack-notification: name: "Slack notification" diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index 8bcbe8c205..0a10a19fe0 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -42,6 +42,10 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" +env: + DATASET: fashion-mnist-784-euclidean.hdf5 + DATASET_ARGS: fashion-mnist + jobs: dump-contexts-to-log: runs-on: ubuntu-latest @@ -65,11 +69,8 @@ jobs: - name: Run grpc-sequential run: | - make hack/benchmark/assets/dataset/${DATASET} - DATASET_ARGS=${DATASET_ARGS} make bench/agent/sequential/grpc - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - DATASET_ARGS: fashion-mnist + make hack/benchmark/assets/dataset/${{ env.DATASET }} + DATASET_ARGS=${{ env.DATASET_ARGS }} make bench/agent/sequential/grpc - name: Upload artifact uses: actions/upload-artifact@v3 @@ -99,11 +100,8 @@ jobs: - name: Run grpc-stream run: | - make hack/benchmark/assets/dataset/${DATASET} - DATASET_ARGS=${DATASET_ARGS} make bench/agent/stream - env: - DATASET: fashion-mnist-784-euclidean.hdf5 - DATASET_ARGS: fashion-mnist + make hack/benchmark/assets/dataset/${{ env.DATASET }} + DATASET_ARGS=${{ env.DATASET_ARGS }} make bench/agent/stream - name: Upload artifact uses: actions/upload-artifact@v3 diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index 46dd6b9379..f9caa4ef4d 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -24,6 +24,9 @@ on: pull_request: types: - "labeled" +env: + DATASET: fashion-mnist-784-euclidean.hdf5 + jobs: dump-contexts-to-log: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' @@ -58,12 +61,12 @@ jobs: - name: Run E2E CRUD run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=5000 \ E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=5000 \ E2E_GET_OBJECT_COUNT=100 \ E2E_UPDATE_COUNT=100 \ E2E_UPSERT_COUNT=100 \ @@ -73,7 +76,6 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-for-operator: @@ -126,12 +128,12 @@ jobs: - name: Run E2E CRUD run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=5000 \ + E2E_SEARCH_COUNT=5000 \ + E2E_SEARCH_BY_ID_COUNT=5000 \ E2E_GET_OBJECT_COUNT=100 \ E2E_UPDATE_COUNT=100 \ E2E_UPSERT_COUNT=100 \ @@ -141,7 +143,6 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-skip-exist-check: @@ -170,9 +171,9 @@ jobs: - name: Run E2E CRUD run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_INSERT_COUNT=10 \ E2E_SEARCH_COUNT=10 \ E2E_SEARCH_BY_ID_COUNT=10 \ @@ -185,7 +186,6 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e/skip env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-multiapis-crud: @@ -214,9 +214,9 @@ jobs: - name: Run E2E CRUD run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_INSERT_COUNT=100 \ E2E_SEARCH_COUNT=10 \ E2E_SEARCH_BY_ID_COUNT=10 \ @@ -225,7 +225,6 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e/multi env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-agent-and-sidecar: @@ -258,17 +257,16 @@ jobs: - name: Run E2E Agent & Sidecar run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ - E2E_INSERT_COUNT=10000 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=5000 \ E2E_SEARCH_COUNT=4000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ E2E_TARGET_POD_NAME=${POD_NAME} \ E2E_TARGET_NAMESPACE=default \ e2e/sidecar env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} slack-notification: diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index 43c9de812c..319f098204 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -25,6 +25,9 @@ on: types: - "labeled" +env: + DATASET: fashion-mnist-784-euclidean.hdf5 + jobs: dump-contexts-to-log: if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' @@ -66,9 +69,9 @@ jobs: - name: Run E2E CRUD continue-on-error: true run: | - make hack/benchmark/assets/dataset/${DATASET} + make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${DATASET} \ + E2E_DATASET_NAME=${{ env.DATASET }} \ E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=100 \ E2E_SEARCH_BY_ID_COUNT=100 \ @@ -81,7 +84,6 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e env: - DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - name: Get profiles From 15434f32f8c5d97d6fab38cbb69828ba9a4c2756 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 14:26:27 +0900 Subject: [PATCH 127/139] add step to dump github context Signed-off-by: hlts2 --- .github/actions/determine-docker-image-tag/action.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/determine-docker-image-tag/action.yaml b/.github/actions/determine-docker-image-tag/action.yaml index 64964417b5..8cefa9d26d 100644 --- a/.github/actions/determine-docker-image-tag/action.yaml +++ b/.github/actions/determine-docker-image-tag/action.yaml @@ -9,14 +9,18 @@ outputs: runs: using: "composite" steps: - - name: Determine tag name + - name: Dump Github context shell: bash - id: determine_tag_name run: | echo "GITHUB_REF $GITHUB_REF" echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH" echo "GITHUB_EVENT_NAME ${{ github.event_name }}" echo "GITHUB_EVENT_NUMBER ${{ github.event.number }}" + + - name: Determine tag name + shell: bash + id: determine_tag_name + run: | if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` primary_tag="${tag_name}" From bf79459ca10c5be734460d0bacb0aa7360c22cd3 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 14:28:56 +0900 Subject: [PATCH 128/139] refactor step name Signed-off-by: hlts2 --- .github/actions/docker-build/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index ff189e509c..10e84dc7fe 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -63,7 +63,7 @@ runs: env: TARGET: ${{ inputs.target }} - - name: Determine Docker image tag + - name: Determine tag name id: determine_tag_name uses: ./.github/actions/determine-docker-image-tag From 124ba4fe218f0babb45ebfcbf5c1df9c833cda07 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 14:53:02 +0900 Subject: [PATCH 129/139] fix actions description to be more clear Signed-off-by: hlts2 --- .github/actions/deploy-chaos-mesh/action.yaml | 2 +- .../detect-docker-image-tags/action.yaml | 10 +++++----- .../determine-docker-image-tag/action.yaml | 4 ++-- .github/actions/docker-build/action.yaml | 18 +++++++++--------- .../e2e-deploy-vald-helm-operator/action.yaml | 16 ++++++++-------- .github/actions/e2e-deploy-vald/action.yaml | 16 ++++++++-------- .github/actions/notify-slack/action.yaml | 4 ++-- .github/actions/scan-docker-image/action.yaml | 4 ++-- .github/actions/setup-e2e/action.yaml | 10 +++++----- .github/actions/setup-go/action.yaml | 2 +- .github/actions/setup-helm/action.yaml | 2 +- .github/actions/setup-k3d/action.yaml | 10 +++++----- .github/actions/setup-yq/action.yaml | 2 +- .../actions/wait-for-docker-image/action.yaml | 4 +++- 14 files changed, 53 insertions(+), 51 deletions(-) diff --git a/.github/actions/deploy-chaos-mesh/action.yaml b/.github/actions/deploy-chaos-mesh/action.yaml index 8fdbcda2da..41de55f9d0 100644 --- a/.github/actions/deploy-chaos-mesh/action.yaml +++ b/.github/actions/deploy-chaos-mesh/action.yaml @@ -1,5 +1,5 @@ name: "Deploy Chaos Mesh" -description: "Deploy Chaos Mesh" +description: "A action to deploy Chaos Mesh" inputs: chaos_mesh_version: diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index df6e365d50..01f9c8891a 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -14,15 +14,15 @@ # limitations under the License. # name: "Detect Docker image tags" -description: "Detect Docker image tags" +description: "A action to detect Docker image tags" inputs: tag_name: - description: "tag name to check whether exists or not" + description: "Tag name to check whether exists or not" required: true default: "" images: - description: "image names" + description: "Image names" required: false default: "vdaas/vald-agent-ngt \ vdaas/vald-discoverer-k8s \ @@ -30,10 +30,10 @@ inputs: vdaas/vald-manager-index" outputs: HELM_EXTRA_OPTIONS: - description: "helm extra options that specifies image tags" + description: "Helm extra options that specifies image tags" value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} IMAGE_TAGS: - description: "specifies image tags" + description: "Specifies image tags" value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} runs: diff --git a/.github/actions/determine-docker-image-tag/action.yaml b/.github/actions/determine-docker-image-tag/action.yaml index 8cefa9d26d..177109f964 100644 --- a/.github/actions/determine-docker-image-tag/action.yaml +++ b/.github/actions/determine-docker-image-tag/action.yaml @@ -1,9 +1,9 @@ name: "Determine Docker image tag" -description: "Determine Docker image tag" +description: "A action to determine Docker image tag" outputs: PRIMARY_TAG: - description: "primary tag" + description: "Primary tag" value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} runs: diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 10e84dc7fe..1817c593df 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -14,36 +14,36 @@ # limitations under the License. # name: "Build Docker images" -description: "Build Docker images and publish them" +description: "A action to build Docker images and publish them" inputs: target: - description: "build target" + description: "Build target" required: true default: "base" builder: - description: "buildx builder name" + description: "Buildx builder name" required: true default: "" platforms: - description: "if it is specified, specified platforms will be used." + description: "If it is specified, specified platforms will be used." required: false default: "" outputs: IMAGE_NAME: - description: "image name" + description: "Image name" value: ${{ steps.image_name.outputs.IMAGE_NAME }} ALTER_IMAGE_NAME: - description: "alter image name" + description: "Alter image name" value: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }} PRIMARY_TAG: - description: "primary tag" + description: "Primary tag" value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} PLATFORMS: - description: "target platforms" + description: "Target platforms" value: ${{ steps.determine_platforms.outputs.PLATFORMS }} EXTRA_TAGS: - description: "extra tags" + description: "Extra tags" value: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }} runs: diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index 025cde14b5..09dc80faef 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Setting E2E test environment" -description: "A workflow to set E2E test environment" +name: "Deploy Vald using vald-helm-operator for E2E test" +description: "A action to deploy vald using vald-helm-operator for E2E test" inputs: require_minio: @@ -22,19 +22,19 @@ inputs: required: false default: "false" helm_extra_options: - description: "extra options that passed to Helm command to deploy vald helm operator" + description: "Extra options that passed to Helm command to deploy vald helm operator" required: false default: "" valdrelease: - description: "path to the valdrelease.yaml that apply to cluster" + description: "Path to the valdrelease.yaml that apply to cluster" required: true default: "true" wait_for_selector: - description: "label selector used for specifying a pod waited for" + description: "Label selector used for specifying a pod waited for" required: false default: "app=vald-lb-gateway" wait_for_timeout: - description: "timeout used for waiting for pods" + description: "Timeout used for waiting for pods" required: false default: "600s" use_local_charts: @@ -43,7 +43,7 @@ inputs: default: "true" outputs: POD_NAME: - description: "a pod name that waited for" + description: "A pod name that waited for" value: ${{ steps.deploy_vald.outputs.POD_NAME }} runs: @@ -56,7 +56,7 @@ runs: run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - name: Output valdrelease + - name: Dump valdrelease shell: bash run: | cat ${{ inputs.valdrelease }} diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 2dfb490414..6a7c2ec8ab 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Setting E2E test environment" -description: "A workflow to set E2E test environment" +name: "Deploy Vald for E2E test" +description: "A action to deploy vald for E2E test" inputs: require_minio: @@ -22,19 +22,19 @@ inputs: required: false default: "false" helm_extra_options: - description: "extra options that passed to Helm command." + description: "Extra options that passed to Helm command." required: false default: "" values: - description: "path to the values.yaml that passed to Helm command." + description: "Path to the values.yaml that passed to Helm command." required: false default: "false" wait_for_selector: - description: "label selector used for specifying a pod waited for" + description: "Label selector used for specifying a pod waited for" required: false default: "app=vald-lb-gateway" wait_for_timeout: - description: "timeout used for waiting for pods" + description: "Timeout used for waiting for pods" required: false default: "600s" use_local_charts: @@ -43,7 +43,7 @@ inputs: default: "true" outputs: POD_NAME: - description: "a pod name that waited for" + description: "A pod name that waited for" value: ${{ steps.get_real_pod_name.outputs.POD_NAME }} runs: @@ -56,7 +56,7 @@ runs: run: | make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - name: Output values + - name: Dump Helm values shell: bash run: | cat ${{ inputs.values }} diff --git a/.github/actions/notify-slack/action.yaml b/.github/actions/notify-slack/action.yaml index fa74f423e2..a386e50f8c 100644 --- a/.github/actions/notify-slack/action.yaml +++ b/.github/actions/notify-slack/action.yaml @@ -1,5 +1,5 @@ name: "Notify Slack" -description: "Notify Slack" +description: "A action to notify Slack" inputs: author_name: @@ -13,7 +13,7 @@ inputs: description: "Slack Notify Webhook URL" required: true status: - description: "Specify success or failure or cancelled or custom." + description: "Specify success or failure or cancelled or custom" required: false runs: diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml index 49fe889ab8..d900db91be 100644 --- a/.github/actions/scan-docker-image/action.yaml +++ b/.github/actions/scan-docker-image/action.yaml @@ -1,5 +1,5 @@ name: "Scan the Docker image" -description: "Scan the Docker image" +description: "A action to scan the Docker image" inputs: image_ref: @@ -7,7 +7,7 @@ inputs: required: true default: "" severity: - description: "severities of vulnerabilities to be displayed" + description: "Severities of vulnerabilities to be displayed" required: false default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 914f0a7cab..2f0ffbffba 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -1,5 +1,5 @@ name: "Setup E2E environment" -description: "Setup the environment to run E2E test" +description: "A action to set up the environment for executing E2E test" inputs: require_libhdf5: @@ -19,11 +19,11 @@ inputs: required: false default: "true" ingress_port: - description: 'if it is not "0", ingress will be exposed to the specified port' + description: 'If it is not "0", ingress will be exposed to the specified port' required: false default: "0" target_images: - description: "image names" + description: "Image names" required: false default: "vdaas/vald-agent-ngt \ vdaas/vald-discoverer-k8s \ @@ -32,10 +32,10 @@ inputs: outputs: HELM_EXTRA_OPTIONS: - description: "helm extra options that specifies E2E target image tags" + description: "Helm extra options that specifies E2E target image tags" value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} IMAGE_TAGS: - description: "specifies E2E target image tags" + description: "Specifies E2E target image tags" value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} runs: diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml index 76faa4f911..7ac61ed210 100644 --- a/.github/actions/setup-go/action.yaml +++ b/.github/actions/setup-go/action.yaml @@ -1,5 +1,5 @@ name: "Setup Go environment" -description: "Setup Go environment" +description: "A action to set up Go environment" inputs: go_version: diff --git a/.github/actions/setup-helm/action.yaml b/.github/actions/setup-helm/action.yaml index 0d0890d4f4..9c2d7f3111 100644 --- a/.github/actions/setup-helm/action.yaml +++ b/.github/actions/setup-helm/action.yaml @@ -1,5 +1,5 @@ name: "Setup Helm environment" -description: "Setup Helm environment" +description: "A action to set up Helm environment" inputs: helm_version: diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 7df94954f7..b23ef0c7f9 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -1,5 +1,5 @@ name: "Setup k3d environment" -description: "GitHub Action for setting up k3d (k3s in Docker). It's lighter than KinD (Kubernetes in Docker)" +description: "A action to set up k3d (k3s in Docker)" inputs: version: @@ -7,19 +7,19 @@ inputs: required: false default: "latest" name: - description: "cluster name" + description: "Cluster name" required: false default: "vald" ingress_port: - description: 'if it is not "0", ingress will be exposed to the specified port' + description: 'If it is not "0", ingress will be exposed to the specified port' required: false default: "0" agents: - description: "number of agents" + description: "Number of agents" required: false default: "3" options: - description: "options for k3d cluster create command" + description: "Options for k3d cluster create command" required: false default: "" diff --git a/.github/actions/setup-yq/action.yaml b/.github/actions/setup-yq/action.yaml index 7831683151..13ad2a1614 100644 --- a/.github/actions/setup-yq/action.yaml +++ b/.github/actions/setup-yq/action.yaml @@ -1,5 +1,5 @@ name: "Setup yq environment" -description: "GitHub Action for setting up yq" +description: "A action to set up yq" inputs: yq_version: diff --git a/.github/actions/wait-for-docker-image/action.yaml b/.github/actions/wait-for-docker-image/action.yaml index 375e03caae..851041813a 100644 --- a/.github/actions/wait-for-docker-image/action.yaml +++ b/.github/actions/wait-for-docker-image/action.yaml @@ -14,7 +14,8 @@ # limitations under the License. # name: "Wait for Docker Images" -description: "Wait for Docker images to be published" +description: "A action to wait for Docker images to be published" + inputs: images: description: "image names" @@ -24,6 +25,7 @@ inputs: vdaas/vald-lb-gateway \ vdaas/vald-manager-index" outputs: {} + runs: using: "composite" steps: From 6467d58925e5d99b4e4d278e30b5be3c9a5b0e7e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 16:14:24 +0900 Subject: [PATCH 130/139] update job name formatting and update workflow description Signed-off-by: hlts2 --- .github/workflows/_docker-image.yaml | 2 +- .github/workflows/reviewdog-markdown.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml index 9de800c8dc..1017841996 100644 --- a/.github/workflows/_docker-image.yaml +++ b/.github/workflows/_docker-image.yaml @@ -8,7 +8,7 @@ on: required: true platforms: type: string - description: "if it is specified, specified platforms will be used." + description: "If it is specified, specified platforms will be used." required: false default: "" diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index 4f29c5b419..c6dda495eb 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/dump-context - languagetool_apis_charts: + languagetool-apis-charts: name: runner / LanguageTool / apis & charts runs-on: ubuntu-latest steps: @@ -49,7 +49,7 @@ jobs: disabled_rules: "DOUBLE_PUNCTUATION,WORD_CONTAINS_UNDERSCORE,ARROWS,CURRENCY,DASH_RULE,EN_QUOTES" disabled_categories: "TYPOS,TYPOGRAPHY,STYLE,CASING" - languagetool_docs: + languagetool-docs: name: runner / LanguageTool / Docs runs-on: ubuntu-latest steps: From 3f0b60179970a60e11984e9bdcb7c999e9d01756 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 16:23:58 +0900 Subject: [PATCH 131/139] debug for slack notifycation Signed-off-by: hlts2 --- .github/workflows/e2e-chaos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index ebee729e56..ab11aaff13 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -215,7 +215,7 @@ jobs: slack-notification: name: "Slack notification" - if: startsWith( github.ref, 'refs/tags/') + # if: startsWith( github.ref, 'refs/tags/') runs-on: ubuntu-latest needs: - agent-failure From 3051c577ee6fa832ad41ba108d30f255d264b799 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 16:24:20 +0900 Subject: [PATCH 132/139] Revert "debug for slack notifycation" This reverts commit 21d7e3dac754631f9098b8be7ea5209f47bea9f3. --- .github/workflows/e2e-chaos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index ab11aaff13..ebee729e56 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -215,7 +215,7 @@ jobs: slack-notification: name: "Slack notification" - # if: startsWith( github.ref, 'refs/tags/') + if: startsWith( github.ref, 'refs/tags/') runs-on: ubuntu-latest needs: - agent-failure From 0e778d9353ea8640843b1c132c1117056c1aac46 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 16:51:17 +0900 Subject: [PATCH 133/139] update description of dump-context action Signed-off-by: hlts2 --- .github/actions/dump-context/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dump-context/action.yaml b/.github/actions/dump-context/action.yaml index 4ab39331d5..33e3415d39 100644 --- a/.github/actions/dump-context/action.yaml +++ b/.github/actions/dump-context/action.yaml @@ -1,5 +1,5 @@ name: "Dump context to log" -description: "Dump context to log" +description: "A action to dump context to log" runs: using: "composite" From b20771120afff1b1c25f9aca330892cb59065cef Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 17:21:52 +0900 Subject: [PATCH 134/139] add command and fix workflow run rule Signed-off-by: hlts2 --- .github/workflows/test-hack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 51dd32c141..1eff12949e 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -30,13 +30,14 @@ on: paths: - "go.mod" - "go.sum" - - ".github/workflows/test.yml" + - ".github/workflows/test-hack.yml" - "hack/gorules/**" - "hack/helm/**" - "hack/license/**" - "hack/tools/**" env: + # NOTE: This variable is used to make gorules run properly. GOPATH: ${{ github.workspace }} PROJECT_ROOT_DIR: src/github.com/vdaas/vald From 4b3d844484c0a2ac109ef374ce419413330e667c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 17:29:58 +0900 Subject: [PATCH 135/139] add workflow execution rule for index manager docker build Signed-off-by: hlts2 --- .github/workflows/dockers-manager-index-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml index 2ff1f2b3db..b517d46a24 100644 --- a/.github/workflows/dockers-manager-index-image.yml +++ b/.github/workflows/dockers-manager-index-image.yml @@ -25,6 +25,7 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" @@ -40,6 +41,7 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" @@ -55,6 +57,7 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" From d201bca66220232835b1f13e163defb38bafc05c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Mon, 14 Aug 2023 17:51:30 +0900 Subject: [PATCH 136/139] enable rebase command Signed-off-by: hlts2 --- .github/workflows/chatops.yml | 88 +++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index 762acde0fb..d3ca87270e 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest if: github.event.comment.user.login != 'vdaas-ci' steps: - - name: check PR Comments + - name: Check PR Comments id: check_comments_label uses: machine-learning-apps/actions-chatops@master with: @@ -41,7 +41,7 @@ jobs: - uses: ./.github/actions/setup-yq if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' - - name: check permissions + - name: Check permissions if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' id: check_permissions run: | @@ -62,7 +62,7 @@ jobs: USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: label - - name: add label + - name: Add label if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | labels=`echo "${TRAILING_LINE}" | jq -cMR 'split(" ")'` @@ -78,6 +78,88 @@ jobs: ISSUE_URL: ${{ github.event.issue.labels_url }} USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} TRAILING_LINE: ${{ steps.check_comments_label.outputs.TRAILING_LINE }} + rebase: + name: Rebase + runs-on: ubuntu-latest + if: github.event.comment.user.login != 'vdaas-ci' + steps: + - name: Check PR Comments + id: check_comments_rebase + uses: machine-learning-apps/actions-chatops@master + with: + TRIGGER_PHRASE: "/rebase" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: ./.github/actions/setup-yq + if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' + + - name: Check permissions + if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' + id: check_permissions + run: | + curl -o chatops_permissions.yaml https://raw.githubusercontent.com/vdaas/vald/main/.github/chatops_permissions.yaml + roles=`yq r chatops_permissions.yaml "users.${USERNAME}.role.*"` + if [ "$roles" = "" ]; then + roles=`yq r chatops_permissions.yaml "default-roles.*"` + fi + for role in `echo $roles` + do + if yq r chatops_permissions.yaml "roles.${role}.policies.*" | grep "^${REQUIRED_POLICY}$" > /dev/null 2>&1 ; then + echo "[OK] rebase requested by ${USERNAME}" + echo "EXECUTABLE=true" >> $GITHUB_OUTPUT + break + fi + done + env: + USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} + REQUIRED_POLICY: rebase + + - name: Comment back to PR + if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + run: | + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request POST \ + --data "{\"body\": \"**[REBASE]** Rebase triggered by ${USERNAME} for branch: ${BRANCH}\"}" \ + $API_URL + env: + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: ${{ github.event.issue.comments_url }} + BRANCH: ${{ steps.check_comments_rebase.outputs.BRANCH_NAME }} + USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} + + - uses: actions/checkout@v3 + if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + with: + fetch-depth: 0 + token: ${{ secrets.DISPATCH_TOKEN }} + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Automatic Rebase + if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + uses: cirrus-actions/rebase@1.5 + env: + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + + - name: Failure comment + if: failure() + run: | + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request POST \ + --data "{\"body\": \"**[REBASE]** Failed to rebase.\"}" \ + $API_URL + env: + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: ${{ github.event.issue.comments_url }} gentest: name: Add tests From 9433595290976b48714dd7a54d619d636a277dbb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 15 Aug 2023 14:58:00 +0900 Subject: [PATCH 137/139] change file name from e2e-deploy to e2e Signed-off-by: hlts2 --- .github/workflows/{e2e-deploy.yml => e2e.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{e2e-deploy.yml => e2e.yml} (100%) diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e.yml similarity index 100% rename from .github/workflows/e2e-deploy.yml rename to .github/workflows/e2e.yml From bfdd348a7b9cd5ec534c972e15c7ce5495629dbb Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 15 Aug 2023 15:04:04 +0900 Subject: [PATCH 138/139] revert moditication about vector count Signed-off-by: hlts2 --- .github/workflows/e2e.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f9caa4ef4d..180f83cd54 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -64,9 +64,9 @@ jobs: make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=5000 \ + E2E_INSERT_COUNT=10000\ E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=5000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ E2E_GET_OBJECT_COUNT=100 \ E2E_UPDATE_COUNT=100 \ E2E_UPSERT_COUNT=100 \ @@ -131,9 +131,9 @@ jobs: make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=5000 \ - E2E_SEARCH_COUNT=5000 \ - E2E_SEARCH_BY_ID_COUNT=5000 \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ E2E_GET_OBJECT_COUNT=100 \ E2E_UPDATE_COUNT=100 \ E2E_UPSERT_COUNT=100 \ @@ -260,7 +260,7 @@ jobs: make hack/benchmark/assets/dataset/${{ env.DATASET }} make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=5000 \ + E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=4000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ E2E_TARGET_POD_NAME=${POD_NAME} \ From ae66bee94262c71dc50ff1d5379bdf71866de327 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 15 Aug 2023 15:10:40 +0900 Subject: [PATCH 139/139] change file name from test to unit-test Signed-off-by: hlts2 --- .github/workflows/{test.yml => unit-test.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test.yml => unit-test.yaml} (99%) diff --git a/.github/workflows/test.yml b/.github/workflows/unit-test.yaml similarity index 99% rename from .github/workflows/test.yml rename to .github/workflows/unit-test.yaml index ffb46c3d7b..3f96d047ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/unit-test.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Run tests" +name: "Run unit tests" on: push: branches: