From 08206fc1d59f7da3c436612af0c5fd86f6acb0e2 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:22:28 +0900 Subject: [PATCH 1/9] fix: add make command for two version feature Signed-off-by: hlts2 --- Makefile | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a7465a69..86f8d00d 100755 --- a/Makefile +++ b/Makefile @@ -24,9 +24,11 @@ PKGREPO = github.com/$(REPO)/$(PKGNAME) VALD_SHA = VALD_SHA VALD_VERSION = VALD_VERSION VALD_DIR = vald-origin +VALD_CHECKOUT_REF ?= main ROOTDIR = $(eval ROOTDIR := $(shell git rev-parse --show-toplevel))$(ROOTDIR) GO_VERSION := $(eval GO_VERSION := $(shell cat GO_VERSION))$(GO_VERSION) +TEST_DATASET_PATH = wordvecs1000.json MAKELISTS = Makefile @@ -62,9 +64,6 @@ help: clean: -@rm -rf $(VALD_DIR) -$(VALD_DIR): - git clone --depth 1 https://$(VALDREPO) $(VALD_DIR) - .PHONY: sync/v1 ## sync/v1 synchronize VALD_DIR's generated v1 pbgo to v1 dir and patch it sync/v1: $(VALD_DIR) @@ -82,6 +81,18 @@ sync/v1: $(VALD_DIR) find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/sync%sync%g" rm -rf $(VALD_DIR) +$(VALD_DIR): + git clone https://$(VALDREPO) $(VALD_DIR) + +## checkout vald repository +vald/checkout: $(VALD_DIR) + cd $(VALD_DIR) && git checkout $(VALD_CHECKOUT_REF) + +.PHONY: vald/origin/sha/print +## print origin VALD_SHA value +vald/origin/sha/print: $(VALD_DIR) + @cd $(VALD_DIR) && git rev-parse HEAD | tr -d '\n' + .PHONY: vald/sha/print ## print VALD_SHA value vald/sha/print: @@ -90,15 +101,50 @@ vald/sha/print: .PHONY: vald/sha/update ## update VALD_SHA value vald/sha/update: $(VALD_DIR) - (cd $(VALD_DIR); git rev-parse HEAD > ../$(VALD_SHA)) - cp $(VALD_DIR)/versions/VALD_VERSION $(VALD_VERSION) - cp $(VALD_DIR)/versions/GO_VERSION $(ROOTDIR)/GO_VERSION + (cd $(VALD_DIR); git rev-parse HEAD | tr -d '\n' > ../$(VALD_SHA)) .PHONY: vald/version/print ## print VALD_VERSION value vald/version/print: @cat $(VALD_VERSION) +.PHONY: vald/client/version/print +## print VALD_CLIENT_JAVA_VERSION value +vald/client/version/print: vald/version/print + +.PHONY: vald/client/version/update +## update VALD_CLIENT_JAVA_VERSION value +vald/client/version/update: $(VALD_DIR) + cp $(VALD_DIR)/versions/VALD_VERSION $(VALD_VERSION) + cp $(VALD_DIR)/versions/GO_VERSION $(ROOTDIR)/GO_VERSION + +.PHONY: test +## Execute test +test: $(TEST_DATASET_PATH) + go test -v ./tests/v1/e2e_test.go + +$(TEST_DATASET_PATH): + curl -L https://raw.githubusercontent.com/rinx/word2vecjson/master/data/wordvecs1000.json -o $(TEST_DATASET_PATH) + +.PHONY: ci/deps/install +## install deps for CI environment +ci/deps/install: + @echo "Nothing do be done" + +.PHONY: ci/deps/update +## update deps for CI environment +ci/deps/update: mod + +.PHONY: ci/package/prepare +## prepare for publich +ci/package/prepare: + @echo "Nothing do be done" + +.PHONY: ci/package/publish +## publich packages +ci/package/publish: mod + @echo "Nothing do be done" + .PHONY: mod ## update go.mod mod: From fbb977057bac648e450a5d467ff58e42e6522744 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:25:06 +0900 Subject: [PATCH 2/9] fix: add workflow for release and e2e and sync Signed-off-by: hlts2 --- .github/workflows/e2e.yaml | 33 +++++++++++++++++++ .github/workflows/release.yml | 32 +++++++++++++++++++ .github/workflows/sync.yaml | 60 +++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/sync.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..e4283720 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,33 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Run E2E test" +on: + push: + branches: + - main + pull_request: + +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # TODO: change to the following before main merge. + # - uses: vdaas/vald-client-ci/.github/actions/e2e@main + - uses: vdaas/vald-client-ci/.github/actions/e2e@feat/add-common-workflows + with: + client_type: go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..ed04e136 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Run release" +on: + push: + tags: + - '*.*.*' + - 'v*.*.*' + - '*.*.*-*' + - 'v*.*.*-*' +jobs: + release: + # TODO: change to the following before main merge. + # uses: vdaas/vald-client-ci/.github/workflows/_release.yaml@main + uses: vdaas/vald-client-ci/.github/workflows/_release.yaml@feat/add-common-workflows + with: + client_type: go + secrets: inherit diff --git a/.github/workflows/sync.yaml b/.github/workflows/sync.yaml new file mode 100644 index 00000000..785796f6 --- /dev/null +++ b/.github/workflows/sync.yaml @@ -0,0 +1,60 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Sync Vald" +on: + workflow_dispatch: + # schedule: + # - cron: "*/5 * * * *" +jobs: + sync: + # TODO: change to the following before main merge. + # uses: vdaas/vald-client-ci/.github/workflows/sync.yaml@main + uses: vdaas/vald-client-ci/.github/workflows/sync.yaml@feat/add-common-workflows + with: + client_type: go + secrets: + CI_USER: ${{ secrets.DISPATCH_USER }} + CI_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + + build-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: debug for build + run: | + echo "Install dependencies" + make ci/deps/install + + NPM_BIN=$(npm prefix) + ls ${NPM_BIN}/node_modules/@bufbuild/buf/bin/buf + + echo "Build proto" + export GOPATH=$HOME/go + export PATH=$HOME/grpc-java:$PATH + make proto + make vald/sha/update + make vald/client/version/update + + echo "Update dependencies" + make ci/deps/update + + git diff + git diff --name-only + - uses: vdaas/vald-client-ci/.github/actions/e2e@feat/add-common-workflows + with: + client_type: go From fe58eb3bdeb14700fc5f4104c4cf3d2a7b64ba22 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:30:51 +0900 Subject: [PATCH 3/9] fix: deleted unnecessary command dependency Signed-off-by: hlts2 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 86f8d00d..66cdc3e4 100755 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ ci/package/prepare: .PHONY: ci/package/publish ## publich packages -ci/package/publish: mod +ci/package/publish: @echo "Nothing do be done" .PHONY: mod From bb4d788291c5578ad48fa8972d1ee5253cccc56c Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:32:58 +0900 Subject: [PATCH 4/9] fix: change debug job definition Signed-off-by: hlts2 --- .github/workflows/e2e.yaml | 28 ++++++++++++++++++++++++++++ .github/workflows/sync.yaml | 28 ---------------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e4283720..15456ac3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -31,3 +31,31 @@ jobs: - uses: vdaas/vald-client-ci/.github/actions/e2e@feat/add-common-workflows with: client_type: go + + build-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: debug for build + run: | + echo "Install dependencies" + make ci/deps/install + + NPM_BIN=$(npm prefix) + ls ${NPM_BIN}/node_modules/@bufbuild/buf/bin/buf + + echo "Build proto" + export GOPATH=$HOME/go + export PATH=$HOME/grpc-java:$PATH + make proto + make vald/sha/update + make vald/client/version/update + + echo "Update dependencies" + make ci/deps/update + + git diff + git diff --name-only + - uses: vdaas/vald-client-ci/.github/actions/e2e@feat/add-common-workflows + with: + client_type: go diff --git a/.github/workflows/sync.yaml b/.github/workflows/sync.yaml index 785796f6..66fceb67 100644 --- a/.github/workflows/sync.yaml +++ b/.github/workflows/sync.yaml @@ -30,31 +30,3 @@ jobs: CI_USER: ${{ secrets.DISPATCH_USER }} CI_TOKEN: ${{ secrets.DISPATCH_TOKEN }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - - build-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: debug for build - run: | - echo "Install dependencies" - make ci/deps/install - - NPM_BIN=$(npm prefix) - ls ${NPM_BIN}/node_modules/@bufbuild/buf/bin/buf - - echo "Build proto" - export GOPATH=$HOME/go - export PATH=$HOME/grpc-java:$PATH - make proto - make vald/sha/update - make vald/client/version/update - - echo "Update dependencies" - make ci/deps/update - - git diff - git diff --name-only - - uses: vdaas/vald-client-ci/.github/actions/e2e@feat/add-common-workflows - with: - client_type: go From 5e32abd7051677a5b7f824be4fa5e87be62feba8 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:34:17 +0900 Subject: [PATCH 5/9] fix: deleted unnecessary command Signed-off-by: hlts2 --- .github/workflows/e2e.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 15456ac3..04fc8428 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -41,9 +41,6 @@ jobs: echo "Install dependencies" make ci/deps/install - NPM_BIN=$(npm prefix) - ls ${NPM_BIN}/node_modules/@bufbuild/buf/bin/buf - echo "Build proto" export GOPATH=$HOME/go export PATH=$HOME/grpc-java:$PATH From 591fd1cf1dd2753c42edf97a897b429482db3f16 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:35:58 +0900 Subject: [PATCH 6/9] fix: add PHONY to make file Signed-off-by: hlts2 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 66cdc3e4..c65fb199 100755 --- a/Makefile +++ b/Makefile @@ -84,6 +84,7 @@ sync/v1: $(VALD_DIR) $(VALD_DIR): git clone https://$(VALDREPO) $(VALD_DIR) +.PHONY: vald/checkout ## checkout vald repository vald/checkout: $(VALD_DIR) cd $(VALD_DIR) && git checkout $(VALD_CHECKOUT_REF) From acdf24749db0504c2229cdfacf696a92da8daa4e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:38:08 +0900 Subject: [PATCH 7/9] fix: change make command name Signed-off-by: hlts2 --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c65fb199..96b5f53a 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ cyan = /bin/echo -e "\x1b[36m\#\# $1\x1b[0m" .PHONY: all ## execute clean and proto -all: clean sync/v1 mod clean +all: clean proto mod clean .PHONY: help ## print all available commands @@ -64,9 +64,9 @@ help: clean: -@rm -rf $(VALD_DIR) -.PHONY: sync/v1 -## sync/v1 synchronize VALD_DIR's generated v1 pbgo to v1 dir and patch it -sync/v1: $(VALD_DIR) +.PHONY: proto +## proto synchronize VALD_DIR's generated v1 pbgo to v1 dir and patch it +proto: $(VALD_DIR) rm -rf $(ROOTDIR)/v1 cp -r $(VALD_DIR)/apis/grpc/v1 $(ROOTDIR)/v1 rm -rf $(ROOTDIR)/v1/discoverer \ From c769e358accd899b5a4c49f3ad327cdbe5322ce5 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:42:14 +0900 Subject: [PATCH 8/9] fix: deleted unnecessary command Signed-off-by: hlts2 --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 96b5f53a..f5d853f6 100755 --- a/Makefile +++ b/Makefile @@ -79,7 +79,6 @@ proto: $(VALD_DIR) find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/apis/grpc/v1%github.com/vdaas/vald-client-go/v1%g" find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/io%io%g" find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/sync%sync%g" - rm -rf $(VALD_DIR) $(VALD_DIR): git clone https://$(VALDREPO) $(VALD_DIR) From 336be5b3c5c9fdb4d0ae7c6508ec4a6f2dda6856 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 7 May 2024 18:50:01 +0900 Subject: [PATCH 9/9] fix: fix unexpected import path Signed-off-by: hlts2 --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f5d853f6..32a7494e 100755 --- a/Makefile +++ b/Makefile @@ -76,7 +76,9 @@ proto: $(VALD_DIR) # $(ROOTDIR)/v1/mirror \ find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/net/grpc/codes%google.golang.org/grpc/codes%g" find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/net/grpc/status%google.golang.org/grpc/status%g" - find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/apis/grpc/v1%github.com/vdaas/vald-client-go/v1%g" + # find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/apis/grpc/v1%github.com/vdaas/vald-client-go/v1%g" + # TODO: The following is for vald-ci-labs, so remove it when implementing in vald-client-go. + find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/apis/grpc/v1%github.com/vdaas/vald-ci-labs/v1%g" find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/io%io%g" find $(ROOTDIR)/v1/* -name '*.go' | xargs sed -i -E "s%github.com/vdaas/vald/internal/sync%sync%g"