From e89e1aea5bd7f019da85331d0e7a81d37a103f14 Mon Sep 17 00:00:00 2001 From: vankichi Date: Wed, 16 Oct 2024 16:30:33 +0900 Subject: [PATCH 1/5] :whale: add example-client docker image Signed-off-by: vankichi --- .../dockers-example-client-image.yaml | 76 +++++++++++++++ Makefile | 3 +- Makefile.d/build.mk | 10 ++ Makefile.d/docker.mk | 16 +++- Makefile.d/functions.mk | 39 ++++++++ dockers/example/client/Dockerfile | 95 +++++++++++++++++++ hack/docker/gen/main.go | 14 +++ 7 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dockers-example-client-image.yaml create mode 100644 dockers/example/client/Dockerfile diff --git a/.github/workflows/dockers-example-client-image.yaml b/.github/workflows/dockers-example-client-image.yaml new file mode 100644 index 0000000000..95e2ac3a71 --- /dev/null +++ b/.github/workflows/dockers-example-client-image.yaml @@ -0,0 +1,76 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go + +name: "Build docker image: example-client" +on: + push: + branches: + - main + - release/v*.* + - "!release/v*.*.*" + tags: + - "*.*.*" + - v*.*.* + - "*.*.*-*" + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/example/client/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-example-client-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - example/client/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - "!internal/**/*_test.go" + - "!internal/**/*_mock.go" + - "!internal/db/**" + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/example/client/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-example-client-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - example/client/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - "!internal/**/*_test.go" + - "!internal/**/*_mock.go" + - "!internal/db/**" + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: example-client + secrets: inherit diff --git a/Makefile b/Makefile index 3280bc5f36..4daf536e08 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,8 @@ TAG ?= latest CRORG ?= $(ORG) GHCRORG = ghcr.io/$(REPO) AGENT_IMAGE = $(NAME)-agent -AGENT_NGT_IMAGE = $(AGENT_IMAGE)-ngt AGENT_FAISS_IMAGE = $(AGENT_IMAGE)-faiss +AGENT_NGT_IMAGE = $(AGENT_IMAGE)-ngt AGENT_SIDECAR_IMAGE = $(AGENT_IMAGE)-sidecar BENCHMARK_JOB_IMAGE = $(NAME)-benchmark-job BENCHMARK_OPERATOR_IMAGE = $(NAME)-benchmark-operator @@ -36,6 +36,7 @@ BUILDKIT_SYFT_SCANNER_IMAGE = $(BUILDKIT_IMAGE)-syft-scanner CI_CONTAINER_IMAGE = $(NAME)-ci-container DEV_CONTAINER_IMAGE = $(NAME)-dev-container DISCOVERER_IMAGE = $(NAME)-discoverer-k8s +EXAMPLE_CLIENT_IMAGE = $(NAME)-example-client FILTER_GATEWAY_IMAGE = $(NAME)-filter-gateway HELM_OPERATOR_IMAGE = $(NAME)-helm-operator INDEX_CORRECTION_IMAGE = $(NAME)-index-correction diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index c2132a1630..074ed42cce 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -31,6 +31,7 @@ binary/build: \ cmd/tools/benchmark/job/job \ cmd/tools/benchmark/operator/operator \ cmd/tools/cli/loadtest/loadtest \ + example/client/client \ cmd/agent/core/ngt/ngt \ cmd/agent/core/faiss/faiss \ rust/target/debug/agent \ @@ -103,6 +104,10 @@ cmd/tools/cli/loadtest/loadtest: $(eval CGO_ENABLED = 1) $(call go-build,tools/cli/loadtest,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@) +example/client/client: + $(eval CGO_ENABLED = 1) + $(call go-example-build,example/client,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@) + rust/target/release/agent: pushd rust && cargo build -p agent --release && popd @@ -119,6 +124,7 @@ binary/build/zip: \ artifacts/vald-benchmark-operator-$(GOOS)-$(GOARCH).zip \ artifacts/vald-cli-loadtest-$(GOOS)-$(GOARCH).zip \ artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip \ artifacts/vald-filter-gateway-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \ @@ -192,3 +198,7 @@ artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip: cmd/index/job/readrepli artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip: cmd/index/operator/index-operator $(call mkdir, $(dir $@)) zip --junk-paths $@ $< + +artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip: example/client/client + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index d9ef1ac5f4..3e0f7563fe 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -29,16 +29,17 @@ docker/build: \ docker/build/ci-container \ docker/build/dev-container \ docker/build/discoverer-k8s \ + docker/build/example-client \ docker/build/gateway-filter \ docker/build/gateway-lb \ docker/build/gateway-mirror \ + docker/build/helm-operator \ docker/build/index-correction \ docker/build/index-creation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ docker/build/manager-index \ - docker/build/helm-operator \ docker/build/readreplica-rotate docker/xpanes/build: @@ -56,6 +57,7 @@ docker/xpanes/build: docker/build/ci-container \ docker/build/dev-container \ docker/build/discoverer-k8s \ + docker/build/example-client \ docker/build/gateway-filter \ docker/build/gateway-lb \ docker/build/gateway-mirror \ @@ -383,3 +385,15 @@ docker/build/benchmark-operator: @make DOCKERFILE="$(ROOTDIR)/dockers/tools/benchmark/operator/Dockerfile" \ IMAGE=$(BENCHMARK_OPERATOR_IMAGE) \ docker/build/image + +.PHONY: docker/name/example-client +docker/name/example-client: + @echo "$(ORG)/$(EXAMPLE_CLIENT_IMAGE)" + +.PHONY: docker/build/example-client +## build example client docker image +docker/build/example-client: + @make DOCKERFILE="$(ROOTDIR)/dockers/example/client/Dockerfile" \ + IMAGE=$(EXAMPLE_CLIENT_IMAGE) \ + DOCKER_OPTS="--build-arg ZLIB_VERSION=$(ZLIB_VERSION) --build-arg HDF5_VERSION=$(HDF5_VERSION)" \ + docker/build/image diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index f9c3312112..fbcb4803b3 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -91,6 +91,45 @@ define go-build $6 -version endef +define go-example-build + echo $(GO_SOURCES_INTERNAL) + echo $(PBGOS) + echo $(shell find $(ROOTDIR)/$1 -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + cd $(ROOTDIR)/$1 && \ + CFLAGS="$(CFLAGS)" \ + CXXFLAGS="$(CXXFLAGS)" \ + CGO_ENABLED=$(CGO_ENABLED) \ + CGO_CXXFLAGS="$3" \ + CGO_FFLAGS="$3" \ + CGO_LDFLAGS="$3" \ + GO111MODULE=on \ + GOARCH=$(GOARCH) \ + GOOS=$(GOOS) \ + GOPRIVATE=$(GOPRIVATE) \ + GO_VERSION=$(GO_VERSION) \ + go build \ + --ldflags "-w $2 \ + -extldflags '$3' \ + -X '$(GOPKG)/internal/info.AlgorithmInfo=$5' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$(if $(filter 1,$(strip $(CGO_ENABLED))),true,false)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -buildid=" \ + -modcacherw \ + -mod=readonly \ + -a \ + -tags "osusergo netgo static_build$4" \ + -trimpath \ + -o $(ROOTDIR)/$6 \ + main.go + $6 -version +endef + define telepresence [ -z $(SWAP_IMAGE) ] && IMAGE=$2 || IMAGE=$(SWAP_IMAGE) \ && echo "telepresence replaces $(SWAP_DEPLOYMENT_TYPE)/$1 with $${IMAGE}:$(SWAP_TAG)" \ diff --git a/dockers/example/client/Dockerfile b/dockers/example/client/Dockerfile new file mode 100644 index 0000000000..a257c883cb --- /dev/null +++ b/dockers/example/client/Dockerfile @@ -0,0 +1,95 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go +ARG UPX_OPTIONS=-9 +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder +LABEL maintainer="vdaas.org vald team " +# skipcq: DOK-DL3002 +USER root:root +ARG TARGETARCH +ARG TARGETOS +ARG GO_VERSION +ARG RUST_VERSION +ENV APP_NAME=client +ENV DEBIAN_FRONTEND=noninteractive +ENV GO111MODULE=on +ENV GOPATH=/go +ENV GOROOT=/opt/go +ENV HOME=/root +ENV INITRD=No +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV ORG=vdaas +ENV PKG=example/client +ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root +ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN --mount=type=bind,target=.,rw \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ + --mount=type=tmpfs,target="${GOPATH}/src" \ + set -ex \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + ca-certificates \ + curl \ + tzdata \ + locales \ + git \ + cmake \ + g++ \ + gcc \ + libssl-dev \ + unzip \ + libhdf5-dev \ + libaec-dev \ + && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ + && apt-get clean \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ + && make hdf5/install \ + && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" ${PKG}/${APP_NAME} \ + && mv "${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" +# skipcq: DOK-DL3026,DOK-DL3007 +FROM gcr.io/distroless/static:nonroot +LABEL maintainer="vdaas.org vald team " +COPY --from=builder /usr/bin/client /usr/bin/client +# skipcq: DOK-DL3002 +USER nonroot:nonroot +ENTRYPOINT ["/usr/bin/client"] \ No newline at end of file diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 0e0bb98d57..f5ceb0c201 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -317,6 +317,10 @@ var ( "make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" cmd/${PKG}/${APP_NAME}", "mv \"cmd/${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"", } + goExampleBuildCommands = []string{ + "make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" ${PKG}/${APP_NAME}", + "mv \"${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"", + } rustBuildCommands = []string{ "make rust/target/release/${APP_NAME}", "mv \"rust/target/release/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"", @@ -707,6 +711,14 @@ func main() { ngtPreprocess, faissPreprocess)...), }, + "vald-example-client": { + AppName: "client", + PackageDir: "example/client", + ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), + Preprocess: []string{ + "make hdf5/install", + }, + }, "vald-buildbase": { AppName: "buildbase", AliasImage: true, @@ -784,6 +796,8 @@ func main() { } if file.Exists(file.Join(os.Args[1], "cmd", data.PackageDir)) { commands = append(commands, goBuildCommands...) + } else if strings.HasPrefix(data.PackageDir, "example") && file.Exists(file.Join(os.Args[1], data.PackageDir)) { + commands = append(commands, goExampleBuildCommands...) } data.RunCommands = commands mounts := make([]string, 0, len(defaultMounts)+len(goDefaultMounts)) From 30a92c290808e3b15366477d41aa85f4c198842e Mon Sep 17 00:00:00 2001 From: vankichi Date: Wed, 16 Oct 2024 17:54:49 +0900 Subject: [PATCH 2/5] :bug: fix build error Signed-off-by: vankichi --- Makefile.d/functions.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index fbcb4803b3..21c310441f 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -127,7 +127,6 @@ define go-example-build -trimpath \ -o $(ROOTDIR)/$6 \ main.go - $6 -version endef define telepresence From 4e38a0a8638f27f4f0e266a25f3ee844e811b385 Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 17 Oct 2024 11:13:33 +0900 Subject: [PATCH 3/5] :recycle: fix Signed-off-by: vankichi --- example/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/helm/values.yaml b/example/helm/values.yaml index 8b292fbcaf..240917aba1 100644 --- a/example/helm/values.yaml +++ b/example/helm/values.yaml @@ -62,7 +62,7 @@ agent: # The number of dimensions for feature vector of fashion-mnist dataset. dimension: 784 # We use L2-Norm for distance_type. - distance_type: cos + distance_type: l2 # The type of fashion-mnist's feature vectors. object_type: float # Check duration of automatic indexing. From b8beaa717bc11e21658392fa994cb093b68c1f07 Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 17 Oct 2024 14:05:51 +0900 Subject: [PATCH 4/5] :recycle: fix Signed-off-by: vankichi --- Makefile.d/functions.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index 21c310441f..f81819b312 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -112,7 +112,6 @@ define go-example-build -extldflags '$3' \ -X '$(GOPKG)/internal/info.AlgorithmInfo=$5' \ -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ - -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ -X '$(GOPKG)/internal/info.CGOEnabled=$(if $(filter 1,$(strip $(CGO_ENABLED))),true,false)' \ -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ From a160aa01717eea34e174d3e72ae245cfd13502fa Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 17 Oct 2024 14:51:52 +0900 Subject: [PATCH 5/5] :recycle: fix Signed-off-by: vankichi --- Makefile.d/functions.mk | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index f81819b312..c44aa8d34c 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -110,14 +110,6 @@ define go-example-build go build \ --ldflags "-w $2 \ -extldflags '$3' \ - -X '$(GOPKG)/internal/info.AlgorithmInfo=$5' \ - -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ - -X '$(GOPKG)/internal/info.CGOEnabled=$(if $(filter 1,$(strip $(CGO_ENABLED))),true,false)' \ - -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ - -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ - -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ - -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ - -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ -buildid=" \ -modcacherw \ -mod=readonly \