From e52428b0c2df316bd614cf3b5c20f3ddbfde5f4a Mon Sep 17 00:00:00 2001 From: kpango Date: Thu, 8 Feb 2024 15:57:57 +0900 Subject: [PATCH] [bugfix] gcc environment for ARM Signed-off-by: kpango --- .../workflows/dockers-agent-faiss-image.yml | 84 +++++++++++++++++++ .github/workflows/dockers-agent-ngt-image.yml | 1 + .../workflows/dockers-benchmark-job-image.yml | 1 + .../workflows/dockers-helm-operator-image.yml | 1 + Makefile | 11 +-- Makefile.d/actions.mk | 2 +- Makefile.d/build.mk | 26 +++--- Makefile.d/docker.mk | 8 +- Makefile.d/functions.mk | 32 ++++++- Makefile.d/helm.mk | 53 ++++-------- Makefile.d/k8s.mk | 18 ++-- dockers/agent/core/faiss/Dockerfile | 12 +++ dockers/agent/core/ngt/Dockerfile | 12 +++ dockers/operator/helm/Dockerfile | 19 ++++- dockers/tools/benchmark/job/Dockerfile | 11 +++ 15 files changed, 214 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/dockers-agent-faiss-image.yml diff --git a/.github/workflows/dockers-agent-faiss-image.yml b/.github/workflows/dockers-agent-faiss-image.yml new file mode 100644 index 00000000000..297aca6ea90 --- /dev/null +++ b/.github/workflows/dockers-agent-faiss-image.yml @@ -0,0 +1,84 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: agent-faiss" +on: + push: + branches: + - "main" + - "release/v*.*" + - "!release/v*.*.*" + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-faiss-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/faiss/**" + - "cmd/agent/core/faiss/**" + - "dockers/agent/core/faiss/Dockerfile" + - "versions/GO_VERSION" + - "versions/FAISS_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-faiss-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/faiss/**" + - "cmd/agent/core/faiss/**" + - "dockers/agent/core/faiss/Dockerfile" + - "versions/GO_VERSION" + - "versions/FAISS_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-faiss-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/faiss/**" + - "cmd/agent/core/faiss/**" + - "dockers/agent/core/faiss/Dockerfile" + - "versions/GO_VERSION" + - "versions/FAISS_VERSION" +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-faiss + platforms: linux/amd64 + secrets: inherit diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index f786cc5bce7..64ab26c4701 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -80,4 +80,5 @@ jobs: uses: ./.github/workflows/_docker-image.yaml with: target: agent-ngt + platforms: linux/amd64,linux/arm64 secrets: inherit diff --git a/.github/workflows/dockers-benchmark-job-image.yml b/.github/workflows/dockers-benchmark-job-image.yml index 52abd5addc7..d86c454edcb 100644 --- a/.github/workflows/dockers-benchmark-job-image.yml +++ b/.github/workflows/dockers-benchmark-job-image.yml @@ -78,4 +78,5 @@ jobs: uses: ./.github/workflows/_docker-image.yaml with: target: benchmark-job + platforms: linux/amd64,linux/arm64 secrets: inherit diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index 1050a6e56bd..1c9aba06169 100644 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -68,4 +68,5 @@ jobs: uses: ./.github/workflows/_docker-image.yaml with: target: operator/helm + platforms: linux/amd64,linux/arm64 secrets: inherit diff --git a/Makefile b/Makefile index f58e60a03fa..2bd45004c5e 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ DATETIME = $(eval DATETIME := $(shell date -u +%Y/%m/%d_% TAG ?= latest CRORG ?= $(ORG) GHCRORG = ghcr.io/$(ORG)/$(NAME) -AGENT_IMAGE = $(NAME)-agent-ngt +AGENT_NGT_IMAGE = $(NAME)-agent-ngt +AGENT_FAISS_IMAGE = $(NAME)-agent-faiss AGENT_SIDECAR_IMAGE = $(NAME)-agent-sidecar CI_CONTAINER_IMAGE = $(NAME)-ci-container DEV_CONTAINER_IMAGE = $(NAME)-dev-container @@ -386,11 +387,7 @@ clean-generated: .PHONY: license ## add license to files license: - GOPRIVATE=$(GOPRIVATE) \ - MAINTAINER=$(MAINTAINER) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/license/gen/main.go $(ROOTDIR) + $(call gen-license,$(ROOTDIR),$(MAINTAINER)) .PHONY: init ## initialize development environment @@ -571,7 +568,7 @@ faiss/install: /usr/local/lib/libfaiss.so curl -LO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/ cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \ - cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -B build . && \ + cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -B build . && \ make -C build -j faiss && \ make -C build install rm -rf v$(FAISS_VERSION).tar.gz diff --git a/Makefile.d/actions.mk b/Makefile.d/actions.mk index dbec6376240..9c0ad6dc6f1 100644 --- a/Makefile.d/actions.mk +++ b/Makefile.d/actions.mk @@ -24,4 +24,4 @@ list/actions: .PHONY: update/actions # update github actions version update/actions: - @$(call update-github-actions, $(ACTIONS_LIST)) + $(call update-github-actions, $(ACTIONS_LIST)) diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index da3854a5687..677324bfdeb 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -36,7 +36,7 @@ binary/build: \ cmd/agent/core/ngt/ngt: \ ngt/install $(eval CGO_ENABLED = 1) - $(call go-build,agent/core/ngt,-linkmode 'external',-fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS), cgo,NGT-$(NGT_VERSION),$@) + $(call go-build,agent/core/ngt,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS), cgo,NGT-$(NGT_VERSION),$@) cmd/agent/core/faiss/faiss: \ faiss/install @@ -45,50 +45,50 @@ cmd/agent/core/faiss/faiss: \ cmd/agent/sidecar/sidecar: $(eval CGO_ENABLED = 0) - $(call go-build,agent/sidecar,,,,,$@) + $(call go-build,agent/sidecar,,-static,,,$@) cmd/discoverer/k8s/discoverer: $(eval CGO_ENABLED = 0) - $(call go-build,discoverer/k8s,,,,,$@) + $(call go-build,discoverer/k8s,,-static,,,$@) cmd/gateway/lb/lb: $(eval CGO_ENABLED = 0) - $(call go-build,gateway/lb,,,,,$@) + $(call go-build,gateway/lb,,-static,,,$@) cmd/gateway/filter/filter: $(eval CGO_ENABLED = 0) - $(call go-build,gateway/filter,,,,,$@) + $(call go-build,gateway/filter,,-static,,,$@) cmd/gateway/mirror/mirror: $(eval CGO_ENABLED = 0) - $(call go-build,gateway/mirror,,,,,$@) + $(call go-build,gateway/mirror,,-static,,,$@) cmd/manager/index/index: $(eval CGO_ENABLED = 0) - $(call go-build,manager/index,,,,,$@) + $(call go-build,manager/index,,-static,,,$@) cmd/index/job/correction/index-correction: $(eval CGO_ENABLED = 0) - $(call go-build,index/job/correction,,,,,$@) + $(call go-build,index/job/correction,,-static,,,$@) cmd/index/job/creation/index-creation: $(eval CGO_ENABLED = 0) - $(call go-build,index/job/creation,,,,,$@) + $(call go-build,index/job/creation,,-static,,,$@) cmd/index/job/save/index-save: $(eval CGO_ENABLED = 0) - $(call go-build,index/job/save,,,,,$@) + $(call go-build,index/job/save,,-static,,,$@) cmd/index/job/readreplica/rotate/readreplica-rotate: $(eval CGO_ENABLED = 0) - $(call go-build,index/job/readreplica/rotate,,,,,$@) + $(call go-build,index/job/readreplica/rotate,,-static,,,$@) cmd/tools/benchmark/job/job: - $(call go-build,tools/benchmark/job,-linkmode 'external',-fPIC -pthread -fopenmp -std=gnu++20 -lhdf5 -lhdf5_hl -lm -ldl, cgo,$(HDF5_VERSION),$@) + $(call go-build,tools/benchmark/job,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lhdf5 -lhdf5_hl -lm -ldl, cgo,$(HDF5_VERSION),$@) cmd/tools/benchmark/operator/operator: $(eval CGO_ENABLED = 0) - $(call go-build,tools/benchmark/operator,,,,,$@) + $(call go-build,tools/benchmark/operator,,-static,,,$@) .PHONY: binary/build/zip ## build all binaries and zip them diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 3eec0178596..059faea0743 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -77,24 +77,24 @@ endif .PHONY: docker/name/agent-ngt docker/name/agent-ngt: - @echo "$(ORG)/$(AGENT_IMAGE)" + @echo "$(ORG)/$(AGENT_NGT_IMAGE)" .PHONY: docker/build/agent-ngt ## build agent-ngt image docker/build/agent-ngt: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/ngt/Dockerfile" \ - IMAGE=$(AGENT_IMAGE) \ + IMAGE=$(AGENT_NGT_IMAGE) \ docker/build/image .PHONY: docker/name/agent-faiss docker/name/agent-faiss: - @echo "$(ORG)/$(AGENT_IMAGE)" + @echo "$(ORG)/$(AGENT_FAISS_IMAGE)" .PHONY: docker/build/agent-faiss ## build agent-faiss image docker/build/agent-faiss: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/faiss/Dockerfile" \ - IMAGE=$(AGENT_IMAGE) \ + IMAGE=$(AGENT_FAISS_IMAGE) \ docker/build/image .PHONY: docker/name/agent-sidecar diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index 753349cd97f..e9ed120e138 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -70,7 +70,7 @@ define go-build GO_VERSION=$(GO_VERSION) \ go build \ --ldflags "-w $2 \ - -extldflags '-static $3' \ + -extldflags '$3' \ -X '$(GOPKG)/internal/info.AlgorithmInfo=$5' \ -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ @@ -278,15 +278,41 @@ define gen-go-option-test-sources done endef +define gen-license + BIN_PATH="$(TEMP_DIR)/vald-helm-schema-gen" + rm -rf $$BIN_PATH + MAINTAINER=$2 \ + GOPRIVATE=$(GOPRIVATE) \ + GOARCH=$(GOARCH) \ + GOOS=$(GOOS) \ + go build -mod=readonly -a -o $$BIN_PATH $(ROOTDIR)/hack/license/gen/main.go $(ROOTDIR) + $$BIN_PATH $1 + rm -rf $$BIN_PATH +endef + +define gen-vald-helm-schema + BIN_PATH="$(TEMP_DIR)/vald-helm-schema-gen" + rm -rf $$BIN_PATH + GOPRIVATE=$(GOPRIVATE) \ + GOARCH=$(GOARCH) \ + GOOS=$(GOOS) \ + go build -mod=readonly -a -o $$BIN_PATH $(ROOTDIR)/hack/helm/schema/gen/main.go + $$BIN_PATH charts/$1.yaml > charts/$1.schema.json + rm -rf $$BIN_PATH +endef + define gen-vald-crd if [[ -f $(ROOTDIR)/charts/$1/crds/$2.yaml ]]; then \ mv $(ROOTDIR)/charts/$1/crds/$2.yaml $(TEMP_DIR)/$2.yaml; \ fi; + BIN_PATH="$(TEMP_DIR)/vald-helm-crd-schema-gen" + rm -rf $$BIN_PATH GOPRIVATE=$(GOPRIVATE) \ GOARCH=$(GOARCH) \ GOOS=$(GOOS) \ - go run -mod=readonly $(ROOTDIR)/hack/helm/schema/crd/main.go \ - $(ROOTDIR)/charts/$3.yaml > $(TEMP_DIR)/$2-spec.yaml + go build -mod=readonly -a -o $$BIN_PATH $(ROOTDIR)/hack/helm/schema/crd/main.go + $$BIN_PATH $(ROOTDIR)/charts/$3.yaml > $(TEMP_DIR)/$2-spec.yaml + rm -rf $$BIN_PATH $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ $(TEMP_DIR)/$2.yaml $(TEMP_DIR)/$2-spec.yaml > $(ROOTDIR)/charts/$1/crds/$2.yaml endef diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 90dccf378ec..bbf971ac867 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -93,61 +93,40 @@ helm/schema/all: \ helm/schema/vald: charts/vald/values.schema.json charts/vald/values.schema.json: \ - charts/vald/values.yaml \ - hack/helm/schema/gen/main.go - GOPRIVATE=$(GOPRIVATE) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/helm/schema/gen/main.go charts/vald/values.yaml > charts/vald/values.schema.json + charts/vald/values.yaml + $(call gen-vald-helm-schema,vald/values) .PHONY: helm/schema/vald-helm-operator ## generate json schema for Vald Helm Operator Chart helm/schema/vald-helm-operator: charts/vald-helm-operator/values.schema.json charts/vald-helm-operator/values.schema.json: \ - charts/vald-helm-operator/values.yaml \ - hack/helm/schema/gen/main.go - GOPRIVATE=$(GOPRIVATE) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-helm-operator/values.yaml > charts/vald-helm-operator/values.schema.json + charts/vald-helm-operator/values.yaml + $(call gen-vald-helm-schema,vald-helm-operator/values) .PHONY: helm/schema/vald-benchmark-job ## generate json schema for Vald Benchmark Job Chart helm/schema/vald-benchmark-job: charts/vald-benchmark-operator/job-values.schema.json charts/vald-benchmark-operator/job-values.schema.json: \ - charts/vald-benchmark-operator/schemas/job-values.yaml \ - hack/helm/schema/gen/main.go - GOPRIVATE=$(GOPRIVATE) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/schemas/job-values.yaml > charts/vald-benchmark-operator/job-values.schema.json + charts/vald-benchmark-operator/schemas/job-values.yaml + $(call gen-vald-helm-schema,vald-benchmark-operator/schemas/job-values) .PHONY: helm/schema/vald-benchmark-scenario ## generate json schema for Vald Benchmark Job Chart helm/schema/vald-benchmark-scenario: charts/vald-benchmark-operator/scenario-values.schema.json charts/vald-benchmark-operator/scenario-values.schema.json: \ - charts/vald-benchmark-operator/schemas/scenario-values.yaml \ - hack/helm/schema/gen/main.go - GOPRIVATE=$(GOPRIVATE) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/schemas/scenario-values.yaml > charts/vald-benchmark-operator/scenario-values.schema.json + charts/vald-benchmark-operator/schemas/scenario-values.yaml + $(call gen-vald-helm-schema,vald-benchmark-operator/schemas/scenario-values) .PHONY: helm/schema/vald-benchmark-operator ## generate json schema for Vald Benchmark Operator Chart helm/schema/vald-benchmark-operator: charts/vald-benchmark-operator/values.schema.json charts/vald-benchmark-operator/values.schema.json: \ - charts/vald-benchmark-operator/values.yaml \ - hack/helm/schema/gen/main.go - GOPRIVATE=$(GOPRIVATE) \ - GOARCH=$(GOARCH) \ - GOOS=$(GOOS) \ - go run -mod=readonly hack/helm/schema/gen/main.go charts/vald-benchmark-operator/values.yaml > charts/vald-benchmark-operator/values.schema.json - + charts/vald-benchmark-operator/values.yaml + $(call gen-vald-helm-schema,vald-benchmark-operator/values) .PHONY: yq/install ## install yq @@ -172,34 +151,34 @@ helm/schema/crd/all: \ ## generate OpenAPI v3 schema for ValdRelease helm/schema/crd/vald: \ yq/install - @$(call gen-vald-crd,vald-helm-operator,valdrelease,vald/values) + $(call gen-vald-crd,vald-helm-operator,valdrelease,vald/values) .PHONY: helm/schema/crd/vald-helm-operator ## generate OpenAPI v3 schema for ValdHelmOperatorRelease helm/schema/crd/vald-helm-operator: \ yq/install - @$(call gen-vald-crd,vald-helm-operator,valdhelmoperatorrelease,vald-helm-operator/values) + $(call gen-vald-crd,vald-helm-operator,valdhelmoperatorrelease,vald-helm-operator/values) .PHONY: helm/schema/crd/vald/mirror-target ## generate OpenAPI v3 schema for ValdMirrorTarget helm/schema/crd/vald/mirror-target: \ yq/install - @$(call gen-vald-crd,vald,valdmirrortarget,vald/schemas/mirror-target-values) + $(call gen-vald-crd,vald,valdmirrortarget,vald/schemas/mirror-target-values) .PHONY: helm/schema/crd/vald-benchmark-job ## generate OpenAPI v3 schema for ValdBenchmarkJobRelease helm/schema/crd/vald-benchmark-job: \ yq/install - @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkjob,vald-benchmark-operator/schemas/job-values) + $(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkjob,vald-benchmark-operator/schemas/job-values) .PHONY: helm/schema/crd/vald-benchmark-scenario ## generate OpenAPI v3 schema for ValdBenchmarkScenarioRelease helm/schema/crd/vald-benchmark-scenario: \ yq/install - @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkscenario,vald-benchmark-operator/schemas/scenario-values) + $(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkscenario,vald-benchmark-operator/schemas/scenario-values) .PHONY: helm/schema/crd/vald-benchmark-operator ## generate OpenAPI v3 schema for ValdBenchmarkOperatorRelease helm/schema/crd/vald-benchmark-operator: \ yq/install - @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkoperatorrelease,vald-benchmark-operator/values) + $(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkoperatorrelease,vald-benchmark-operator/values) diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index a522123f70b..2c4227756ba 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -95,7 +95,7 @@ k8s/vald/deploy: helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ - --set agent.image.repository=$(CRORG)/$(AGENT_IMAGE) \ + --set agent.image.repository=$(CRORG)/$(AGENT_NGT_IMAGE) \ --set agent.sidecar.image.repository=$(CRORG)/$(AGENT_SIDECAR_IMAGE) \ --set discoverer.image.repository=$(CRORG)/$(DISCOVERER_IMAGE) \ --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \ @@ -131,7 +131,7 @@ k8s/vald/delete: helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ - --set agent.image.repository=$(CRORG)/$(AGENT_IMAGE) \ + --set agent.image.repository=$(CRORG)/$(AGENT_NGT_IMAGE) \ --set agent.sidecar.image.repository=$(CRORG)/$(AGENT_SIDECAR_IMAGE) \ --set discoverer.image.repository=$(CRORG)/$(DISCOVERER_IMAGE) \ --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \ @@ -224,7 +224,7 @@ k8s/vald-readreplica/deploy: helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ - --set agent.image.repository=$(CRORG)/$(AGENT_IMAGE) \ + --set agent.image.repository=$(CRORG)/$(AGENT_NGT_IMAGE) \ --set agent.sidecar.image.repository=$(CRORG)/$(AGENT_SIDECAR_IMAGE) \ --set discoverer.image.repository=$(CRORG)/$(DISCOVERER_IMAGE) \ --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \ @@ -245,7 +245,7 @@ k8s/vald-readreplica/delete: helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ - --set agent.image.repository=$(CRORG)/$(AGENT_IMAGE) \ + --set agent.image.repository=$(CRORG)/$(AGENT_NGT_IMAGE) \ --set agent.sidecar.image.repository=$(CRORG)/$(AGENT_SIDECAR_IMAGE) \ --set discoverer.image.repository=$(CRORG)/$(DISCOVERER_IMAGE) \ --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \ @@ -516,27 +516,27 @@ $(BINDIR)/telepresence: .PHONY: telepresence/swap/agent-ngt ## swap agent-ngt deployment using telepresence telepresence/swap/agent-ngt: - @$(call telepresence,vald-agent-ngt,vdaas/vald-agent-ngt) + $(call telepresence,vald-agent-ngt,vdaas/vald-agent-ngt) .PHONY: telepresence/swap/agent-faiss ## swap agent-faiss deployment using telepresence telepresence/swap/agent-faiss: - @$(call telepresence,vald-agent-faiss,vdaas/vald-agent-faiss) + $(call telepresence,vald-agent-faiss,vdaas/vald-agent-faiss) .PHONY: telepresence/swap/discoverer ## swap discoverer deployment using telepresence telepresence/swap/discoverer: - @$(call telepresence,vald-discoverer,vdaas/vald-discoverer-k8s) + $(call telepresence,vald-discoverer,vdaas/vald-discoverer-k8s) .PHONY: telepresence/swap/manager-index ## swap manager-index deployment using telepresence telepresence/swap/manager-index: - @$(call telepresence,vald-manager-index,vdaas/vald-manager-index) + $(call telepresence,vald-manager-index,vdaas/vald-manager-index) .PHONY: telepresence/swap/lb-gateway ## swap lb-gateway deployment using telepresence telepresence/swap/lb-gateway: - @$(call telepresence,vald-lb-gateway,vdaas/vald-lb-gateway) + $(call telepresence,vald-lb-gateway,vdaas/vald-lb-gateway) .PHONY: kubelinter/install ## install kubelinter diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile index f037f4b94fb..3f8ea849496 100644 --- a/dockers/agent/core/faiss/Dockerfile +++ b/dockers/agent/core/faiss/Dockerfile @@ -37,6 +37,8 @@ ENV REPO vald ENV PKG agent/core/faiss ENV PKG_INTERNAL agent/internal ENV APP_NAME faiss +ENV AARCH aarch_64 +ENV XARCH x86_64 # skipcq: DOK-DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -50,6 +52,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ intel-mkl \ unzip \ upx \ + && if [ "${TARGETARCH}" = "amd64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${XARCH}-${TARGETOS}-gnu; \ + export CC=${XARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + elif [ "${TARGETARCH}" = "arm64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${AARCH}-${TARGETOS}-gnu; \ + export CC=${AARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + fi \ + && ldconfig \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index 8bdb671c5e0..b6141957074 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -37,6 +37,8 @@ ENV REPO vald ENV PKG agent/core/ngt ENV PKG_INTERNAL agent/internal ENV APP_NAME ngt +ENV AARCH aarch_64 +ENV XARCH x86_64 # skipcq: DOK-DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -52,6 +54,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libopenblas-dev \ unzip \ upx \ + && if [ "${TARGETARCH}" = "amd64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${XARCH}-${TARGETOS}-gnu; \ + export CC=${XARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + elif [ "${TARGETARCH}" = "arm64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${AARCH}-${TARGETOS}-gnu; \ + export CC=${AARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + fi \ + && ldconfig \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile index bc9ac58900a..4c707534a40 100644 --- a/dockers/operator/helm/Dockerfile +++ b/dockers/operator/helm/Dockerfile @@ -48,6 +48,8 @@ ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin ENV ORG vdaas ENV REPO vald ENV APP_NAME helm-operator +ENV AARCH aarch_64 +ENV XARCH x86_64 # skipcq: DOK-DL3008 RUN apt-get clean \ @@ -62,6 +64,16 @@ RUN apt-get clean \ curl \ upx \ git \ + && if [ "${TARGETARCH}" = "amd64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${XARCH}-${TARGETOS}-gnu; \ + export CC=${XARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + elif [ "${TARGETARCH}" = "arm64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${AARCH}-${TARGETOS}-gnu; \ + export CC=${AARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + fi \ + && ldconfig \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -121,11 +133,12 @@ COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/charts COPY charts . -WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/hack/helm/schema/gen -COPY hack/helm/schema/gen . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/hack +COPY hack . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald helm/schema/vald-helm-operator \ +RUN make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald \ + && make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald-helm-operator \ && cp -r charts /charts COPY --from=operator /usr/local/bin/${APP_NAME} /usr/bin/${APP_NAME} diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index 4edb54929f3..1ab1c492bae 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -45,6 +45,8 @@ ENV PKG tools/benchmark/${APP_NAME} ENV BUILD_DIR=/usr/local ENV LIB_DIR=/usr/local/lib ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${LIB_DIR}:/lib +ENV AARCH aarch_64 +ENV XARCH x86_64 # skipcq: DOK-DL3008, DOK-DL3003 RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -54,6 +56,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ upx \ g++ \ git \ + && if [ "${TARGETARCH}" = "amd64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${XARCH}-${TARGETOS}-gnu; \ + export CC=${XARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + elif [ "${TARGETARCH}" = "arm64" ] ; then \ + apt-get install -y --no-install-recommends --fix-missing gcc-${AARCH}-${TARGETOS}-gnu; \ + export CC=${AARCH}-${TARGETOS}-gnu-gcc; \ + export CC_FOR_TARGET=${CC}; \ + fi \ && ldconfig \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \