diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index d1ceb3bd6a..95105e7fc2 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -206,3 +206,12 @@ define gen-go-option-test-sources fi; \ done endef + +define gen-vald-crd + mv charts/$1/crds/$2.yaml $(TEMP_DIR)/$2.yaml + GOPRIVATE=$(GOPRIVATE) \ + go run -mod=readonly hack/helm/schema/crd/main.go \ + charts/$1/$3.yaml > $(TEMP_DIR)/$2-spec.yaml + $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ + $(TEMP_DIR)/$2.yaml $(TEMP_DIR)/$2-spec.yaml > charts/$1/crds/$2.yaml +endef diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 65b14151ed..f532bf0c44 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -157,31 +157,16 @@ helm/schema/crd/vald-helm-operator: \ ## generate OpenAPI v3 schema for ValdBenchmarkJobRelease helm/schema/crd/vald-benchmark-job: \ yq/install - mv charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml $(TEMP_DIR)/valdbenchmarkjob.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald-benchmark-operator/schemas/job-values.yaml > $(TEMP_DIR)/valdbenchmarkjob-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdbenchmarkjob.yaml $(TEMP_DIR)/valdbenchmarkjob-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml + @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkjob,schemas/job-values) .PHONY: helm/schema/crd/vald-benchmark-scenario ## generate OpenAPI v3 schema for ValdBenchmarkScenarioRelease helm/schema/crd/vald-benchmark-scenario: \ yq/install - mv charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml $(TEMP_DIR)/valdbenchmarkscenario.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald-benchmark-operator/schemas/scenario-values.yaml > $(TEMP_DIR)/valdbenchmarkscenario-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdbenchmarkscenario.yaml $(TEMP_DIR)/valdbenchmarkscenario-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml + @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkscenario,schemas/scenario-values) .PHONY: helm/schema/crd/vald-benchmark-operator ## generate OpenAPI v3 schema for ValdBenchmarkOperatorRelease helm/schema/crd/vald-benchmark-operator: \ yq/install - mv charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml $(TEMP_DIR)/valdbenchmarkoperatorrelease.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald-benchmark-operator/values.yaml > $(TEMP_DIR)/valdbenchmarkoperatorrelease-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdbenchmarkoperatorrelease.yaml $(TEMP_DIR)/valdbenchmarkoperatorrelease-spec.yaml > charts/vald-benchmark-operator/crds/valdbenchmarkoperatorrelease.yaml + @$(call gen-vald-crd,vald-benchmark-operator,valdbenchmarkoperatorrelease,values) diff --git a/cmd/tools/benchmark/operator/sample.yaml b/cmd/tools/benchmark/operator/sample.yaml index e69de29bb2..adf6dd16b3 100644 --- a/cmd/tools/benchmark/operator/sample.yaml +++ b/cmd/tools/benchmark/operator/sample.yaml @@ -0,0 +1,175 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: debug + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + probe_wait_time: 3s + socket_path: "" + mode: GRPC + grpc: + bidirectional_stream_concurrency: 20 + max_receive_message_size: 0 + max_send_message_size: 0 + initial_window_size: 1048576 + initial_conn_window_size: 2097152 + keepalive: + max_conn_idle: "" + max_conn_age: "" + max_conn_age_grace: "" + time: "3h" + timeout: "60s" + min_time: "10m" + permit_without_stream: true + write_buffer_size: 0 + read_buffer_size: 0 + connection_timeout: "" + max_header_list_size: 0 + header_table_size: 0 + interceptors: + - "RecoverInterceptor" + enable_reflection: true + socket_option: + reuse_port: true + reuse_addr: true + tcp_fast_open: false + tcp_no_delay: false + tcp_cork: false + tcp_quick_ack: false + tcp_defer_accept: false + ip_transparent: false + ip_recover_destination_addr: false + restart: true + health_check_servers: + - name: liveness + host: 0.0.0.0 + port: 3000 + mode: "" + probe_wait_time: "3s" + network: tcp + socket_path: "" + http: + shutdown_duration: "5s" + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + write_timeout: "" + socket_option: + reuse_port: true + reuse_addr: true + tcp_fast_open: true + tcp_no_delay: true + tcp_cork: false + tcp_quick_ack: true + tcp_defer_accept: false + ip_transparent: false + ip_recover_destination_addr: false + - name: readiness + host: 0.0.0.0 + port: 3001 + mode: "" + probe_wait_time: "3s" + network: tcp + socket_path: "" + http: + shutdown_duration: "0s" + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + write_timeout: "" + socket_option: + reuse_port: true + reuse_addr: true + tcp_fast_open: true + tcp_no_delay: true + tcp_cork: false + tcp_quick_ack: true + tcp_defer_accept: false + ip_transparent: false + ip_recover_destination_addr: false + metrics_servers: + - name: pprof + host: 0.0.0.0 + port: 8081 + probe_wait_time: "3s" + socket_path: "" + mode: REST + network: tcp + http: + handler_timeout: "5s" + idle_timeout: "2s" + read_header_timeout: "1s" + read_timeout: "1s" + shutdown_duration: "5s" + write_timeout: "1m" + socket_option: + reuse_port: true + reuse_addr: true + tcp_fast_open: false + tcp_no_delay: false + tcp_cork: false + tcp_quick_ack: false + tcp_defer_accept: false + ip_transparent: false + ip_recover_destination_addr: false + startup_strategy: + - liveness + - readiness + - grpc + full_shutdown_duration: 30s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +observability: + enabled: false + otlp: + collector_endpoint: "" + attribute: + namespace: _MY_POD_NAMESPACE_ + pod_name: _MY_POD_NAME_ + node_name: _MY_NODE_NAME_ + service_name: vald-benchmark + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - ngt_version + trace: + enabled: false diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index a806553afb..ddea7f215d 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -86,54 +86,30 @@ RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} RUN cp sample.yaml /tmp/config.yaml -FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} + +# FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +FROM ubuntu:devel LABEL maintainer="${MAINTAINER}" + ENV APP_NAME job -COPY --from=builder /usr/lib/x86_64-linux-gnu/libaec* /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libhdf5* /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libsz* /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libhdf5_serial.so /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so /usr/lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libz.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libcrypto.so.3 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libcurl.so.4 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libnghttp2.so.14 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libidn2.so.0 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/librtmp.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libssh.so.4 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libpsl.so.5 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libssl.so.3 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libgssapi_krb5.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libldap.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/liblber.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libzstd.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libbrotlidec.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libunistring.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libgnutls.so.30 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libhogweed.so.6 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libnettle.so.8 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libgmp.so.10 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libkrb5.so.3 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libk5crypto.so.3 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libkrb5support.so.0 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libcom_err.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libsasl2.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libbrotlicommon.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libp11-kit.so.0 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libtasn1.so.6 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libkeyutils.so.1 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libresolv.so.2 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib/x86_64-linux-gnu/libffi.so.8 /lib/x86_64-linux-gnu/ -COPY --from=builder /lib64/ld-linux-x86-64.so.2 /lib64/ +RUN apt-get update && apt-get install -y --no-install-recommends \ + libhdf5-dev \ + && ldconfig \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy certificates for SSL/TLS +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# Copy permissions +COPY --from=builder /etc/passwd /etc/passwd +# workaround: https://github.com/moby/moby/issues/37965 +RUN true COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} COPY --from=builder /tmp/config.yaml /etc/server/config.yaml -USER nonroot:nonroot +# For distroless +# USER nonroot:nonroot ENTRYPOINT ["/go/bin/job"] diff --git a/internal/k8s/client/client.go b/internal/k8s/client/client.go index e595294efa..c0a86a341e 100644 --- a/internal/k8s/client/client.go +++ b/internal/k8s/client/client.go @@ -54,10 +54,6 @@ const ( SelectionOpEquals = selection.Equals ) -const ( - DeletePropagationBackground = metav1.DeletePropagationBackground -) - type Client interface { // Get retrieves an obj for the given object key from the Kubernetes Cluster. // obj must be a struct pointer so that obj can be updated with the response diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go index 299b114172..342bd32c9f 100644 --- a/pkg/tools/benchmark/job/service/job.go +++ b/pkg/tools/benchmark/job/service/job.go @@ -258,6 +258,7 @@ func (j *job) Start(ctx context.Context) (<-chan error, error) { return nil, err } j.eg.Go(func() error { + defer close(ech) for { select { case <-ctx.Done(): @@ -281,7 +282,6 @@ func (j *job) Start(ctx context.Context) (<-chan error, error) { case ech <- err: } } - close(ech) if err := p.Signal(syscall.SIGTERM); err != nil { log.Error(err) } diff --git a/pkg/tools/benchmark/job/service/search.go b/pkg/tools/benchmark/job/service/search.go index f35b5775f1..2837763ccf 100644 --- a/pkg/tools/benchmark/job/service/search.go +++ b/pkg/tools/benchmark/job/service/search.go @@ -85,13 +85,14 @@ func (j *job) search(ctx context.Context, ech chan error) error { log.Errorf("[benchmark job] err: %s", err.Error()) } } - if res != nil && j.searchConfig.EnableLinearSearch { - sres[iter-j.dataset.Range.Start] = res + if res != nil { + if j.searchConfig.EnableLinearSearch { + sres[iter-j.dataset.Range.Start] = res + } log.Debugf("[benchmark job] Finish search: iter = %d, len = %d", iter, len(res.Results)) } else { log.Debugf("[benchmark job] Finish search: iter = %d, res = %v", iter, res) } - log.Debugf("[benchmark job] Finish search: iter = %d, len = %d", iter, len(res.Results)) return nil }) } diff --git a/pkg/tools/benchmark/operator/service/operator.go b/pkg/tools/benchmark/operator/service/operator.go index 13ee6ac498..6b8b7f8de0 100644 --- a/pkg/tools/benchmark/operator/service/operator.go +++ b/pkg/tools/benchmark/operator/service/operator.go @@ -173,16 +173,16 @@ func (o *operator) getAtomicJob() map[string]string { // Then, it processes according STATUS. func (o *operator) jobReconcile(ctx context.Context, jobList map[string][]job.Job) { log.Debug("[reconcile job] start") + cjobs := o.getAtomicJob() + if cjobs == nil { + cjobs = map[string]string{} + } if len(jobList) == 0 { log.Info("[reconcile job] no job is founded") - o.jobs.Store(&map[string]string{}) + o.jobs.Store(&(map[string]string{})) log.Debug("[reconcile job] finish") return } - cjobs := o.getAtomicJob() - if cjobs == nil { - cjobs = map[string]string{} - } // benchmarkJobStatus is used for update benchmark job resource status benchmarkJobStatus := make(map[string]v1.BenchmarkJobStatus) // jobNames is used for check whether cjobs has delted job. @@ -230,16 +230,16 @@ func (o *operator) jobReconcile(ctx context.Context, jobList map[string][]job.Jo // benchJobReconcile gets the vald benchmark job resource list and create Job for running benchmark job. func (o *operator) benchJobReconcile(ctx context.Context, benchJobList map[string]v1.ValdBenchmarkJob) { log.Debugf("[reconcile benchmark job resource] job list: %#v", benchJobList) + cbjl := o.getAtomicBenchJob() + if cbjl == nil { + cbjl = make(map[string]*v1.ValdBenchmarkJob, 0) + } if len(benchJobList) == 0 { log.Info("[reconcile benchmark job resource] job resource not found") o.benchjobs.Store(&(map[string]*v1.ValdBenchmarkJob{})) log.Debug("[reconcile benchmark job resource] finish") return } - cbjl := o.getAtomicBenchJob() - if cbjl == nil { - cbjl = make(map[string]*v1.ValdBenchmarkJob, 0) - } // jobStatus is used for update benchmarkJob CR status if updating is needed. jobStatus := make(map[string]v1.BenchmarkJobStatus) for k := range benchJobList { @@ -306,16 +306,16 @@ func (o *operator) benchJobReconcile(ctx context.Context, benchJobList map[strin // benchScenarioReconcile gets the vald benchmark scenario list and create vald benchmark job resource according to it. func (o *operator) benchScenarioReconcile(ctx context.Context, scenarioList map[string]v1.ValdBenchmarkScenario) { log.Debugf("[reconcile benchmark scenario resource] scenario list: %#v", scenarioList) + cbsl := o.getAtomicScenario() + if cbsl == nil { + cbsl = map[string]*scenario{} + } if len(scenarioList) == 0 { log.Info("[reconcile benchmark scenario resource]: scenario not found") o.scenarios.Store(&(map[string]*scenario{})) log.Debug("[reconcile benchmark scenario resource] finish") return } - cbsl := o.getAtomicScenario() - if cbsl == nil { - cbsl = map[string]*scenario{} - } scenarioStatus := make(map[string]v1.ValdBenchmarkScenarioStatus) for name := range scenarioList { sc := scenarioList[name]