[BUGFIX] resolve agent GetGraphStatistics API double-free error problem

Signed-off-by: kpango <[email protected]>
vdaas · Nov 6, 2024 · b76192b · b76192b
1 parent a583f09
commit b76192b
Show file tree

Hide file tree

Showing 62 changed files with 893 additions and 575 deletions.
diff --git a/.gitfiles b/.gitfiles
@@ -89,6 +89,7 @@
 .github/workflows/dockers-ci-container-image.yaml
 .github/workflows/dockers-dev-container-image.yaml
 .github/workflows/dockers-discoverer-k8s-image.yaml
+.github/workflows/dockers-example-client-image.yaml
 .github/workflows/dockers-gateway-filter-image.yaml
 .github/workflows/dockers-gateway-lb-image.yaml
 .github/workflows/dockers-gateway-mirror-image.yaml
@@ -97,6 +98,7 @@
 .github/workflows/dockers-index-correction-image.yaml
 .github/workflows/dockers-index-creation-image.yaml
 .github/workflows/dockers-index-deletion-image.yaml
+.github/workflows/dockers-index-deletion.yaml
 .github/workflows/dockers-index-operator-image.yaml
 .github/workflows/dockers-index-save-image.yaml
 .github/workflows/dockers-loadtest-image.yaml
@@ -123,7 +125,7 @@
 .github/workflows/semver-patch.yaml
 .github/workflows/test-hack.yaml
 .github/workflows/unit-test.yaml
-.github/workflows/update-actions.yaml
+.github/workflows/update-deps.yaml
 .github/workflows/update-protobuf.yaml
 .github/workflows/update-pull-request-and-issue-template.yaml
 .github/workflows/update-web-docs.yaml
@@ -199,8 +201,6 @@ apis/grpc/v1/vald/update_vtproto.pb.go
 apis/grpc/v1/vald/upsert.pb.go
 apis/grpc/v1/vald/upsert_vtproto.pb.go
 apis/grpc/v1/vald/vald.go
-apis/proto/buf.lock
-apis/proto/buf.yaml
 apis/proto/v1/agent/core/agent.proto
 apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
@@ -304,7 +304,8 @@ assets/test/templates/option/function.tmpl
 assets/test/templates/option/header.tmpl
 assets/test/templates/option/results.tmpl
 buf.gen.yaml
-buf.work.yaml
+buf.lock
+buf.yaml
 charts/vald-benchmark-operator/Chart.yaml
 charts/vald-benchmark-operator/README.md
 charts/vald-benchmark-operator/README.md.gotmpl
@@ -490,6 +491,8 @@ cmd/index/job/correction/sample.yaml
 cmd/index/job/creation/main.go
 cmd/index/job/creation/main_test.go
 cmd/index/job/creation/sample.yaml
+cmd/index/job/deletion/main.go
+cmd/index/job/deletion/sample.yaml
 cmd/index/job/readreplica/rotate/main.go
 cmd/index/job/readreplica/rotate/main_test.go
 cmd/index/job/readreplica/rotate/sample.yaml
@@ -537,13 +540,15 @@ dockers/dev/Dockerfile
 dockers/dev/README.md
 dockers/discoverer/k8s/Dockerfile
 dockers/discoverer/k8s/README.md
+dockers/example/client/Dockerfile
 dockers/gateway/filter/Dockerfile
 dockers/gateway/filter/README.md
 dockers/gateway/lb/Dockerfile
 dockers/gateway/lb/README.md
 dockers/gateway/mirror/Dockerfile
 dockers/index/job/correction/Dockerfile
 dockers/index/job/creation/Dockerfile
+dockers/index/job/deletion/Dockerfile
 dockers/index/job/readreplica/rotate/Dockerfile
 dockers/index/job/save/Dockerfile
 dockers/index/operator/Dockerfile
@@ -838,6 +843,7 @@ internal/config/grpc_test.go
 internal/config/index.go
 internal/config/index_creation.go
 internal/config/index_creation_test.go
+internal/config/index_deleter.go
 internal/config/index_operator.go
 internal/config/index_operator_test.go
 internal/config/index_save.go
@@ -1455,6 +1461,8 @@ k8s/index/job/correction/configmap.yaml
 k8s/index/job/correction/cronjob.yaml
 k8s/index/job/creation/configmap.yaml
 k8s/index/job/creation/cronjob.yaml
+k8s/index/job/deletion/configmap.yaml
+k8s/index/job/deletion/cronjob.yaml
 k8s/index/job/save/configmap.yaml
 k8s/index/job/save/cronjob.yaml
 k8s/index/operator/configmap.yaml
@@ -1789,6 +1797,10 @@ pkg/index/job/creation/service/options.go
 pkg/index/job/creation/service/options_test.go
 pkg/index/job/creation/usecase/creation.go
 pkg/index/job/creation/usecase/creation_test.go
+pkg/index/job/deletion/config/config.go
+pkg/index/job/deletion/service/deleter.go
+pkg/index/job/deletion/service/options.go
+pkg/index/job/deletion/usecase/deletion.go
 pkg/index/job/readreplica/rotate/config/config.go
 pkg/index/job/readreplica/rotate/config/config_test.go
 pkg/index/job/readreplica/rotate/service/options.go

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -22,11 +22,11 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Vald Version: v1.7.13
+- Vald Version: v1.7.14
 - Go Version: v1.23.2
-- Rust Version: v1.81.0
+- Rust Version: v1.82.0
 - Docker Version: v27.3.1
-- Kubernetes Version: v1.31.1
+- Kubernetes Version: v1.31.2
 - Helm Version: v3.16.2
-- NGT Version: v2.2.4
+- NGT Version: v2.3.3
 - Faiss Version: v1.9.0
diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md
@@ -16,11 +16,11 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Vald Version: v1.7.13
+- Vald Version: v1.7.14
 - Go Version: v1.23.2
-- Rust Version: v1.81.0
+- Rust Version: v1.82.0
 - Docker Version: v27.3.1
-- Kubernetes Version: v1.31.1
+- Kubernetes Version: v1.31.2
 - Helm Version: v3.16.2
-- NGT Version: v2.2.4
+- NGT Version: v2.3.3
 - Faiss Version: v1.9.0
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -15,13 +15,13 @@
 ### Versions
 
 <!--- Please change the versions below along with your environment -->
-- Vald Version: v1.7.13
+- Vald Version: v1.7.14
 - Go Version: v1.23.2
-- Rust Version: v1.81.0
+- Rust Version: v1.82.0
 - Docker Version: v27.3.1
-- Kubernetes Version: v1.31.1
+- Kubernetes Version: v1.31.2
 - Helm Version: v3.16.2
-- NGT Version: v2.2.4
+- NGT Version: v2.3.3
 - Faiss Version: v1.9.0
 
 ### Checklist

diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile
@@ -93,4 +93,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/agent /usr/bin/agent
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]
diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile
@@ -95,4 +95,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss
 COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/faiss"]
+ENTRYPOINT ["/usr/bin/faiss"]
diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile
@@ -94,4 +94,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt
 COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/ngt"]
+ENTRYPOINT ["/usr/bin/ngt"]
diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile
@@ -84,4 +84,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/sidecar"]
+ENTRYPOINT ["/usr/bin/sidecar"]
diff --git a/dockers/binfmt/Dockerfile b/dockers/binfmt/Dockerfile
@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM tonistiigi/binfmt:master AS builder
+FROM tonistiigi/binfmt:master AS builder
diff --git a/dockers/buildbase/Dockerfile b/dockers/buildbase/Dockerfile
@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM ubuntu:devel AS builder
+FROM ubuntu:devel AS builder
diff --git a/dockers/buildkit/Dockerfile b/dockers/buildkit/Dockerfile
@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM moby/buildkit:master AS builder
+FROM moby/buildkit:master AS builder
diff --git a/dockers/buildkit/syft/scanner/Dockerfile b/dockers/buildkit/syft/scanner/Dockerfile
@@ -16,4 +16,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM docker/buildkit-syft-scanner:edge AS scanner
+FROM docker/buildkit-syft-scanner:edge AS scanner
diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile
@@ -127,4 +127,4 @@ RUN --mount=type=bind,target=.,rw \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
 USER root:root
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/bin/bash"]
diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile
@@ -145,4 +145,4 @@ RUN --mount=type=bind,target=.,rw \
     && make faiss/install \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
-USER root:root
+USER root:root
diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer
 COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/discoverer"]
+ENTRYPOINT ["/usr/bin/discoverer"]
diff --git a/dockers/example/client/Dockerfile b/dockers/example/client/Dockerfile
@@ -92,4 +92,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/client /usr/bin/client
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/client"]
+ENTRYPOINT ["/usr/bin/client"]
diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter
 COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/filter"]
+ENTRYPOINT ["/usr/bin/filter"]
diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb
 COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/lb"]
+ENTRYPOINT ["/usr/bin/lb"]
diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror
 COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/mirror"]
+ENTRYPOINT ["/usr/bin/mirror"]
diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction
 COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-correction"]
+ENTRYPOINT ["/usr/bin/index-correction"]
diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation
 COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-creation"]
+ENTRYPOINT ["/usr/bin/index-creation"]
diff --git a/dockers/index/job/deletion/Dockerfile b/dockers/index/job/deletion/Dockerfile
@@ -82,6 +82,7 @@ RUN --mount=type=bind,target=.,rw \
 FROM gcr.io/distroless/static:nonroot
 LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/index-deletion /usr/bin/index-deletion
+COPY cmd/index/job/deletion/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
 ENTRYPOINT ["/usr/bin/index-deletion"]
diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate
 COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/readreplica-rotate"]
+ENTRYPOINT ["/usr/bin/readreplica-rotate"]
diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save
 COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-save"]
+ENTRYPOINT ["/usr/bin/index-save"]
diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator
 COPY cmd/index/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-operator"]
+ENTRYPOINT ["/usr/bin/index-operator"]
diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index
 COPY cmd/manager/index/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index"]
+ENTRYPOINT ["/usr/bin/index"]
diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile
@@ -106,4 +106,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald
 COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]
+ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]
diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile
@@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job
 COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/job"]
+ENTRYPOINT ["/usr/bin/job"]
diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile
@@ -85,4 +85,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator
 COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/operator"]
+ENTRYPOINT ["/usr/bin/operator"]
diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile
@@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/loadtest /usr/bin/loadtest
 COPY cmd/tools/cli/loadtest/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/loadtest"]
+ENTRYPOINT ["/usr/bin/loadtest"]
diff --git a/example/client/go.mod b/example/client/go.mod
@@ -7,13 +7,13 @@ replace (
 	github.com/goccy/go-json => github.com/goccy/go-json v0.10.3
 	github.com/golang/protobuf => github.com/golang/protobuf v1.5.4
 	github.com/kpango/glg => github.com/kpango/glg v1.6.15
-	github.com/pkg/sftp => github.com/pkg/sftp v1.13.6
+	github.com/pkg/sftp => github.com/pkg/sftp v1.13.7
 	golang.org/x/crypto => golang.org/x/crypto v0.28.0
 	golang.org/x/net => golang.org/x/net v0.30.0
 	golang.org/x/text => golang.org/x/text v0.19.0
-	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9
-	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9
-	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9
+	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20241104194629-dd2ea8efbc28
+	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28
+	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28
 	google.golang.org/grpc => google.golang.org/grpc v1.67.1
 	google.golang.org/protobuf => google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0
@@ -23,20 +23,20 @@ replace (
 require (
 	github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1
 	github.com/kpango/glg v1.6.14
-	github.com/vdaas/vald-client-go v1.7.13
+	github.com/vdaas/vald-client-go v1.7.14
 	gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946
 	google.golang.org/grpc v1.67.1
 )
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240717164558-a6c49f84cc0f.2 // indirect
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.1-20240920164238-5a7b106cbb87.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/kpango/fastime v1.1.9 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
 )