impl faiss agent

Makefile

@@ -38,6 +38,7 @@ VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
 
 NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION)
 NGT_REPO = github.com/yahoojapan/NGT
+FAISS_VERSION := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION)
 
 GOPROXY=direct
 GO_VERSION := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION)
@@ -508,6 +509,18 @@ ngt/install: /usr/local/include/NGT/Capi.h
 	rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	ldconfig
 
+.PHONY: faiss/install
+## install Faiss
+faiss/install: /usr/local/lib/libfaiss.so
+/usr/local/lib/libfaiss.so:
+	curl -LO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz
+	tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
+	cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \
+		cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -B build . && \
+		make -C build -j faiss && \
+		make -C build install
+	ldconfig
+
 .PHONY: lint
 ## run lints
 lint: vet

Makefile.d/build.mk

@@ -61,6 +61,42 @@ cmd/agent/core/ngt/ngt: \
 		$(dir $@)main.go
 	$@ -version
 
+cmd/agent/core/faiss/faiss: \
+	faiss/install \
+	$(GO_SOURCES_INTERNAL) \
+	$(PBGOS) \
+	$(shell find ./cmd/agent/core/faiss -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find ./pkg/agent/core/faiss ./pkg/agent/core/ngt/service/kvs ./pkg/agent/core/ngt/service/vqueue ./pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
+	CFLAGS="$(CFLAGS)" \
+	CXXFLAGS="$(CXXFLAGS)" \
+	CGO_ENABLED=1 \
+	CGO_CXXFLAGS="-g -Ofast -march=native" \
+	CGO_FFLAGS="-g -Ofast -march=native" \
+	CGO_LDFLAGS="-g -Ofast -march=native" \
+	GO111MODULE=on \
+	GOPRIVATE=$(GOPRIVATE) \
+	go build \
+		--ldflags "-w -linkmode 'external' \
+		-extldflags '-fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS)' \
+		-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
+		-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
+		-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
+		-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
+		-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
+		-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
+		-X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \
+		-X '$(GOPKG)/internal/info.FaissVersion=$(FAISS_VERSION)' \
+		-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
+		-buildid=" \
+		-mod=readonly \
+		-modcacherw \
+		-a \
+		-tags "cgo osusergo netgo static_build" \
+		-trimpath \
+		-o $@ \
+		$(dir $@)main.go
+	$@ -version
+
 cmd/agent/sidecar/sidecar: \
 	$(GO_SOURCES_INTERNAL) \
 	$(PBGOS) \

Makefile.d/docker.mk

@@ -52,6 +52,18 @@ docker/build/agent-ngt:
 	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \
 	    --build-arg MAINTAINER=$(MAINTAINER)
 
+.PHONY: docker/build/agent-faiss
+## build agent-faiss image
+docker/build/agent-faiss:
+	$(DOCKER) build \
+	    $(DOCKER_OPTS) \
+	    -f dockers/agent/core/faiss/Dockerfile \
+	    -t $(ORG)/vald-agent-faiss:$(TAG) . \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
+	    --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \
+	    --build-arg MAINTAINER=$(MAINTAINER)
+
 .PHONY: docker/name/agent-sidecar
 docker/name/agent-sidecar:
 	@echo "$(ORG)/$(AGENT_SIDECAR_IMAGE)"

Makefile.d/e2e.mk

@@ -19,6 +19,16 @@
 e2e:
 	$(call run-e2e-crud-test,-run TestE2EStandardCRUD)
 
+.PHONY: e2e/faiss
+## run e2e/faiss
+e2e/faiss:
+	#$(call run-e2e-crud-faiss-test,-run TestE2EInsertOnly)
+	#$(call run-e2e-crud-faiss-test,-run TestE2ESearchOnly)
+	#$(call run-e2e-crud-faiss-test,-run TestE2EUpdateOnly)
+	#$(call run-e2e-crud-faiss-test,-run TestE2ERemoveOnly)
+	#$(call run-e2e-crud-faiss-test,-run TestE2EInsertAndSearch)
+	$(call run-e2e-crud-faiss-test,-run TestE2EStandardCRUD)
+
 .PHONY: e2e/multi
 ## run e2e multiple apis
 e2e/multi:

Makefile.d/functions.mk

@@ -113,6 +113,28 @@ define run-e2e-crud-test
 	    -namespace=$(E2E_TARGET_NAMESPACE)
 endef
 
+define run-e2e-crud-faiss-test
+	go test \
+	    -race \
+	    -mod=readonly \
+	    $1 \
+	    -v $(ROOTDIR)/tests/e2e/crud/crud_faiss_test.go \
+	    -tags "e2e" \
+	    -timeout $(E2E_TIMEOUT) \
+	    -host=$(E2E_BIND_HOST) \
+	    -port=$(E2E_BIND_PORT) \
+	    -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME).hdf5 \
+	    -insert-num=$(E2E_INSERT_COUNT) \
+	    -search-num=$(E2E_SEARCH_COUNT) \
+	    -update-num=$(E2E_UPDATE_COUNT) \
+	    -remove-num=$(E2E_REMOVE_COUNT) \
+	    -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
+	    -portforward=$(E2E_PORTFORWARD_ENABLED) \
+	    -portforward-pod-name=$(E2E_TARGET_POD_NAME) \
+	    -portforward-pod-port=$(E2E_TARGET_PORT) \
+	    -namespace=$(E2E_TARGET_NAMESPACE)
+endef
+
 define run-e2e-multi-crud-test
 	go test \
 	    -race \

cmd/agent/core/faiss/main.go

@@ -0,0 +1,59 @@
+//
+// Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package main provides program main
+package main
+
+import (
+	"context"
+
+	"github.com/vdaas/vald/internal/errors"
+	"github.com/vdaas/vald/internal/info"
+	"github.com/vdaas/vald/internal/log"
+	"github.com/vdaas/vald/internal/runner"
+	"github.com/vdaas/vald/internal/safety"
+	"github.com/vdaas/vald/pkg/agent/core/faiss/config"
+	"github.com/vdaas/vald/pkg/agent/core/faiss/usecase"
+)
+
+const (
+	maxVersion = "v0.0.10"
+	minVersion = "v0.0.0"
+	name       = "agent faiss"
+)
+
+func main() {
+	if err := safety.RecoverFunc(func() error {
+		return runner.Do(
+			context.Background(),
+			runner.WithName(name),
+			runner.WithVersion(info.Version, maxVersion, minVersion),
+			runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) {
+				cfg, err := config.NewConfig(path)
+				if err != nil {
+					return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration")
+				}
+				return cfg, &cfg.GlobalConfig, nil
+			}),
+			runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) {
+				return usecase.New(cfg.(*config.Data))
+			}),
+		)
+	})(); err != nil {
+		log.Fatal(err, info.Get())
+		return
+	}
+}

cmd/agent/core/faiss/sample.yaml

@@ -0,0 +1,123 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+version: v0.0.0
+time_zone: JST
+logging:
+  format: raw
+  level: debug
+  logger: glg
+server_config:
+  servers:
+    - name: grpc
+      host: 0.0.0.0
+      port: 8081
+      grpc:
+        bidirectional_stream_concurrency: 20
+        connection_timeout: ""
+        header_table_size: 0
+        initial_conn_window_size: 0
+        initial_window_size: 0
+        interceptors: []
+        keepalive:
+          max_conn_age: ""
+          max_conn_age_grace: ""
+          max_conn_idle: ""
+          time: ""
+          timeout: ""
+        max_header_list_size: 0
+        max_receive_message_size: 0
+        max_send_message_size: 0
+        read_buffer_size: 0
+        write_buffer_size: 0
+      mode: GRPC
+      probe_wait_time: 3s
+      restart: true
+  health_check_servers:
+    - name: readiness
+      host: 0.0.0.0
+      port: 3001
+      http:
+        handler_timeout: ""
+        idle_timeout: ""
+        read_header_timeout: ""
+        read_timeout: ""
+        shutdown_duration: 0s
+        write_timeout: ""
+      mode: ""
+      probe_wait_time: 3s
+  metrics_servers:
+  startup_strategy:
+    - grpc
+    - readiness
+  full_shutdown_duration: 600s
+  tls:
+    ca: /path/to/ca
+    cert: /path/to/cert
+    enabled: false
+    key: /path/to/key
+observability:
+  enabled: false
+  collector:
+    duration: 5s
+    metrics:
+      enable_cgo: true
+      enable_goroutine: true
+      enable_memory: true
+      enable_version_info: true
+      version_info_labels:
+        - vald_version
+        - server_name
+        - git_commit
+        - build_time
+        - go_version
+        - go_os
+        - go_arch
+        - faiss_version
+  trace:
+    enabled: false
+    sampling_rate: 1
+  prometheus:
+    enabled: false
+    endpoint: /metrics
+    namespace: vald
+  jaeger:
+    enabled: false
+    collector_endpoint: ""
+    agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831"
+    username: ""
+    password: ""
+    service_name: "vald-agent-faiss"
+    buffer_max_count: 10
+faiss:
+  auto_index_check_duration: 30m
+  auto_index_duration_limit: 24h
+  auto_index_length: 100
+  auto_save_index_duration: 35m
+  dimension: 64
+  enable_copy_on_write: false
+  enable_in_memory_mode: true
+  enable_proactive_gc: true
+  index_path: ""
+  initial_delay_max_duration: 3m
+  load_index_timeout_factor: 1ms
+  m: 8 # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid
+  max_load_index_timeout: 10m
+  metric_type: "inner_product"
+  min_load_index_timeout: 3m
+  nbits_per_idx: 8
+  nlist: 100