[WIP] feat: Implement index importer

.github/workflows/dockers-index-importation-image.yaml

@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
+
+
+name: 'Build docker image: index-importation'
+on:
+  push:
+    branches:
+    - main
+    - release/v*.*
+    - '!release/v*.*.*'
+    tags:
+    - '*.*.*'
+    - '*.*.*-*'
+    - v*.*.*
+    - v*.*.*-*
+  pull_request:
+    paths:
+    - '!**/*_mock.go'
+    - '!**/*_test.go'
+    - .github/actions/docker-build/action.yaml
+    - .github/workflows/_docker-image.yaml
+    - .github/workflows/dockers-index-importation-image.yaml
+    - Makefile
+    - Makefile.d/**
+    - apis/proto/**
+    - cmd/index/job/importation/*.go
+    - dockers/index/job/importation/Dockerfile
+    - go.mod
+    - go.sum
+    - hack/docker/gen/main.go
+    - versions/GO_VERSION
+  pull_request_target:
+    paths:
+    - '!**/*_mock.go'
+    - '!**/*_test.go'
+    - .github/actions/docker-build/action.yaml
+    - .github/workflows/_docker-image.yaml
+    - .github/workflows/dockers-index-importation-image.yaml
+    - Makefile
+    - Makefile.d/**
+    - apis/proto/**
+    - cmd/index/job/importation/*.go
+    - dockers/index/job/importation/Dockerfile
+    - go.mod
+    - go.sum
+    - hack/docker/gen/main.go
+    - versions/GO_VERSION
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: index-importation
+    secrets: inherit

Makefile

@@ -42,6 +42,7 @@ HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
 INDEX_CORRECTION_IMAGE          = $(NAME)-index-correction
 INDEX_CREATION_IMAGE            = $(NAME)-index-creation
 INDEX_DELETION_IMAGE            = $(NAME)-index-deletion
+INDEX_IMPORTATION_IMAGE         = $(NAME)-index-importation
 INDEX_OPERATOR_IMAGE            = $(NAME)-index-operator
 INDEX_SAVE_IMAGE                = $(NAME)-index-save
 LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway

Makefile.d/build.mk

@@ -25,6 +25,7 @@ binary/build: \
 	cmd/index/job/correction/index-correction \
 	cmd/index/job/creation/index-creation \
 	cmd/index/job/deletion/index-deletion \
+	cmd/index/job/importation/index-importation \
 	cmd/index/job/readreplica/rotate/readreplica-rotate \
 	cmd/index/job/save/index-save \
 	cmd/index/operator/index-operator \
@@ -85,6 +86,10 @@ cmd/index/job/deletion/index-deletion:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,index/job/deletion,,-static,,,$@)
 
+cmd/index/job/importation/index-importation:
+	$(eval CGO_ENABLED = 0)
+	$(call go-build,index/job/importation,,-static,,,$@)
+
 cmd/index/job/save/index-save:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,index/job/save,,-static,,,$@)
@@ -134,6 +139,7 @@ binary/build/zip: \
 	artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip \
+	artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \
@@ -197,6 +203,10 @@ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip: cmd/index/job/deletion/inde
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<
 
+artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip: cmd/index/job/importation/index-importation
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<
+
 artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<

Makefile.d/docker.mk

@@ -37,6 +37,7 @@ docker/build: \
 	docker/build/index-correction \
 	docker/build/index-creation \
 	docker/build/index-deletion \
+	docker/build/index-importation \
 	docker/build/index-operator \
 	docker/build/index-save \
 	docker/build/loadtest \
@@ -65,6 +66,7 @@ docker/xpanes/build:
 		docker/build/index-correction \
 		docker/build/index-creation \
 		docker/build/index-deletion \
+		docker/build/index-importation \
 		docker/build/index-operator \
 		docker/build/index-save \
 		docker/build/loadtest \
@@ -354,6 +356,13 @@ docker/build/index-deletion:
 		IMAGE=$(INDEX_DELETION_IMAGE) \
 		docker/build/image
 
+.PHONY: docker/build/index-importation
+## build index-importation image
+docker/build/index-importation:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/index/job/importation/Dockerfile" \
+		IMAGE=$(INDEX_IMPORTATION_IMAGE) \
+		docker/build/image
+
 .PHONY: docker/name/index-operator
 docker/name/index-operator:
 	@echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)"

cmd/index/job/importation/main.go

@@ -0,0 +1,60 @@
+// Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/vdaas/vald/internal/errors"
+	"github.com/vdaas/vald/internal/info"
+	"github.com/vdaas/vald/internal/runner"
+	"github.com/vdaas/vald/internal/safety"
+	"github.com/vdaas/vald/pkg/index/job/importation/config"
+	"github.com/vdaas/vald/pkg/index/job/importation/usecase"
+)
+
+const (
+	maxVersion = "v0.0.10"
+	minVersion = "v0.0.0"
+	name       = "index importation job"
+)
+
+func main() {
+	if err := safety.RecoverFunc(func() error {
+		return runner.Do(
+			context.Background(),
+			runner.WithName(name),
+			runner.WithVersion(info.Version, maxVersion, minVersion),
+			runner.WithConfigLoader(func(path string) (any, *config.GlobalConfig, error) {
+				// cfg, err := config.NewConfig(path)
+				cfg, err := config.NewConfig("cmd/index/job/importation/sample.yaml")
+				if err != nil {
+					return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration")
+				}
+				return cfg, &cfg.GlobalConfig, nil
+			}),
+			runner.WithDaemonInitializer(func(cfg any) (runner.Runner, error) {
+				c, ok := cfg.(*config.Data)
+				if !ok {
+					return nil, errors.ErrInvalidConfig
+				}
+				return usecase.New(c)
+			}),
+		)
+	})(); err != nil {
+		log.Fatal(err, info.Get())
+		return
+	}
+}

cmd/index/job/importation/sample.yaml

@@ -0,0 +1,191 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+version: v0.0.0
+time_zone: JST
+logging:
+  format: raw
+  level: info
+  logger: glg
+server_config:
+  servers:
+    - name: grpc
+      host: 0.0.0.0
+      port: 8081
+      grpc:
+        bidirectional_stream_concurrency: 20
+        connection_timeout: ""
+        header_table_size: 0
+        initial_conn_window_size: 0
+        initial_window_size: 0
+        interceptors: []
+        keepalive:
+          max_conn_age: ""
+          max_conn_age_grace: ""
+          max_conn_idle: ""
+          time: ""
+          timeout: ""
+        max_header_list_size: 0
+        max_receive_message_size: 0
+        max_send_message_size: 0
+        read_buffer_size: 0
+        write_buffer_size: 0
+      mode: GRPC
+      probe_wait_time: 3s
+      restart: true
+  health_check_servers:
+    - name: readiness
+      host: 0.0.0.0
+      port: 3001
+      http:
+        handler_timeout: ""
+        idle_timeout: ""
+        read_header_timeout: ""
+        read_timeout: ""
+        shutdown_duration: 0s
+        write_timeout: ""
+      mode: ""
+      probe_wait_time: 3s
+  metrics_servers:
+  startup_strategy:
+    - grpc
+    - readiness
+  full_shutdown_duration: 600s
+  tls:
+    ca: /path/to/ca
+    cert: /path/to/cert
+    enabled: false
+    key: /path/to/key
+importer:
+  concurrency: 1
+  index_path: "/var/export/index/1733964783.db"
+  gateway:
+    addrs:
+      - localhost:20000
+    health_check_duration: "1s"
+    connection_pool:
+      enable_dns_resolver: true
+      enable_rebalance: true
+      old_conn_close_duration: 2m
+      rebalance_duration: 30m
+      size: 3
+    backoff:
+      backoff_factor: 1.1
+      backoff_time_limit: 5s
+      enable_error_log: true
+      initial_duration: 5ms
+      jitter_limit: 100ms
+      maximum_duration: 5s
+      retry_count: 100
+    circuit_breaker:
+      closed_error_rate: 0.7
+      closed_refresh_timeout: 10s
+      half_open_error_rate: 0.5
+      min_samples: 1000
+      open_timeout: 1s
+    call_option:
+      content_subtype: ""
+      max_recv_msg_size: 0
+      max_retry_rpc_buffer_size: 0
+      max_send_msg_size: 0
+      wait_for_ready: true
+    dial_option:
+      authority: ""
+      backoff_base_delay: 1s
+      backoff_jitter: 0.2
+      backoff_max_delay: 120s
+      backoff_multiplier: 1.6
+      disable_retry: false
+      enable_backoff: false
+      idle_timeout: 1h
+      initial_connection_window_size: 2097152
+      initial_window_size: 1048576
+      insecure: true
+      interceptors: []
+      keepalive:
+        permit_without_stream: false
+        time: ""
+        timeout: 30s
+      max_call_attempts: 0
+      max_header_list_size: 0
+      max_msg_size: 0
+      min_connection_timeout: 20s
+      net:
+        dialer:
+          dual_stack_enabled: true
+          keepalive: ""
+          timeout: ""
+        dns:
+          cache_enabled: true
+          cache_expiration: 1h
+          refresh_duration: 30m
+        socket_option:
+          ip_recover_destination_addr: false
+          ip_transparent: false
+          reuse_addr: true
+          reuse_port: true
+          tcp_cork: false
+          tcp_defer_accept: false
+          tcp_fast_open: false
+          tcp_no_delay: false
+          tcp_quick_ack: false
+        tls:
+          ca: /path/to/ca
+          cert: /path/to/cert
+          enabled: false
+          insecure_skip_verify: false
+          key: /path/to/key
+      read_buffer_size: 0
+      shared_write_buffer: false
+      timeout: ""
+      user_agent: Vald-gRPC
+      write_buffer_size: 0
+    tls:
+      ca: /path/to/ca
+      cert: /path/to/cert
+      enabled: false
+      insecure_skip_verify: false
+      key: /path/to/key
+observability:
+  enabled: false
+  otlp:
+    collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317"
+    trace_batch_timeout: "1s"
+    trace_export_timeout: "1m"
+    trace_max_export_batch_size: 1024
+    trace_max_queue_size: 256
+    metrics_export_interval: "1s"
+    metrics_export_timeout: "1m"
+    attribute:
+      namespace: "_MY_POD_NAMESPACE_"
+      pod_name: "_MY_POD_NAME_"
+      node_name: "_MY_NODE_NAME_"
+      service_name: "vald-index-deletion"
+  metrics:
+    enable_cgo: true
+    enable_goroutine: true
+    enable_memory: true
+    enable_version_info: true
+    version_info_labels:
+      - vald_version
+      - server_name
+      - git_commit
+      - build_time
+      - go_version
+      - go_os
+      - go_arch
+      - algorithm_info
+  trace:
+    enabled: true