From c14564bb988472ce31853cc1da24507d3eb848a4 Mon Sep 17 00:00:00 2001 From: ykadowak Date: Thu, 30 Nov 2023 07:49:38 +0000 Subject: [PATCH] Revert "Add verification for index correction e2e and add clusterrole cronjobs for operator to deploy index correction (#2205)" This reverts commit b4028e780af500909738d57b2fa18611a3c55892. --- .github/helm/values/values-lb.yaml | 1 - .github/valdrelease/valdrelease.yaml | 4 - .../templates/clusterrole.yaml | 13 - charts/vald/values.yaml | 2 +- k8s/operator/helm/clusterrole.yaml | 12 - k8s/operator/helm/crds/valdrelease.yaml | 965 ------------------ tests/e2e/crud/crud_test.go | 58 +- tests/e2e/operation/job.go | 4 +- tests/e2e/operation/stream.go | 2 - 9 files changed, 11 insertions(+), 1050 deletions(-) diff --git a/.github/helm/values/values-lb.yaml b/.github/helm/values/values-lb.yaml index 7075b90708..bc0043b6a9 100644 --- a/.github/helm/values/values-lb.yaml +++ b/.github/helm/values/values-lb.yaml @@ -74,4 +74,3 @@ manager: # suspend because you do not want corrector to start automatically in CI # instead run it manually suspend: true - schedule: "1 2 3 4 5" diff --git a/.github/valdrelease/valdrelease.yaml b/.github/valdrelease/valdrelease.yaml index e49752f4b4..d2df3e28d1 100644 --- a/.github/valdrelease/valdrelease.yaml +++ b/.github/valdrelease/valdrelease.yaml @@ -76,7 +76,3 @@ spec: auto_index_duration_limit: 2m auto_index_check_duration: 30s auto_index_length: 1000 - corrector: - enabled: true - suspend: true - schedule: "1 2 3 4 5" diff --git a/charts/vald-helm-operator/templates/clusterrole.yaml b/charts/vald-helm-operator/templates/clusterrole.yaml index b027ea16a4..c0403124b5 100644 --- a/charts/vald-helm-operator/templates/clusterrole.yaml +++ b/charts/vald-helm-operator/templates/clusterrole.yaml @@ -175,17 +175,4 @@ rules: - get - patch - update - - apiGroups: - - batch - resources: - - cronjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - {{- end }} diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 46035e6197..f095a921bf 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -2682,7 +2682,7 @@ manager: enabled: false # @schema {"name": "manager.index.corrector.schedule", "type": "string"} # manager.index.corrector.schedule -- CronJob schedule setting for index correction - schedule: "6 3 * * *" + schedule: "5 * * * *" # @schema {"name": "manager.index.corrector.suspend", "type": "boolean"} # manager.index.corrector.suspend -- CronJob suspend setting for index correction suspend: false diff --git a/k8s/operator/helm/clusterrole.yaml b/k8s/operator/helm/clusterrole.yaml index ab151aaa8f..5137df5e71 100644 --- a/k8s/operator/helm/clusterrole.yaml +++ b/k8s/operator/helm/clusterrole.yaml @@ -175,15 +175,3 @@ rules: - get - patch - update - - apiGroups: - - batch - resources: - - cronjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/k8s/operator/helm/crds/valdrelease.yaml b/k8s/operator/helm/crds/valdrelease.yaml index 3854881577..46c151ab94 100644 --- a/k8s/operator/helm/crds/valdrelease.yaml +++ b/k8s/operator/helm/crds/valdrelease.yaml @@ -6068,971 +6068,6 @@ spec: annotations: type: object x-kubernetes-preserve-unknown-fields: true - corrector: - type: object - properties: - agent_namespace: - type: string - bbolt_async_write_concurrency: - type: integer - minimum: 1 - discoverer: - type: object - properties: - agent_client_options: - type: object - properties: - addrs: - type: array - items: - type: string - backoff: - type: object - properties: - backoff_factor: - type: number - backoff_time_limit: - type: string - enable_error_log: - type: boolean - initial_duration: - type: string - jitter_limit: - type: string - maximum_duration: - type: string - retry_count: - type: integer - call_option: - type: object - x-kubernetes-preserve-unknown-fields: true - circuit_breaker: - type: object - properties: - closed_error_rate: - type: number - closed_refresh_timeout: - type: string - half_open_error_rate: - type: number - min_samples: - type: integer - open_timeout: - type: string - connection_pool: - type: object - properties: - enable_dns_resolver: - type: boolean - enable_rebalance: - type: boolean - old_conn_close_duration: - type: string - rebalance_duration: - type: string - size: - type: integer - dial_option: - type: object - properties: - backoff_base_delay: - type: string - backoff_jitter: - type: number - backoff_max_delay: - type: string - backoff_multiplier: - type: number - enable_backoff: - type: boolean - initial_connection_window_size: - type: integer - initial_window_size: - type: integer - insecure: - type: boolean - interceptors: - type: array - items: - type: string - enum: - - TraceInterceptor - keepalive: - type: object - properties: - permit_without_stream: - type: boolean - time: - type: string - timeout: - type: string - max_msg_size: - type: integer - min_connection_timeout: - type: string - net: - type: object - properties: - dialer: - type: object - properties: - dual_stack_enabled: - type: boolean - keepalive: - type: string - timeout: - type: string - dns: - type: object - properties: - cache_enabled: - type: boolean - cache_expiration: - type: string - refresh_duration: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - tls: - type: object - properties: - ca: - type: string - cert: - type: string - enabled: - type: boolean - insecure_skip_verify: - type: boolean - key: - type: string - read_buffer_size: - type: integer - timeout: - type: string - write_buffer_size: - type: integer - health_check_duration: - type: string - max_recv_msg_size: - type: integer - max_retry_rpc_buffer_size: - type: integer - max_send_msg_size: - type: integer - tls: - type: object - properties: - ca: - type: string - cert: - type: string - enabled: - type: boolean - insecure_skip_verify: - type: boolean - key: - type: string - wait_for_ready: - type: boolean - client: - type: object - properties: - addrs: - type: array - items: - type: string - backoff: - type: object - properties: - backoff_factor: - type: number - backoff_time_limit: - type: string - enable_error_log: - type: boolean - initial_duration: - type: string - jitter_limit: - type: string - maximum_duration: - type: string - retry_count: - type: integer - call_option: - type: object - x-kubernetes-preserve-unknown-fields: true - circuit_breaker: - type: object - properties: - closed_error_rate: - type: number - closed_refresh_timeout: - type: string - half_open_error_rate: - type: number - min_samples: - type: integer - open_timeout: - type: string - connection_pool: - type: object - properties: - enable_dns_resolver: - type: boolean - enable_rebalance: - type: boolean - old_conn_close_duration: - type: string - rebalance_duration: - type: string - size: - type: integer - dial_option: - type: object - properties: - backoff_base_delay: - type: string - backoff_jitter: - type: number - backoff_max_delay: - type: string - backoff_multiplier: - type: number - enable_backoff: - type: boolean - initial_connection_window_size: - type: integer - initial_window_size: - type: integer - insecure: - type: boolean - interceptors: - type: array - items: - type: string - enum: - - TraceInterceptor - keepalive: - type: object - properties: - permit_without_stream: - type: boolean - time: - type: string - timeout: - type: string - max_msg_size: - type: integer - min_connection_timeout: - type: string - net: - type: object - properties: - dialer: - type: object - properties: - dual_stack_enabled: - type: boolean - keepalive: - type: string - timeout: - type: string - dns: - type: object - properties: - cache_enabled: - type: boolean - cache_expiration: - type: string - refresh_duration: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - tls: - type: object - properties: - ca: - type: string - cert: - type: string - enabled: - type: boolean - insecure_skip_verify: - type: boolean - key: - type: string - read_buffer_size: - type: integer - timeout: - type: string - write_buffer_size: - type: integer - health_check_duration: - type: string - max_recv_msg_size: - type: integer - max_retry_rpc_buffer_size: - type: integer - max_send_msg_size: - type: integer - tls: - type: object - properties: - ca: - type: string - cert: - type: string - enabled: - type: boolean - insecure_skip_verify: - type: boolean - key: - type: string - wait_for_ready: - type: boolean - duration: - type: string - enabled: - type: boolean - env: - type: array - items: - type: object - x-kubernetes-preserve-unknown-fields: true - image: - type: object - properties: - pullPolicy: - type: string - enum: - - Always - - Never - - IfNotPresent - repository: - type: string - tag: - type: string - name: - type: string - node_name: - type: string - observability: - type: object - properties: - enabled: - type: boolean - metrics: - type: object - properties: - enable_cgo: - type: boolean - enable_goroutine: - type: boolean - enable_memory: - type: boolean - enable_version_info: - type: boolean - version_info_labels: - type: array - items: - type: string - enum: - - vald_version - - server_name - - git_commit - - build_time - - go_version - - go_os - - go_arch - - cgo_enabled - - ngt_version - - build_cpu_info_flags - otlp: - type: object - properties: - attribute: - type: object - properties: - namespace: - type: string - node_name: - type: string - pod_name: - type: string - service_name: - type: string - collector_endpoint: - type: string - metrics_export_interval: - type: string - metrics_export_timeout: - type: string - trace_batch_timeout: - type: string - trace_export_timeout: - type: string - trace_max_export_batch_size: - type: integer - trace_max_queue_size: - type: integer - trace: - type: object - properties: - enabled: - type: boolean - schedule: - type: string - server_config: - type: object - properties: - full_shutdown_duration: - type: string - healths: - type: object - properties: - liveness: - type: object - properties: - enabled: - type: boolean - host: - type: string - livenessProbe: - type: object - properties: - failureThreshold: - type: integer - httpGet: - type: object - properties: - path: - type: string - port: - type: string - scheme: - type: string - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - successThreshold: - type: integer - timeoutSeconds: - type: integer - port: - type: integer - maximum: 65535 - minimum: 0 - server: - type: object - properties: - http: - type: object - properties: - handler_timeout: - type: string - idle_timeout: - type: string - read_header_timeout: - type: string - read_timeout: - type: string - shutdown_duration: - type: string - write_timeout: - type: string - mode: - type: string - network: - type: string - enum: - - tcp - - tcp4 - - tcp6 - - udp - - udp4 - - udp6 - - unix - - unixgram - - unixpacket - probe_wait_time: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - socket_path: - type: string - servicePort: - type: integer - maximum: 65535 - minimum: 0 - readiness: - type: object - properties: - enabled: - type: boolean - host: - type: string - port: - type: integer - maximum: 65535 - minimum: 0 - readinessProbe: - type: object - properties: - failureThreshold: - type: integer - httpGet: - type: object - properties: - path: - type: string - port: - type: string - scheme: - type: string - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - successThreshold: - type: integer - timeoutSeconds: - type: integer - server: - type: object - properties: - http: - type: object - properties: - handler_timeout: - type: string - idle_timeout: - type: string - read_header_timeout: - type: string - read_timeout: - type: string - shutdown_duration: - type: string - write_timeout: - type: string - mode: - type: string - network: - type: string - enum: - - tcp - - tcp4 - - tcp6 - - udp - - udp4 - - udp6 - - unix - - unixgram - - unixpacket - probe_wait_time: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - socket_path: - type: string - servicePort: - type: integer - maximum: 65535 - minimum: 0 - startup: - type: object - properties: - enabled: - type: boolean - port: - type: integer - maximum: 65535 - minimum: 0 - startupProbe: - type: object - properties: - failureThreshold: - type: integer - httpGet: - type: object - properties: - path: - type: string - port: - type: string - scheme: - type: string - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - successThreshold: - type: integer - timeoutSeconds: - type: integer - metrics: - type: object - properties: - pprof: - type: object - properties: - enabled: - type: boolean - host: - type: string - port: - type: integer - maximum: 65535 - minimum: 0 - server: - type: object - properties: - http: - type: object - properties: - handler_timeout: - type: string - idle_timeout: - type: string - read_header_timeout: - type: string - read_timeout: - type: string - shutdown_duration: - type: string - write_timeout: - type: string - mode: - type: string - network: - type: string - enum: - - tcp - - tcp4 - - tcp6 - - udp - - udp4 - - udp6 - - unix - - unixgram - - unixpacket - probe_wait_time: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - socket_path: - type: string - servicePort: - type: integer - maximum: 65535 - minimum: 0 - servers: - type: object - properties: - grpc: - type: object - properties: - enabled: - type: boolean - host: - type: string - port: - type: integer - maximum: 65535 - minimum: 0 - server: - type: object - properties: - grpc: - type: object - properties: - bidirectional_stream_concurrency: - type: integer - connection_timeout: - type: string - enable_reflection: - type: boolean - header_table_size: - type: integer - initial_conn_window_size: - type: integer - initial_window_size: - type: integer - interceptors: - type: array - items: - type: string - enum: - - RecoverInterceptor - - AccessLogInterceptor - - TraceInterceptor - - MetricInterceptor - keepalive: - type: object - properties: - max_conn_age: - type: string - max_conn_age_grace: - type: string - max_conn_idle: - type: string - min_time: - type: string - permit_without_stream: - type: boolean - time: - type: string - timeout: - type: string - max_header_list_size: - type: integer - max_receive_message_size: - type: integer - max_send_message_size: - type: integer - read_buffer_size: - type: integer - write_buffer_size: - type: integer - mode: - type: string - network: - type: string - enum: - - tcp - - tcp4 - - tcp6 - - udp - - udp4 - - udp6 - - unix - - unixgram - - unixpacket - probe_wait_time: - type: string - restart: - type: boolean - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - socket_path: - type: string - servicePort: - type: integer - maximum: 65535 - minimum: 0 - rest: - type: object - properties: - enabled: - type: boolean - host: - type: string - port: - type: integer - maximum: 65535 - minimum: 0 - server: - type: object - properties: - http: - type: object - properties: - handler_timeout: - type: string - idle_timeout: - type: string - read_header_timeout: - type: string - read_timeout: - type: string - shutdown_duration: - type: string - write_timeout: - type: string - mode: - type: string - network: - type: string - enum: - - tcp - - tcp4 - - tcp6 - - udp - - udp4 - - udp6 - - unix - - unixgram - - unixpacket - probe_wait_time: - type: string - socket_option: - type: object - properties: - ip_recover_destination_addr: - type: boolean - ip_transparent: - type: boolean - reuse_addr: - type: boolean - reuse_port: - type: boolean - tcp_cork: - type: boolean - tcp_defer_accept: - type: boolean - tcp_fast_open: - type: boolean - tcp_no_delay: - type: boolean - tcp_quick_ack: - type: boolean - socket_path: - type: string - servicePort: - type: integer - maximum: 65535 - minimum: 0 - tls: - type: object - properties: - ca: - type: string - cert: - type: string - enabled: - type: boolean - insecure_skip_verify: - type: boolean - key: - type: string - startingDeadlineSeconds: - type: integer - stream_list_concurrency: - type: integer - minimum: 1 - suspend: - type: boolean - ttlSecondsAfterFinished: - type: integer - version: - type: string - pattern: ^v[0-9]+\.[0-9]+\.[0-9]$ enabled: type: boolean env: diff --git a/tests/e2e/crud/crud_test.go b/tests/e2e/crud/crud_test.go index bb2ce4c2a5..b5a774b88b 100644 --- a/tests/e2e/crud/crud_test.go +++ b/tests/e2e/crud/crud_test.go @@ -24,7 +24,6 @@ import ( "flag" "fmt" "os" - "os/exec" "testing" "time" @@ -43,14 +42,13 @@ var ( port int ds *hdf5.Dataset - insertNum int - correctionInsertNum int - searchNum int - searchByIDNum int - getObjectNum int - updateNum int - upsertNum int - removeNum int + insertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + upsertNum int + removeNum int insertFrom int searchFrom int @@ -75,7 +73,6 @@ func init() { flag.IntVar(&port, "port", 8081, "gRPC port") flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") - flag.IntVar(&correctionInsertNum, "correction-insert-num", 3000, "number of id-vector pairs used for insert") flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") flag.IntVar(&searchByIDNum, "search-by-id-num", 100, "number of id-vector pairs used for search-by-id") flag.IntVar(&getObjectNum, "get-object-num", 100, "number of id-vector pairs used for get-object") @@ -761,9 +758,7 @@ func TestE2EIndexJobCorrection(t *testing.T) { t.Fatalf("an error occurred: %s", err) } - // prepare train data - train := ds.Train[insertFrom : insertFrom+correctionInsertNum] - + train := ds.Train[insertFrom : insertFrom+insertNum] err = op.Insert(t, ctx, operation.Dataset{ Train: train, }) @@ -773,49 +768,12 @@ func TestE2EIndexJobCorrection(t *testing.T) { sleep(t, waitAfterInsertDuration) - t.Log("Test case 1: just execute index correction and check if replica number is correct after correction") exe := operation.NewCronJobExecutor("vald-index-correction") err = exe.CreateAndWait(t, ctx, "correction-test") if err != nil { t.Fatalf("an error occurred: %s", err) } - // check if replica number is correct - err = op.StreamListObject(t, ctx, operation.Dataset{ - Train: train, - }) - if err != nil { - t.Fatalf("an error occurred: %s", err) - } - - t.Log("Test case 2: execute index correction after one agent removed") - t.Log("removing vald-agent-ngt-0...") - cmd := exec.CommandContext(ctx, "sh", "-c", "kubectl delete pod vald-agent-ngt-0 && kubectl wait --for=condition=Ready pod/vald-agent-ngt-0") - out, err := cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - t.Fatalf("%s, %s, %v", string(out), string(exitErr.Stderr), err) - } else { - t.Fatalf("unexpected error on creating job: %v", err) - } - } - t.Log(string(out)) - - // correct the deleted index - err = exe.CreateAndWait(t, ctx, "correction-test") - if err != nil { - t.Fatalf("an error occurred: %s", err) - } - - // check if replica number is correct - err = op.StreamListObject(t, ctx, operation.Dataset{ - Train: train, - }) - if err != nil { - t.Fatalf("an error occurred: %s", err) - } - - t.Log("Tear down. Removing all vectors...") err = op.Remove(t, ctx, operation.Dataset{ Train: train, }) diff --git a/tests/e2e/operation/job.go b/tests/e2e/operation/job.go index 20b20fb17d..9c3567929d 100644 --- a/tests/e2e/operation/job.go +++ b/tests/e2e/operation/job.go @@ -56,8 +56,8 @@ func deleteJob(t *testing.T, jobName string) error { func waitJob(t *testing.T, ctx context.Context, jobName string) error { t.Helper() t.Log("waiting for the correction job to complete or fail") - waitCompleteCmd := fmt.Sprintf("kubectl wait --timeout=10m job/%s --for=condition=complete", jobName) - waitFailedCmd := fmt.Sprintf("kubectl wait --timeout=10m job/%s --for=condition=failed", jobName) + waitCompleteCmd := fmt.Sprintf("kubectl wait --timeout=-1s job/%s --for=condition=complete", jobName) + waitFailedCmd := fmt.Sprintf("kubectl wait --timeout=-1s job/%s --for=condition=failed", jobName) ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go index fb6a46ac36..dd37d6ad67 100644 --- a/tests/e2e/operation/stream.go +++ b/tests/e2e/operation/stream.go @@ -1224,7 +1224,5 @@ exit_loop: return fmt.Errorf("the number of vectors returned is different at index id %v: got %v, want %v", k, v, replica) } } - - t.Log("StreamListObject operation finished successfully and all vectors are returned with correct replica number") return nil }