From cc7ab0499d62888e8a4fa776f4b45f35cf9678d3 Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Wed, 21 Feb 2024 11:54:16 +0900 Subject: [PATCH] create continous benchmark doc (#2352) * improve Backport Pull Request Title & Description Quality (#2373) Signed-off-by: kpango Co-authored-by: Kosuke Morimoto * :pencil: create continous benchmark doc Signed-off-by: vankichi * style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in ace001a according to the output from Gofumpt and Prettier. Details: https://github.com/vdaas/vald/pull/2352 * :recycle: rafactor default params Signed-off-by: vankichi * style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in 9880dfc according to the output from Gofumpt and Prettier. Details: https://github.com/vdaas/vald/pull/2352 * :recycle: Fix format Signed-off-by: vankichi * Revert "improve Backport Pull Request Title & Description Quality (#2373)" This reverts commit 4246def0bf5e90494d647528abf58be519927f90. * :recycle: remove dimension from con-bench Signed-off-by: vankichi * :pencil: Add sample benchmark manifest Signed-off-by: vankichi * Update docs/performance/continuos-benchmatk.md Co-authored-by: Yusuke Kato Signed-off-by: Kiichiro YUKAWA * Update docs/performance/continuos-benchmatk.md Co-authored-by: Hiroto Funakoshi Signed-off-by: Kiichiro YUKAWA --------- Signed-off-by: kpango Signed-off-by: vankichi Signed-off-by: Kiichiro YUKAWA Co-authored-by: Yusuke Kato Co-authored-by: Kosuke Morimoto Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com> Co-authored-by: Hiroto Funakoshi --- .../crds/valdbenchmarkjob.yaml | 3 - .../schemas/job-values.yaml | 3 - .../schemas/scenario-values.yaml | 1 - .../values/benchmark-job.yaml | 1 - .../values/benchmark-scenario.yaml | 8 - docs/performance/continuos-benchmatk.md | 280 ++++++++++++++++++ example/helm/benchmark/job-values.yaml | 46 +++ example/helm/benchmark/scenario-values.yaml | 183 ++++++++++++ internal/config/benchmark.go | 1 - .../k8s/vald/benchmark/api/v1/job_types.go | 1 - .../vald/benchmark/job/job_template_option.go | 12 +- .../operator/crds/valdbenchmarkjob.yaml | 3 - pkg/tools/benchmark/job/service/job.go | 1 - pkg/tools/benchmark/job/service/option.go | 12 - pkg/tools/benchmark/job/service/search.go | 8 - pkg/tools/benchmark/job/usecase/benchmarkd.go | 1 - 16 files changed, 519 insertions(+), 45 deletions(-) create mode 100644 docs/performance/continuos-benchmatk.md create mode 100644 example/helm/benchmark/job-values.yaml create mode 100644 example/helm/benchmark/scenario-values.yaml diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml index 53367d67c8..b0aad134ae 100644 --- a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml +++ b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml @@ -277,9 +277,6 @@ spec: - indexes - group - range - dimension: - type: integer - minimum: 1 global_config: type: object properties: diff --git a/charts/vald-benchmark-operator/schemas/job-values.yaml b/charts/vald-benchmark-operator/schemas/job-values.yaml index 8b7de934cd..a619e55d4d 100644 --- a/charts/vald-benchmark-operator/schemas/job-values.yaml +++ b/charts/vald-benchmark-operator/schemas/job-values.yaml @@ -47,9 +47,6 @@ dataset: # @schema {"name": "dataset.url", "type": "string"} # dataset.url -- the dataset url which is used for executing benchmark job with user defined hdf5 file url: "" -# @schema {"name": "dimension", "type": "integer", "minimum": 1} -# dimension -- vector dimension -dimension: 784 # @schema {"name": "replica", "type": "integer", "minimum": 1} # replica -- the number of running concurrency job replica: 1 diff --git a/charts/vald-benchmark-operator/schemas/scenario-values.yaml b/charts/vald-benchmark-operator/schemas/scenario-values.yaml index 2600b42a15..28a1d5786c 100644 --- a/charts/vald-benchmark-operator/schemas/scenario-values.yaml +++ b/charts/vald-benchmark-operator/schemas/scenario-values.yaml @@ -59,7 +59,6 @@ jobs: range: start: 1 end: 1000 - dimension: 784 replica: 1 repetition: 1 job_type: "search" diff --git a/charts/vald-benchmark-operator/values/benchmark-job.yaml b/charts/vald-benchmark-operator/values/benchmark-job.yaml index b11d8dfafa..3a24f856ef 100644 --- a/charts/vald-benchmark-operator/values/benchmark-job.yaml +++ b/charts/vald-benchmark-operator/values/benchmark-job.yaml @@ -27,7 +27,6 @@ spec: start: 1 end: 1000 job_type: "search" - dimension: 784 repetition: 1 replica: 1 rules: [] diff --git a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml index ac9df3aa56..ef018a3ea6 100644 --- a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml +++ b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml @@ -45,7 +45,6 @@ spec: jobs: # @schema {"name": "jobs.items.dataset", "type": "object"} - job_type: "insert" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -62,7 +61,6 @@ spec: health_check_duration: "10s" rps: 500 - job_type: "update" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -79,7 +77,6 @@ spec: health_check_duration: "10s" rps: 500 - job_type: "search" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -94,7 +91,6 @@ spec: health_check_duration: "10s" rps: 2000 - job_type: "upsert" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -111,7 +107,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "search" - dimension: 784 repetition: 2 replica: 1 rules: [] @@ -133,7 +128,6 @@ spec: health_check_duration: "10s" rps: 4000 - job_type: "exists" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -148,7 +142,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "getobject" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -163,7 +156,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "remove" - dimension: 784 repetition: 1 replica: 1 rules: [] diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md new file mode 100644 index 0000000000..d5bbdae4ed --- /dev/null +++ b/docs/performance/continuos-benchmatk.md @@ -0,0 +1,280 @@ +# Continuous Benchmark Tool + +## What is the Continuous Benchmark Tool? + +Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/7. + +Assumed use case is: + +- Verification with workload close to the production environment +- Verification before service installation when Vald version up + +## Architecture + +Continuous Benchmark Tool has following 2 components: + +- Benchmark Operator: Manages benchmark jobs +- Benchmark Job: Executes CRUDs request to the target Vald cluster + +## Benchmark component and its feature + +### Benchmark Operator + +- Manages benchmark jobs according to applied manifest. +- Apply method: + - Scenario method: one manifest with multiple benchmark jobs + - Job method: one manifest with one benchmark job + +### Benchmark Job + +- Executes CRUD request to the target Vald cluster based on defined config. +- Execute steps are: + 1. Load dataset (valid only for HDF5 format ) + 1. Execute request with load dataset + +## Benchmark CRD + +Benchmark workload can be set by applying the Kubernetes Custom Resources(CRDs), `ValdBenchmarkScenarioResource` or `ValdBenchmarkJobResource`. +Benchmark Operator manages benchmark job according to the applied manifest. + +### ValdBenchmarkJob + +[`ValdBenchmarkJob`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml) is used for executing single benchmark job. + +And, Benchmark Operator also applies it to the Kubernetes cluster based on `ValdBenchmarkScenarioResource`. + +**main properties** + +| Name | mandatory | Description | type | sample | +| :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :------------------------------------------------------------------------------------------- | +| target | \* | target Vald cluster | object | ref: [target](#target-prop) | +| dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) | +| job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search | +| repetition | | the number of job repetitions
default: `1` | integer | 1 | +| replica | | the number of job concurrent job executions
default: `1` | integer | 2 | +| rps | | designed request per sec to the target cluster
default: `1000` | integer | 1000 | +| concurrency_limit | | goroutine count limit for rps adjustment
default: `200` | integer | 20 | +| ttl_seconds_after_finished | | time until deletion of Pod after job end
default: `600` | integer | 120 | +| insert_config | | request config for insert job | object | ref: [config](#insert-cfg-props) | +| update_config | | request config for update job | object | ref: [config](#update-cfg-props) | +| upsert_config | | request config for upsert job | object | ref: [config](#upsert-cfg-props) | +| search_config | | request config for search job | object | ref: [config](#search-cfg-props) | +| remove_config | | request config for remove job | object | ref: [config](#remove-cfg-props) | +| object_config | | request config for object job | object | ref: [config](#object-cfg-props) | +| client_config | | gRPC client config for running benchmark job
Tune if can not getting the expected performance with default config. | object | ref: [defaults.grpc](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | +| server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | + + + +**target** + +- target Vald cluster information +- type: object + +| property | mandatory | description | type | sample | +| :------- | :-------- | :-------------------- | :------ | :-------- | +| host | \* | target cluster's host | string | localhost | +| port | \* | target cluster's port | integer | 8081 | + + + +**dataset** + +- dataset which is used for executing job operation +- type: object + +| property | mandatory | description | type | sample | +| :---------- | :-------- | :--------------------------------------------------------------------------------------------------- | :------------------------------------- | :------------ | +| name | \* | dataset name | string enum: [fashion-mnist, original] | fashion-mnist | +| group | \* | group name | string enum: [train, test, neighbors] | train | +| indexes | \* | amount of index size | integer | 1000000 | +| range | \* | range of indexes to be used (if there are many indexes, the range will be corrected on the job side) | object | - | +| range.start | \* | start of range | integer | 1 | +| range.end | \* | end of range | integer | 1000000 | +| url | | the dataset url. It should be set when set `name` as `original` | string | | + + + +**insert_config** + +- rpc config for insert request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + + + +**update_config** + +- rpc config for update request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | +| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + + + +**upsert_config** + +- rpc config for upsert request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | +| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + + + +**upsert_config** + +- rpc config for search request +- type: object + +| property | mandatory | description | type | sample | +| :-------------------- | :-------- | :---------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------- | :----- | +| radius | | The search radius.
default: `-1` | number | -1 | +| epsilon | | The search coefficient.
default: `0.05` | number | 0.05 | +| num | \* | The maximum number of results to be returned. | integer | 10 | +| min_num | | The minimum number of results to be returned. | integer | 5 | +| timeout | | Search timeout in nanoseconds
default: `10s` | string | 3s | +| enable_linear_search | | A flag to enable linear search operation for estimating search recall.
If it is `true`, search operation with linear operation will execute. | bool | false | +| aggregation_algorithm | | The search aggregation algorithm option.
default: `Unknown` | string enum: ["Unknown", "ConcurrentQueue", "SortSlice", "SortPoolSlice", "PairingHeap"] | | + + + +**remove_config** + +- rpc config for remove request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + + + +**object_config** + +- rpc config for get object request +- type: object + +| property | mandatory | description | type | sample | +| :-------------------- | :-------- | :---------------------------------------------------------- | :------- | :----- | +| filter_config.targets | | filter target host and port for bypassing filter component. | []object | | + +### ValdBenchmarkScenario + +[`ValdBenchmarkScenario`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml) is used for executing single or multiple benchmark job. + +Benchmark Operator decomposes manifest and creates benchmark resources one by one. +The `target` and `dataset` property are the global config for scenario, they can be overwritten when each job has own config. + +**main properties** + +| property | mandatory | description | type | sample | +| :------- | :-------- | :------------------------------------------------------------------------------------- | :----- | :-------------------------------------- | +| target | \* | target Vald cluster information
It will be overwritten when each job has own config | object | ref: [target](#target-prop) | +| dataset | \* | dataset information
It will be overwritten when each job has own config | object | ref: [dataset](#dataset-prop) | +| jobs | \* | benchmark job config
The jobs written above will be executed in order. | object | ref: [benchmark job](#valdbenchmarkjob) | + +## Deploy Benchmark Operator + +Continuous benchmark operator can be applied with `Helm` same as Vald cluster. + +It requires `ValdBenchmarkOperatorRelease` for deploying `vald-benchmark-operator`. + +It is not must to apply, so please edit and apply as necessary. + +
Sample ValdBenchmarkOperatorRelease YAML + +```yaml +# @schema {"name": "name", "type": "string"} +# name -- name of the deployment +name: vald-benchmark-operator +# @schema {"name": "time_zone", "type": "string"} +# time_zone -- time_zone +time_zone: "" +# @schema {"name": "image", "type": "object"} +image: + # @schema {"name": "image.repository", "type": "string"} + # image.repository -- image repository + repository: vdaas/vald-benchmark-operator + # @schema {"name": "image.tag", "type": "string"} + # image.tag -- image tag + tag: v1.7.5 + # @schema {"name": "image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]} + # image.pullPolicy -- image pull policy + pullPolicy: Always +# @schema {"name": "job_image", "type": "object"} +job_image: + # @schema {"name": "job_image.repository", "type": "string"} + # image.repository -- job image repository + repository: vdaas/vald-benchmark-job + # @schema {"name": "job_image.tag", "type": "string"} + # image.tag -- image tag for job docker image + tag: v1.7.5 + # @schema {"name": "job_image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]} + # image.pullPolicy -- image pull policy + pullPolicy: Always +# @schema {"name": "resources", "type": "object"} +# resources -- kubernetes resources of pod +resources: + # @schema {"name": "resources.limits", "type": "object"} + limits: + cpu: 300m + memory: 300Mi + # @schema {"name": "resources.requests", "type": "object"} + requests: + cpu: 200m + memory: 200Mi +# @schema {"name": "logging", "type": "object"} +logging: + # @schema {"name": "logging.logger", "type": "string", "enum": ["glg", "zap"]} + # logging.logger -- logger name. + logger: glg + # @schema {"name": "logging.level", "type": "string", "enum": ["debug", "info", "warn", "error", "fatal"]} + # logging.level -- logging level. + level: debug + # @schema {"name": "logging.format", "type": "string", "enum": ["raw", "json"]} + # logging.format -- logging format. + format: raw +``` + +
+ +For more details of the configuration of `vald-benchmark-operator-release`, please refer to [here](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/values.yaml) + +1. Add Vald repo into the helm repo + + ```bash + helm repo add vald https://vdaas.vald.org + ``` + +1. Deploy `vald-benchmark-operator-release` + + ```bash + helm install vald-benchmark-operator-release vald/vald-benchmark-operator + ``` + +1. Apply `vbor.yaml` (optional) + + ```bash + kubectl apply -f vbor.yaml + ``` + +## Running Continuous Benchmarks + +After deploy the benchmark operator, you can execute continuous benchmark by applying `ValdBenchmarkScenarioRelease` or `ValdBenchmarkJobRelease`. + +Please configure designed benchmark and apply by `kubectl` command. + +The sample manifests are [here](https://github.com/vdaas/vald/tree/main/example/helm/benchmark). diff --git a/example/helm/benchmark/job-values.yaml b/example/helm/benchmark/job-values.yaml new file mode 100644 index 0000000000..3df5553bcd --- /dev/null +++ b/example/helm/benchmark/job-values.yaml @@ -0,0 +1,46 @@ +apiVersion: vald.vdaas.org/v1 +kind: ValdBenchmarkJob +metadata: + name: sample-job +spec: + # @schema {"name": "dataset", "type": "object"} + # dataset -- dataset information + dataset: + # @schema {"name": "dataset.name", "type": "string" } + # dataset.name -- the name of dataset + name: "fashion-mnist" + # @schema {"name": "dataset.indexes", "type": "integer"} + # dataset.indexes -- the amount of indexes + indexes: 60000 + # @schema {"name": "dataset.group", "type": "string"} + # dataset.group -- the hdf5 group name of dataset + group: "train" + # @schema {"name": "dataset.range", "type": "object"} + # dataset.range -- the data range of indexes + range: + # @schema {"name": "dataset.range.start", "type": "integer"} + # dataset.range.start -- start index number + start: 1 + # @schema {"name": "dataset.range.end", "type": "integer"} + # dataset.range.end -- end index number + end: 60000 + url: "http://ann-benchmarks.com/fashion-mnist-784-euclidean.hdf5" + job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + client_config: + health_check_duration: "10s" + rps: 2000 + insert_config: + skip_strict_exist_check: false + upsert_config: + skip_strict_exist_check: false + target: + host: "vald-lb-gateway.default.svc.cluster.local" + port: 8081 + global_config: + version: v0.0.1 + time_zone: JST + logging: + level: debug diff --git a/example/helm/benchmark/scenario-values.yaml b/example/helm/benchmark/scenario-values.yaml new file mode 100644 index 0000000000..14a3ea051c --- /dev/null +++ b/example/helm/benchmark/scenario-values.yaml @@ -0,0 +1,183 @@ +apiVersion: vald.vdaas.org/v1 +kind: ValdBenchmarkScenario +metadata: + name: insert-search +spec: + # @schema {"name": "dataset", "type": "object"} + # dataset -- dataset information + dataset: + # @schema {"name": "dataset.name", "type": "string" } + # dataset.name -- the name of dataset + name: "fashion-mnist" + # @schema {"name": "dataset.indexes", "type": "integer"} + # dataset.indexes -- the amount of indexes + indexes: 100000 + # @schema {"name": "dataset.group", "type": "string"} + # dataset.group -- the hdf5 group name of dataset + group: "train" + # @schema {"name": "dataset.range", "type": "object"} + # dataset.range -- the data range of indexes + range: + # @schema {"name": "dataset.range.start", "type": "integer"} + # dataset.range.start -- start index number + start: 1 + # @schema {"name": "dataset.range.end", "type": "integer"} + # dataset.range.end -- end index number + end: 100000 + # @schema {"name": "jobs", "type": "array", "items": {"type": "object"}} + # jobs -- benchmark jobs + jobs: + - job_type: "insert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 10000 + group: "train" + range: + start: 1 + end: 10000 + insert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + upsert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "search" + repetition: 1 + replica: 1 + rules: [] + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 10 + timeout: "1m" + enable_linear_search: false + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 30000 + group: "train" + range: + start: 10001 + end: 40000 + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 5 + timeout: "1m" + upsert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "search" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "test" + range: + start: 1 + end: 20000 + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 10 + timeout: "1m" + enable_linear_search: false + client_config: + health_check_duration: "10s" + rps: 4000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "exists" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "getobject" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "remove" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 40000 + group: "train" + range: + start: 1 + end: 40000 + remove_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + # @schema {"name": "target", "type": "array", "items": {"type": "object"}} + # target -- target cluster host&port + target: + host: "vald-lb-gateway.default.svc.cluster.local" + port: 8081 diff --git a/internal/config/benchmark.go b/internal/config/benchmark.go index e7e9acb4e4..68272a58bc 100644 --- a/internal/config/benchmark.go +++ b/internal/config/benchmark.go @@ -21,7 +21,6 @@ package config type BenchmarkJob struct { Target *BenchmarkTarget `json:"target,omitempty" yaml:"target"` Dataset *BenchmarkDataset `json:"dataset,omitempty" yaml:"dataset"` - Dimension int `json:"dimension,omitempty" yaml:"dimension"` Replica int `json:"replica,omitempty" yaml:"replica"` Repetition int `json:"repetition,omitempty" yaml:"repetition"` JobType string `json:"job_type,omitempty" yaml:"job_type"` diff --git a/internal/k8s/vald/benchmark/api/v1/job_types.go b/internal/k8s/vald/benchmark/api/v1/job_types.go index 8c82d48e37..cc1fedb018 100644 --- a/internal/k8s/vald/benchmark/api/v1/job_types.go +++ b/internal/k8s/vald/benchmark/api/v1/job_types.go @@ -28,7 +28,6 @@ type BenchmarkJobSpec struct { ServerConfig *config.Servers `json:"server_config,omitempty" yaml:"server_config"` Target *BenchmarkTarget `json:"target,omitempty" yaml:"target"` Dataset *BenchmarkDataset `json:"dataset,omitempty" yaml:"dataset"` - Dimension int `json:"dimension,omitempty" yaml:"dimension"` Replica int `json:"replica,omitempty" yaml:"replica"` Repetition int `json:"repetition,omitempty" yaml:"repetition"` JobType string `json:"job_type,omitempty" yaml:"job_type"` diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go index 861c5c6d17..35df5e426a 100644 --- a/internal/k8s/vald/benchmark/job/job_template_option.go +++ b/internal/k8s/vald/benchmark/job/job_template_option.go @@ -64,13 +64,21 @@ func WithImagePullPolicy(p ImagePullPolicy) BenchmarkJobTplOption { // BenchmarkJobOption represents the option for create benchmark job template. type BenchmarkJobOption func(b *jobs.Job) error -// defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. -const defaultTTLSeconds int32 = 600 +const ( + // defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. + defaultTTLSeconds int32 = 600 + // defaultCompletions represents the default completions for benchmark job template. + defaultCompletions int32 = 1 + // defaultParallelism represents the default parallelism for benchmark job template. + defaultParallelism int32 = 1 +) var defaultBenchmarkJobOpts = []BenchmarkJobOption{ WithSvcAccountName(svcAccount), WithRestartPolicy(RestartPolicyNever), WithTTLSecondsAfterFinished(defaultTTLSeconds), + WithCompletions(defaultCompletions), + WithParallelism(defaultParallelism), } // WithSvcAccountName sets the service account name for benchmark job. diff --git a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml index 53367d67c8..b0aad134ae 100644 --- a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml +++ b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml @@ -277,9 +277,6 @@ spec: - indexes - group - range - dimension: - type: integer - minimum: 1 global_config: type: object properties: diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go index 455d6f1794..905501534c 100644 --- a/pkg/tools/benchmark/job/service/job.go +++ b/pkg/tools/benchmark/job/service/job.go @@ -82,7 +82,6 @@ func (jt jobType) String() string { type job struct { eg errgroup.Group - dimension int dataset *config.BenchmarkDataset jobType jobType jobFunc func(context.Context, chan error) error diff --git a/pkg/tools/benchmark/job/service/option.go b/pkg/tools/benchmark/job/service/option.go index 879b5263e1..4b0fb7e459 100644 --- a/pkg/tools/benchmark/job/service/option.go +++ b/pkg/tools/benchmark/job/service/option.go @@ -32,23 +32,11 @@ import ( type Option func(j *job) error var defaultOpts = []Option{ - // TODO: set default config for client - WithDimension(748), WithBeforeJobDuration("30s"), WithRPS(1000), WithConcurencyLimit(200), } -// WithDimension sets the vector's dimension for running benchmark job with dataset. -func WithDimension(dim int) Option { - return func(j *job) error { - if dim > 0 { - j.dimension = dim - } - return nil - } -} - // WithInsertConfig sets the insert API config for running insert request job. func WithInsertConfig(c *config.InsertConfig) Option { return func(j *job) error { diff --git a/pkg/tools/benchmark/job/service/search.go b/pkg/tools/benchmark/job/service/search.go index 4128584344..79beaadaf9 100644 --- a/pkg/tools/benchmark/job/service/search.go +++ b/pkg/tools/benchmark/job/service/search.go @@ -66,10 +66,6 @@ func (j *job) search(ctx context.Context, ech chan error) error { } // idx is the modulo, which takes between <0, len(vecs)-1>. idx := (iter - 1) % len(vecs) - if len(vecs[idx]) != j.dimension { - log.Warn("len(vecs) ", len(vecs[iter]), "is not matched with ", j.dimension) - return nil - } res, err := j.client.Search(egctx, &payload.Search_Request{ Vector: vecs[idx], Config: cfg, @@ -121,10 +117,6 @@ func (j *job) search(ctx context.Context, ech chan error) error { log.Debugf("[benchmark job] Start linear search: iter = %d", iter) // idx is the modulo, which takes between <0, len(vecs)-1>. idx := (iter - 1) % len(vecs) - if len(vecs[idx]) != j.dimension { - log.Warn("len(vecs) ", len(vecs[idx]), "is not matched with ", j.dimension) - return nil - } res, err := j.client.LinearSearch(egctx, &payload.Search_Request{ Vector: vecs[idx], Config: cfg, diff --git a/pkg/tools/benchmark/job/usecase/benchmarkd.go b/pkg/tools/benchmark/job/usecase/benchmarkd.go index 173edf6d8b..1036125b2e 100644 --- a/pkg/tools/benchmark/job/usecase/benchmarkd.go +++ b/pkg/tools/benchmark/job/usecase/benchmarkd.go @@ -91,7 +91,6 @@ func New(cfg *config.Config) (r runner.Runner, err error) { service.WithValdClient(vcli), service.WithDataset(cfg.Job.Dataset), service.WithJobTypeByString(cfg.Job.JobType), - service.WithDimension(cfg.Job.Dimension), service.WithInsertConfig(cfg.Job.InsertConfig), service.WithUpdateConfig(cfg.Job.UpdateConfig), service.WithUpsertConfig(cfg.Job.UpsertConfig),