diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml
index 53367d67c8..b0aad134ae 100644
--- a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml
+++ b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml
@@ -277,9 +277,6 @@ spec:
- indexes
- group
- range
- dimension:
- type: integer
- minimum: 1
global_config:
type: object
properties:
diff --git a/charts/vald-benchmark-operator/schemas/job-values.yaml b/charts/vald-benchmark-operator/schemas/job-values.yaml
index 8b7de934cd..a619e55d4d 100644
--- a/charts/vald-benchmark-operator/schemas/job-values.yaml
+++ b/charts/vald-benchmark-operator/schemas/job-values.yaml
@@ -47,9 +47,6 @@ dataset:
# @schema {"name": "dataset.url", "type": "string"}
# dataset.url -- the dataset url which is used for executing benchmark job with user defined hdf5 file
url: ""
-# @schema {"name": "dimension", "type": "integer", "minimum": 1}
-# dimension -- vector dimension
-dimension: 784
# @schema {"name": "replica", "type": "integer", "minimum": 1}
# replica -- the number of running concurrency job
replica: 1
diff --git a/charts/vald-benchmark-operator/schemas/scenario-values.yaml b/charts/vald-benchmark-operator/schemas/scenario-values.yaml
index 2600b42a15..28a1d5786c 100644
--- a/charts/vald-benchmark-operator/schemas/scenario-values.yaml
+++ b/charts/vald-benchmark-operator/schemas/scenario-values.yaml
@@ -59,7 +59,6 @@ jobs:
range:
start: 1
end: 1000
- dimension: 784
replica: 1
repetition: 1
job_type: "search"
diff --git a/charts/vald-benchmark-operator/values/benchmark-job.yaml b/charts/vald-benchmark-operator/values/benchmark-job.yaml
index b11d8dfafa..3a24f856ef 100644
--- a/charts/vald-benchmark-operator/values/benchmark-job.yaml
+++ b/charts/vald-benchmark-operator/values/benchmark-job.yaml
@@ -27,7 +27,6 @@ spec:
start: 1
end: 1000
job_type: "search"
- dimension: 784
repetition: 1
replica: 1
rules: []
diff --git a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml
index ac9df3aa56..ef018a3ea6 100644
--- a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml
+++ b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml
@@ -45,7 +45,6 @@ spec:
jobs:
# @schema {"name": "jobs.items.dataset", "type": "object"}
- job_type: "insert"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -62,7 +61,6 @@ spec:
health_check_duration: "10s"
rps: 500
- job_type: "update"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -79,7 +77,6 @@ spec:
health_check_duration: "10s"
rps: 500
- job_type: "search"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -94,7 +91,6 @@ spec:
health_check_duration: "10s"
rps: 2000
- job_type: "upsert"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -111,7 +107,6 @@ spec:
health_check_duration: "10s"
rps: 1000
- job_type: "search"
- dimension: 784
repetition: 2
replica: 1
rules: []
@@ -133,7 +128,6 @@ spec:
health_check_duration: "10s"
rps: 4000
- job_type: "exists"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -148,7 +142,6 @@ spec:
health_check_duration: "10s"
rps: 1000
- job_type: "getobject"
- dimension: 784
repetition: 1
replica: 1
rules: []
@@ -163,7 +156,6 @@ spec:
health_check_duration: "10s"
rps: 1000
- job_type: "remove"
- dimension: 784
repetition: 1
replica: 1
rules: []
diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md
new file mode 100644
index 0000000000..d5bbdae4ed
--- /dev/null
+++ b/docs/performance/continuos-benchmatk.md
@@ -0,0 +1,280 @@
+# Continuous Benchmark Tool
+
+## What is the Continuous Benchmark Tool?
+
+Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/7.
+
+Assumed use case is:
+
+- Verification with workload close to the production environment
+- Verification before service installation when Vald version up
+
+## Architecture
+
+Continuous Benchmark Tool has following 2 components:
+
+- Benchmark Operator: Manages benchmark jobs
+- Benchmark Job: Executes CRUDs request to the target Vald cluster
+
+## Benchmark component and its feature
+
+### Benchmark Operator
+
+- Manages benchmark jobs according to applied manifest.
+- Apply method:
+ - Scenario method: one manifest with multiple benchmark jobs
+ - Job method: one manifest with one benchmark job
+
+### Benchmark Job
+
+- Executes CRUD request to the target Vald cluster based on defined config.
+- Execute steps are:
+ 1. Load dataset (valid only for HDF5 format )
+ 1. Execute request with load dataset
+
+## Benchmark CRD
+
+Benchmark workload can be set by applying the Kubernetes Custom Resources(CRDs), `ValdBenchmarkScenarioResource` or `ValdBenchmarkJobResource`.
+Benchmark Operator manages benchmark job according to the applied manifest.
+
+### ValdBenchmarkJob
+
+[`ValdBenchmarkJob`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml) is used for executing single benchmark job.
+
+And, Benchmark Operator also applies it to the Kubernetes cluster based on `ValdBenchmarkScenarioResource`.
+
+**main properties**
+
+| Name | mandatory | Description | type | sample |
+| :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :------------------------------------------------------------------------------------------- |
+| target | \* | target Vald cluster | object | ref: [target](#target-prop) |
+| dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) |
+| job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search |
+| repetition | | the number of job repetitions
default: `1` | integer | 1 |
+| replica | | the number of job concurrent job executions
default: `1` | integer | 2 |
+| rps | | designed request per sec to the target cluster
default: `1000` | integer | 1000 |
+| concurrency_limit | | goroutine count limit for rps adjustment
default: `200` | integer | 20 |
+| ttl_seconds_after_finished | | time until deletion of Pod after job end
default: `600` | integer | 120 |
+| insert_config | | request config for insert job | object | ref: [config](#insert-cfg-props) |
+| update_config | | request config for update job | object | ref: [config](#update-cfg-props) |
+| upsert_config | | request config for upsert job | object | ref: [config](#upsert-cfg-props) |
+| search_config | | request config for search job | object | ref: [config](#search-cfg-props) |
+| remove_config | | request config for remove job | object | ref: [config](#remove-cfg-props) |
+| object_config | | request config for object job | object | ref: [config](#object-cfg-props) |
+| client_config | | gRPC client config for running benchmark job
Tune if can not getting the expected performance with default config. | object | ref: [defaults.grpc](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) |
+| server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) |
+
+
+
+**target**
+
+- target Vald cluster information
+- type: object
+
+| property | mandatory | description | type | sample |
+| :------- | :-------- | :-------------------- | :------ | :-------- |
+| host | \* | target cluster's host | string | localhost |
+| port | \* | target cluster's port | integer | 8081 |
+
+
+
+**dataset**
+
+- dataset which is used for executing job operation
+- type: object
+
+| property | mandatory | description | type | sample |
+| :---------- | :-------- | :--------------------------------------------------------------------------------------------------- | :------------------------------------- | :------------ |
+| name | \* | dataset name | string enum: [fashion-mnist, original] | fashion-mnist |
+| group | \* | group name | string enum: [train, test, neighbors] | train |
+| indexes | \* | amount of index size | integer | 1000000 |
+| range | \* | range of indexes to be used (if there are many indexes, the range will be corrected on the job side) | object | - |
+| range.start | \* | start of range | integer | 1 |
+| range.end | \* | end of range | integer | 1000000 |
+| url | | the dataset url. It should be set when set `name` as `original` | string | |
+
+
+
+**insert_config**
+
+- rpc config for insert request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- |
+| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false |
+| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 |
+
+
+
+**update_config**
+
+- rpc config for update request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- |
+| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false |
+| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 |
+| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false |
+
+
+
+**upsert_config**
+
+- rpc config for upsert request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- |
+| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false |
+| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 |
+| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false |
+
+
+
+**upsert_config**
+
+- rpc config for search request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :-------------------- | :-------- | :---------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------- | :----- |
+| radius | | The search radius.
default: `-1` | number | -1 |
+| epsilon | | The search coefficient.
default: `0.05` | number | 0.05 |
+| num | \* | The maximum number of results to be returned. | integer | 10 |
+| min_num | | The minimum number of results to be returned. | integer | 5 |
+| timeout | | Search timeout in nanoseconds
default: `10s` | string | 3s |
+| enable_linear_search | | A flag to enable linear search operation for estimating search recall.
If it is `true`, search operation with linear operation will execute. | bool | false |
+| aggregation_algorithm | | The search aggregation algorithm option.
default: `Unknown` | string enum: ["Unknown", "ConcurrentQueue", "SortSlice", "SortPoolSlice", "PairingHeap"] | |
+
+
+
+**remove_config**
+
+- rpc config for remove request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- |
+| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false |
+| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 |
+
+
+
+**object_config**
+
+- rpc config for get object request
+- type: object
+
+| property | mandatory | description | type | sample |
+| :-------------------- | :-------- | :---------------------------------------------------------- | :------- | :----- |
+| filter_config.targets | | filter target host and port for bypassing filter component. | []object | |
+
+### ValdBenchmarkScenario
+
+[`ValdBenchmarkScenario`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml) is used for executing single or multiple benchmark job.
+
+Benchmark Operator decomposes manifest and creates benchmark resources one by one.
+The `target` and `dataset` property are the global config for scenario, they can be overwritten when each job has own config.
+
+**main properties**
+
+| property | mandatory | description | type | sample |
+| :------- | :-------- | :------------------------------------------------------------------------------------- | :----- | :-------------------------------------- |
+| target | \* | target Vald cluster information
It will be overwritten when each job has own config | object | ref: [target](#target-prop) |
+| dataset | \* | dataset information
It will be overwritten when each job has own config | object | ref: [dataset](#dataset-prop) |
+| jobs | \* | benchmark job config
The jobs written above will be executed in order. | object | ref: [benchmark job](#valdbenchmarkjob) |
+
+## Deploy Benchmark Operator
+
+Continuous benchmark operator can be applied with `Helm` same as Vald cluster.
+
+It requires `ValdBenchmarkOperatorRelease` for deploying `vald-benchmark-operator`.
+
+It is not must to apply, so please edit and apply as necessary.
+
+Sample ValdBenchmarkOperatorRelease YAML
+
+```yaml
+# @schema {"name": "name", "type": "string"}
+# name -- name of the deployment
+name: vald-benchmark-operator
+# @schema {"name": "time_zone", "type": "string"}
+# time_zone -- time_zone
+time_zone: ""
+# @schema {"name": "image", "type": "object"}
+image:
+ # @schema {"name": "image.repository", "type": "string"}
+ # image.repository -- image repository
+ repository: vdaas/vald-benchmark-operator
+ # @schema {"name": "image.tag", "type": "string"}
+ # image.tag -- image tag
+ tag: v1.7.5
+ # @schema {"name": "image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]}
+ # image.pullPolicy -- image pull policy
+ pullPolicy: Always
+# @schema {"name": "job_image", "type": "object"}
+job_image:
+ # @schema {"name": "job_image.repository", "type": "string"}
+ # image.repository -- job image repository
+ repository: vdaas/vald-benchmark-job
+ # @schema {"name": "job_image.tag", "type": "string"}
+ # image.tag -- image tag for job docker image
+ tag: v1.7.5
+ # @schema {"name": "job_image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]}
+ # image.pullPolicy -- image pull policy
+ pullPolicy: Always
+# @schema {"name": "resources", "type": "object"}
+# resources -- kubernetes resources of pod
+resources:
+ # @schema {"name": "resources.limits", "type": "object"}
+ limits:
+ cpu: 300m
+ memory: 300Mi
+ # @schema {"name": "resources.requests", "type": "object"}
+ requests:
+ cpu: 200m
+ memory: 200Mi
+# @schema {"name": "logging", "type": "object"}
+logging:
+ # @schema {"name": "logging.logger", "type": "string", "enum": ["glg", "zap"]}
+ # logging.logger -- logger name.
+ logger: glg
+ # @schema {"name": "logging.level", "type": "string", "enum": ["debug", "info", "warn", "error", "fatal"]}
+ # logging.level -- logging level.
+ level: debug
+ # @schema {"name": "logging.format", "type": "string", "enum": ["raw", "json"]}
+ # logging.format -- logging format.
+ format: raw
+```
+
+