From 78550b8678eb5bdc00007592384d7d321d3cbdc5 Mon Sep 17 00:00:00 2001 From: Yusuke Kato Date: Fri, 16 Feb 2024 21:53:57 +0900 Subject: [PATCH 01/11] improve Backport Pull Request Title & Description Quality (#2373) Signed-off-by: kpango Co-authored-by: Kosuke Morimoto --- .github/workflows/backport.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 5d041ce025..ff6bd63414 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -44,25 +44,20 @@ jobs: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - name: Set context - id: set_context + - name: Get PR title and body + id: pr_info run: | - LABEL_NAMES=`cat ${GITHUB_EVENT_PATH} | jq -r --arg PREFIX $TARGET_LABEL_NAME_PREFIX '[.pull_request.labels[]? | select(.name | startswith($PREFIX)) | .name] | join(" ")'` - echo "LABEL_NAMES=${LABEL_NAMES}" >> $GITHUB_OUTPUT # e.g.) actions/backport/v1.7 actions/backport/v1.8 - echo "${LABEL_NAMES}" + echo "PR_TITLE=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + echo "PR_BODY=$(jq --raw-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - name: Create PR - if: ${{ steps.set_context.outputs.LABEL_NAMES != '' }} + if: ${{ github.event.pull_request.labels.*.name == '${TARGET_LABEL_NAME_PREFIX}*' }} env: - LABEL_NAMES: ${{ steps.set_context.outputs.LABEL_NAMES }} + PR_TITLE: ${{ steps.pr_info.outputs.PR_TITEL }} + PR_BODY: ${{ steps.pr_info.outputs.PR_BODY }} GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} run: | - PR_TITLE=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.title"` - PR_BODY=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.body"` - PR_NUM=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.number"` - - echo "${PR_NUM} ${PR_TITLE}: ${PR_BODY}" - + LABEL_NAMES=`echo "${{ github.event.pull_request.labels.*.name }}" | jq -r '[.[] | select(startswith("actions/backport/"))] | join(" ")'` for LABEL_NAME in ${LABEL_NAMES}; do BRANCH_NAME=`echo "${LABEL_NAME}" | sed -e "s:^${TARGET_LABEL_NAME_PREFIX}::"` # e.g) release/vx.x, main BACKPORT_BRANCH_NAME="${BACKPORT_BRANCH_NAME_PREFIX}/${BRANCH_NAME}/${GITHUB_HEAD_REF}" # e.g) backport/release/vx.x/{current branch name} @@ -79,6 +74,6 @@ jobs: git push origin ${BACKPORT_BRANCH_NAME} gh pr create --base ${BRANCH_NAME} \ - --title "Backport PR #${PR_NUM} to ${BRANCH_NAME} for ${PR_TITLE}" \ + --title "Backport to ${BRANCH_NAME} for ${PR_TITLE}" \ --body "${PR_BODY}" done From 28b38b83509a8af33d3f3eb6bc89b59b2444535e Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 15 Feb 2024 12:08:23 +0900 Subject: [PATCH 02/11] :pencil: create continous benchmark doc Signed-off-by: vankichi --- docs/performance/continuos-benchmatk.md | 272 ++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 docs/performance/continuos-benchmatk.md diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md new file mode 100644 index 0000000000..83929da844 --- /dev/null +++ b/docs/performance/continuos-benchmatk.md @@ -0,0 +1,272 @@ +# Continuous Benchmark Tool + +## What is the Continuous Benchmark Tool? + +Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/365. + +Assumed use case is: + +- Verification with workload close to production environment +- Verification before service installation when Vald version up + +## Architecture + +Continuous Benchmark Tool has following 2 components: + +- Benchmark Operator: Manages benchmark jobs +- Benchmark Job: Executes CRUDs request to the target Vald cluster + +## Benchmark component and its feature + +### Benchmark Operator + +- Manages benchmark jobs according to applied manifest. +- Apply method: + - Scenario method: one manifest with multiple benchmark jobs + - Job method: one manifest with one benchmark job + +### Benchmark Job + +- Executes CRUD request to the target Vald cluster based on defined config. +- Execute steps are: + 1. Load dataset (valid only for HDF5 format ) + 1. Execute request with load dataset + +## Benchmark CRD + +Benchmark workload can be set by applying the Kubernetes Custom Resources(CRDs), `ValdBenchmarkScenarioResource` or `ValdBenchmarkJobResource`. +Benchmark Operator manages benchmark job according to the applied manifest. + +### ValdBenchmarkJob + +[`ValdBenchmarkJob`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml) is used for executing single benchmark job. + +And, Benchmark Operator also applies it to the Kubernetes cluster based on `ValdBenchmarkScenarioResource`. + +**main properties** + +| Name | mandatory | Description | type | sample | +| :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :------------------------------------------------------------------------------------------- | +| target | \* | target Vald cluster | object | ref: [target](#target-prop) | +| dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) | +| dimension | \* | dimension of vector | integer | 784 | +| job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search | +| repetition | | the number of job repetitions
default: `1` | integer | 1 | +| replica | | the number of job concurrent job executions
default: `1` | integer | 2 | +| rps | | designed request per sec to the target cluster
default: `1000` | integer | 1000 | +| concurrency_limit | | goroutine count limit for rps adjustment
default: `200` | integer | 20 | +| ttl_seconds_after_finished | | time until deletion of Pod after job end
default: `600` | integer | 120 | +| insert_config | | request config for insert job | object | ref: [config](#insert-cfg-props) | +| update_config | | request config for update job | object | ref: [config](#update-cfg-props) | +| upsert_config | | request config for upsert job | object | ref: [config](#upsert-cfg-props) | +| search_config | | request config for search job | object | ref: [config](#search-cfg-props) | +| remove_config | | request config for remove job | object | ref: [config](#remove-cfg-props) | +| object_config | | request config for object job | object | ref: [config](#object-cfg-props) | +| client_config | | gRPC client config for running benchmark job
Tune if can not getting the expected performance with default config. | object | ref: [defaults.grpc](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | +| server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | + + +**target** + +- target Vald cluster information +- type: object + +| property | mandatory | description | type | sample | +| :------- | :-------- | :-------------------- | :------ | :-------- | +| host | \* | target cluster's host | string | localhost | +| port | \* | target cluster's port | integer | 8081 | + + +**dataset** + +- dataset which is used for executing job operation +- type: object + +| property | mandatory | description | type | sample | +| :---------- | :-------- | :--------------------------------------------------------------------------------------------------- | :------------------------------------- | :------------ | +| name | \* | dataset name | string enum: [fashion-mnist, original] | fashion-mnist | +| group | \* | group name | string enum: [train, test, neighbors] | train | +| indexes | \* | amount of index size | integer | 1000000 | +| range | \* | range of indexes to be used (if there are many indexes, the range will be corrected on the job side) | object | - | +| range.start | \* | start of range | integer | 1 | +| range.end | \* | end of range | integer | 1000000 | +| url | | the dataset url. It should be set when set `name` as `original` | string | | + + +**insert_config** + +- rpc config for insert request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + + +**update_config** + +- rpc config for update request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | +| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + + +**upsert_config** + +- rpc config for upsert request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | +| disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + + +**upsert_config** + +- rpc config for search request +- type: object + +| property | mandatory | description | type | sample | +| :-------------------- | :-------- | :---------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------- | :----- | +| radius | | The search radius.
default: `-1` | number | -1 | +| epsilon | | The search coefficient.
default: `0.05` | number | 0.05 | +| num | \* | The maximum number of results to be returned. | integer | 10 | +| min_num | | The minimum number of results to be returned. | integer | 5 | +| timeout | | Search timeout in nanoseconds
default: `10s` | string | 3s | +| enable_linear_search | | A flag to enable linear search operation for estimating search recall.
If it is `true`, search operation with linear operation will execute. | bool | false | +| aggregation_algorithm | | The search aggregation algorithm option.
default: `Unknown` | string enum: ["Unknown", "ConcurrentQueue", "SortSlice", "SortPoolSlice", "PairingHeap"] | | + + +**remove_config** + +- rpc config for remove request +- type: object + +| property | mandatory | description | type | sample | +| :---------------------- | :-------- | :----------------------------------------------------------------------------------------------------------- | :----- | :--------- | +| skip_strict_exist_check | | Check whether the same vector is already inserted or not.
The ID should be unique if the value is `true`. | bool | false | +| timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + + +**object_config** + +- rpc config for get object request +- type: object + +| property | mandatory | description | type | sample | +| :-------------------- | :-------- | :---------------------------------------------------------- | :------- | :----- | +| filter_config.targets | | filter target host and port for bypassing filter component. | []object | | + +### ValdBenchmarkScenario + +[`ValdBenchmarkScenario`](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/crds/valdbenchmarkscenario.yaml) is used for executing single or multiple benchmark job. + +Benchmark Operator decomposes manifest and creates benchmark resources one by one. +The `target` and `dataset` property are the global config for scenario, they can be overwritten when each job has own config. + +**main properties** + +| property | mandatory | description | type | sample | +| :------- | :-------- | :------------------------------------------------------------------------------------- | :----- | :-------------------------------------- | +| target | \* | target Vald cluster information
It will be overwritten when each job has own config | object | ref: [target](#target-prop) | +| dataset | \* | dataset information
It will be overwritten when each job has own config | object | ref: [dataset](#dataset-prop) | +| jobs | \* | benchmark job config
The jobs written above will be executed in order. | object | ref: [benchmark job](#valdbenchmarkjob) | + +## Deploy Benchmark Operator + +Continuous benchmark operator can be applied with `Helm` same as Vald cluster. + +It requires `ValdBenchmarkOperatorRelease` for deploying `vald-benchmark-operator`. + +It is not must to apply, so please edit and apply as necessary. + +
Sample ValdBenchmarkOperatorRelease YAML + +```yaml +# @schema {"name": "name", "type": "string"} +# name -- name of the deployment +name: vald-benchmark-operator +# @schema {"name": "time_zone", "type": "string"} +# time_zone -- time_zone +time_zone: "" +# @schema {"name": "image", "type": "object"} +image: + # @schema {"name": "image.repository", "type": "string"} + # image.repository -- image repository + repository: vdaas/vald-benchmark-operator + # @schema {"name": "image.tag", "type": "string"} + # image.tag -- image tag + tag: v1.7.5 + # @schema {"name": "image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]} + # image.pullPolicy -- image pull policy + pullPolicy: Always +# @schema {"name": "job_image", "type": "object"} +job_image: + # @schema {"name": "job_image.repository", "type": "string"} + # image.repository -- job image repository + repository: vdaas/vald-benchmark-job + # @schema {"name": "job_image.tag", "type": "string"} + # image.tag -- image tag for job docker image + tag: v1.7.5 + # @schema {"name": "job_image.pullPolicy", "type": "string", "enum": ["Always", "Never", "IfNotPresent"]} + # image.pullPolicy -- image pull policy + pullPolicy: Always +# @schema {"name": "resources", "type": "object"} +# resources -- kubernetes resources of pod +resources: + # @schema {"name": "resources.limits", "type": "object"} + limits: + cpu: 300m + memory: 300Mi + # @schema {"name": "resources.requests", "type": "object"} + requests: + cpu: 200m + memory: 200Mi +# @schema {"name": "logging", "type": "object"} +logging: + # @schema {"name": "logging.logger", "type": "string", "enum": ["glg", "zap"]} + # logging.logger -- logger name. + logger: glg + # @schema {"name": "logging.level", "type": "string", "enum": ["debug", "info", "warn", "error", "fatal"]} + # logging.level -- logging level. + level: debug + # @schema {"name": "logging.format", "type": "string", "enum": ["raw", "json"]} + # logging.format -- logging format. + format: raw +``` + +
+ +For more details of the configuration of `vald-benchmark-operator-release`, please refer to [here](https://github.com/vdaas/vald/blob/main/charts/vald-benchmark-operator/values.yaml) + +1. Add Vald repo into the helm repo + + ```bash + helm repo add vald https://vdaas.vald.org + ``` + +1. Deploy `vald-benchmark-operator-release` + + ```bash + helm install vald-benchmark-operator-release vald/vald-benchmark-operator + ``` + +1. Apply `vbor.yaml` (optional) + + ```bash + kubectl apply -f vbor.yaml + ``` + +After deploy the benchmark operator, you can execute continuous benchmark by applying `ValdBenchmarkScenarioRelease` or `ValdBenchmarkJobRelease`. + +Please configure designed benchmark and apply by `kubectl` command. + +The sample manifests are [here](https://github.com/vdaas/vald/tree/main/charts/vald-benchmark-operator/values). + From a186e35a18bca41a84ff3ce4f26111367878732f Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 03:14:27 +0000 Subject: [PATCH 03/11] style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in ace001a according to the output from Gofumpt and Prettier. Details: https://github.com/vdaas/vald/pull/2352 --- docs/performance/continuos-benchmatk.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index 83929da844..2998e2f57f 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -269,4 +269,3 @@ After deploy the benchmark operator, you can execute continuous benchmark by app Please configure designed benchmark and apply by `kubectl` command. The sample manifests are [here](https://github.com/vdaas/vald/tree/main/charts/vald-benchmark-operator/values). - From 27915675df18ba7775e9b9b3b9a5e6a3a5faf2b1 Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 15 Feb 2024 12:52:38 +0900 Subject: [PATCH 04/11] :recycle: rafactor default params Signed-off-by: vankichi --- docs/performance/continuos-benchmatk.md | 8 ++++++++ internal/k8s/vald/benchmark/job/job_template_option.go | 8 +++++++- pkg/tools/benchmark/job/service/option.go | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index 2998e2f57f..34a483cc93 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -66,6 +66,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | + **target** - target Vald cluster information @@ -77,6 +78,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | port | \* | target cluster's port | integer | 8081 | + **dataset** - dataset which is used for executing job operation @@ -93,6 +95,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | url | | the dataset url. It should be set when set `name` as `original` | string | | + **insert_config** - rpc config for insert request @@ -104,6 +107,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + **update_config** - rpc config for update request @@ -116,6 +120,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + **upsert_config** - rpc config for upsert request @@ -128,6 +133,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | disable_balanced_update | | A flag to disable balanced update (split remove -> insert operation) during update operation. | bool | false | + **upsert_config** - rpc config for search request @@ -144,6 +150,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | aggregation_algorithm | | The search aggregation algorithm option.
default: `Unknown` | string enum: ["Unknown", "ConcurrentQueue", "SortSlice", "SortPoolSlice", "PairingHeap"] | | + **remove_config** - rpc config for remove request @@ -155,6 +162,7 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | timestamp | | The timestamp of the vector inserted.
If it is N/A, the current time will be used. | string | 1707272658 | + **object_config** - rpc config for get object request diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go index 861c5c6d17..4e5e03c146 100644 --- a/internal/k8s/vald/benchmark/job/job_template_option.go +++ b/internal/k8s/vald/benchmark/job/job_template_option.go @@ -65,12 +65,18 @@ func WithImagePullPolicy(p ImagePullPolicy) BenchmarkJobTplOption { type BenchmarkJobOption func(b *jobs.Job) error // defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. -const defaultTTLSeconds int32 = 600 +const ( + defaultTTLSeconds int32 = 600 + defaultCompletions int32 = 1 + defaultParallelism int32 = 1 +) var defaultBenchmarkJobOpts = []BenchmarkJobOption{ WithSvcAccountName(svcAccount), WithRestartPolicy(RestartPolicyNever), WithTTLSecondsAfterFinished(defaultTTLSeconds), + WithCompletions(defaultCompletions), + WithParallelism(defaultParallelism), } // WithSvcAccountName sets the service account name for benchmark job. diff --git a/pkg/tools/benchmark/job/service/option.go b/pkg/tools/benchmark/job/service/option.go index 879b5263e1..10b04dbebc 100644 --- a/pkg/tools/benchmark/job/service/option.go +++ b/pkg/tools/benchmark/job/service/option.go @@ -33,7 +33,7 @@ type Option func(j *job) error var defaultOpts = []Option{ // TODO: set default config for client - WithDimension(748), + WithDimension(784), WithBeforeJobDuration("30s"), WithRPS(1000), WithConcurencyLimit(200), From 1a18979e8e54bbe52e22ea53ff4af31a0146d4db Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 03:53:08 +0000 Subject: [PATCH 05/11] style: format code with Gofumpt and Prettier This commit fixes the style issues introduced in 9880dfc according to the output from Gofumpt and Prettier. Details: https://github.com/vdaas/vald/pull/2352 --- internal/k8s/vald/benchmark/job/job_template_option.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go index 4e5e03c146..3f3b5e8981 100644 --- a/internal/k8s/vald/benchmark/job/job_template_option.go +++ b/internal/k8s/vald/benchmark/job/job_template_option.go @@ -66,7 +66,7 @@ type BenchmarkJobOption func(b *jobs.Job) error // defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. const ( - defaultTTLSeconds int32 = 600 + defaultTTLSeconds int32 = 600 defaultCompletions int32 = 1 defaultParallelism int32 = 1 ) From 850f5279a530b739cf4172109e434517f539b2b2 Mon Sep 17 00:00:00 2001 From: vankichi Date: Thu, 15 Feb 2024 15:22:37 +0900 Subject: [PATCH 06/11] :recycle: Fix format Signed-off-by: vankichi --- internal/k8s/vald/benchmark/job/job_template_option.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/k8s/vald/benchmark/job/job_template_option.go b/internal/k8s/vald/benchmark/job/job_template_option.go index 3f3b5e8981..35df5e426a 100644 --- a/internal/k8s/vald/benchmark/job/job_template_option.go +++ b/internal/k8s/vald/benchmark/job/job_template_option.go @@ -64,10 +64,12 @@ func WithImagePullPolicy(p ImagePullPolicy) BenchmarkJobTplOption { // BenchmarkJobOption represents the option for create benchmark job template. type BenchmarkJobOption func(b *jobs.Job) error -// defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. const ( - defaultTTLSeconds int32 = 600 + // defaultTTLSeconds represents the default TTLSecondsAfterFinished for benchmark job template. + defaultTTLSeconds int32 = 600 + // defaultCompletions represents the default completions for benchmark job template. defaultCompletions int32 = 1 + // defaultParallelism represents the default parallelism for benchmark job template. defaultParallelism int32 = 1 ) From 789b32253708506106da61206363aace136efe50 Mon Sep 17 00:00:00 2001 From: vankichi Date: Mon, 19 Feb 2024 17:15:25 +0900 Subject: [PATCH 07/11] Revert "improve Backport Pull Request Title & Description Quality (#2373)" This reverts commit 4246def0bf5e90494d647528abf58be519927f90. --- .github/workflows/backport.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ff6bd63414..5d041ce025 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -44,20 +44,25 @@ jobs: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - name: Get PR title and body - id: pr_info + - name: Set context + id: set_context run: | - echo "PR_TITLE=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV - echo "PR_BODY=$(jq --raw-output .pull_request.body $GITHUB_EVENT_PATH)" >> $GITHUB_ENV + LABEL_NAMES=`cat ${GITHUB_EVENT_PATH} | jq -r --arg PREFIX $TARGET_LABEL_NAME_PREFIX '[.pull_request.labels[]? | select(.name | startswith($PREFIX)) | .name] | join(" ")'` + echo "LABEL_NAMES=${LABEL_NAMES}" >> $GITHUB_OUTPUT # e.g.) actions/backport/v1.7 actions/backport/v1.8 + echo "${LABEL_NAMES}" - name: Create PR - if: ${{ github.event.pull_request.labels.*.name == '${TARGET_LABEL_NAME_PREFIX}*' }} + if: ${{ steps.set_context.outputs.LABEL_NAMES != '' }} env: - PR_TITLE: ${{ steps.pr_info.outputs.PR_TITEL }} - PR_BODY: ${{ steps.pr_info.outputs.PR_BODY }} + LABEL_NAMES: ${{ steps.set_context.outputs.LABEL_NAMES }} GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} run: | - LABEL_NAMES=`echo "${{ github.event.pull_request.labels.*.name }}" | jq -r '[.[] | select(startswith("actions/backport/"))] | join(" ")'` + PR_TITLE=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.title"` + PR_BODY=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.body"` + PR_NUM=`cat $GITHUB_EVENT_PATH | jq -r ".pull_request.number"` + + echo "${PR_NUM} ${PR_TITLE}: ${PR_BODY}" + for LABEL_NAME in ${LABEL_NAMES}; do BRANCH_NAME=`echo "${LABEL_NAME}" | sed -e "s:^${TARGET_LABEL_NAME_PREFIX}::"` # e.g) release/vx.x, main BACKPORT_BRANCH_NAME="${BACKPORT_BRANCH_NAME_PREFIX}/${BRANCH_NAME}/${GITHUB_HEAD_REF}" # e.g) backport/release/vx.x/{current branch name} @@ -74,6 +79,6 @@ jobs: git push origin ${BACKPORT_BRANCH_NAME} gh pr create --base ${BRANCH_NAME} \ - --title "Backport to ${BRANCH_NAME} for ${PR_TITLE}" \ + --title "Backport PR #${PR_NUM} to ${BRANCH_NAME} for ${PR_TITLE}" \ --body "${PR_BODY}" done From 31ebfc777d0f2861a89315f0e6bced08f524fa86 Mon Sep 17 00:00:00 2001 From: vankichi Date: Tue, 20 Feb 2024 11:31:27 +0900 Subject: [PATCH 08/11] :recycle: remove dimension from con-bench Signed-off-by: vankichi --- .../crds/valdbenchmarkjob.yaml | 3 --- .../vald-benchmark-operator/schemas/job-values.yaml | 3 --- .../schemas/scenario-values.yaml | 1 - .../values/benchmark-job.yaml | 1 - .../values/benchmark-scenario.yaml | 8 -------- docs/performance/continuos-benchmatk.md | 1 - internal/config/benchmark.go | 1 - internal/k8s/vald/benchmark/api/v1/job_types.go | 1 - .../benchmark/operator/crds/valdbenchmarkjob.yaml | 3 --- pkg/tools/benchmark/job/service/job.go | 1 - pkg/tools/benchmark/job/service/option.go | 12 ------------ pkg/tools/benchmark/job/service/search.go | 8 -------- pkg/tools/benchmark/job/usecase/benchmarkd.go | 1 - 13 files changed, 44 deletions(-) diff --git a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml index 53367d67c8..b0aad134ae 100644 --- a/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml +++ b/charts/vald-benchmark-operator/crds/valdbenchmarkjob.yaml @@ -277,9 +277,6 @@ spec: - indexes - group - range - dimension: - type: integer - minimum: 1 global_config: type: object properties: diff --git a/charts/vald-benchmark-operator/schemas/job-values.yaml b/charts/vald-benchmark-operator/schemas/job-values.yaml index 8b7de934cd..a619e55d4d 100644 --- a/charts/vald-benchmark-operator/schemas/job-values.yaml +++ b/charts/vald-benchmark-operator/schemas/job-values.yaml @@ -47,9 +47,6 @@ dataset: # @schema {"name": "dataset.url", "type": "string"} # dataset.url -- the dataset url which is used for executing benchmark job with user defined hdf5 file url: "" -# @schema {"name": "dimension", "type": "integer", "minimum": 1} -# dimension -- vector dimension -dimension: 784 # @schema {"name": "replica", "type": "integer", "minimum": 1} # replica -- the number of running concurrency job replica: 1 diff --git a/charts/vald-benchmark-operator/schemas/scenario-values.yaml b/charts/vald-benchmark-operator/schemas/scenario-values.yaml index 2600b42a15..28a1d5786c 100644 --- a/charts/vald-benchmark-operator/schemas/scenario-values.yaml +++ b/charts/vald-benchmark-operator/schemas/scenario-values.yaml @@ -59,7 +59,6 @@ jobs: range: start: 1 end: 1000 - dimension: 784 replica: 1 repetition: 1 job_type: "search" diff --git a/charts/vald-benchmark-operator/values/benchmark-job.yaml b/charts/vald-benchmark-operator/values/benchmark-job.yaml index b11d8dfafa..3a24f856ef 100644 --- a/charts/vald-benchmark-operator/values/benchmark-job.yaml +++ b/charts/vald-benchmark-operator/values/benchmark-job.yaml @@ -27,7 +27,6 @@ spec: start: 1 end: 1000 job_type: "search" - dimension: 784 repetition: 1 replica: 1 rules: [] diff --git a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml index ac9df3aa56..ef018a3ea6 100644 --- a/charts/vald-benchmark-operator/values/benchmark-scenario.yaml +++ b/charts/vald-benchmark-operator/values/benchmark-scenario.yaml @@ -45,7 +45,6 @@ spec: jobs: # @schema {"name": "jobs.items.dataset", "type": "object"} - job_type: "insert" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -62,7 +61,6 @@ spec: health_check_duration: "10s" rps: 500 - job_type: "update" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -79,7 +77,6 @@ spec: health_check_duration: "10s" rps: 500 - job_type: "search" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -94,7 +91,6 @@ spec: health_check_duration: "10s" rps: 2000 - job_type: "upsert" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -111,7 +107,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "search" - dimension: 784 repetition: 2 replica: 1 rules: [] @@ -133,7 +128,6 @@ spec: health_check_duration: "10s" rps: 4000 - job_type: "exists" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -148,7 +142,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "getobject" - dimension: 784 repetition: 1 replica: 1 rules: [] @@ -163,7 +156,6 @@ spec: health_check_duration: "10s" rps: 1000 - job_type: "remove" - dimension: 784 repetition: 1 replica: 1 rules: [] diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index 34a483cc93..41bdab43ee 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -49,7 +49,6 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald | :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :------------------------------------------------------------------------------------------- | | target | \* | target Vald cluster | object | ref: [target](#target-prop) | | dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) | -| dimension | \* | dimension of vector | integer | 784 | | job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search | | repetition | | the number of job repetitions
default: `1` | integer | 1 | | replica | | the number of job concurrent job executions
default: `1` | integer | 2 | diff --git a/internal/config/benchmark.go b/internal/config/benchmark.go index e7e9acb4e4..68272a58bc 100644 --- a/internal/config/benchmark.go +++ b/internal/config/benchmark.go @@ -21,7 +21,6 @@ package config type BenchmarkJob struct { Target *BenchmarkTarget `json:"target,omitempty" yaml:"target"` Dataset *BenchmarkDataset `json:"dataset,omitempty" yaml:"dataset"` - Dimension int `json:"dimension,omitempty" yaml:"dimension"` Replica int `json:"replica,omitempty" yaml:"replica"` Repetition int `json:"repetition,omitempty" yaml:"repetition"` JobType string `json:"job_type,omitempty" yaml:"job_type"` diff --git a/internal/k8s/vald/benchmark/api/v1/job_types.go b/internal/k8s/vald/benchmark/api/v1/job_types.go index 8c82d48e37..cc1fedb018 100644 --- a/internal/k8s/vald/benchmark/api/v1/job_types.go +++ b/internal/k8s/vald/benchmark/api/v1/job_types.go @@ -28,7 +28,6 @@ type BenchmarkJobSpec struct { ServerConfig *config.Servers `json:"server_config,omitempty" yaml:"server_config"` Target *BenchmarkTarget `json:"target,omitempty" yaml:"target"` Dataset *BenchmarkDataset `json:"dataset,omitempty" yaml:"dataset"` - Dimension int `json:"dimension,omitempty" yaml:"dimension"` Replica int `json:"replica,omitempty" yaml:"replica"` Repetition int `json:"repetition,omitempty" yaml:"repetition"` JobType string `json:"job_type,omitempty" yaml:"job_type"` diff --git a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml index 53367d67c8..b0aad134ae 100644 --- a/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml +++ b/k8s/tools/benchmark/operator/crds/valdbenchmarkjob.yaml @@ -277,9 +277,6 @@ spec: - indexes - group - range - dimension: - type: integer - minimum: 1 global_config: type: object properties: diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go index 455d6f1794..905501534c 100644 --- a/pkg/tools/benchmark/job/service/job.go +++ b/pkg/tools/benchmark/job/service/job.go @@ -82,7 +82,6 @@ func (jt jobType) String() string { type job struct { eg errgroup.Group - dimension int dataset *config.BenchmarkDataset jobType jobType jobFunc func(context.Context, chan error) error diff --git a/pkg/tools/benchmark/job/service/option.go b/pkg/tools/benchmark/job/service/option.go index 10b04dbebc..4b0fb7e459 100644 --- a/pkg/tools/benchmark/job/service/option.go +++ b/pkg/tools/benchmark/job/service/option.go @@ -32,23 +32,11 @@ import ( type Option func(j *job) error var defaultOpts = []Option{ - // TODO: set default config for client - WithDimension(784), WithBeforeJobDuration("30s"), WithRPS(1000), WithConcurencyLimit(200), } -// WithDimension sets the vector's dimension for running benchmark job with dataset. -func WithDimension(dim int) Option { - return func(j *job) error { - if dim > 0 { - j.dimension = dim - } - return nil - } -} - // WithInsertConfig sets the insert API config for running insert request job. func WithInsertConfig(c *config.InsertConfig) Option { return func(j *job) error { diff --git a/pkg/tools/benchmark/job/service/search.go b/pkg/tools/benchmark/job/service/search.go index 4128584344..79beaadaf9 100644 --- a/pkg/tools/benchmark/job/service/search.go +++ b/pkg/tools/benchmark/job/service/search.go @@ -66,10 +66,6 @@ func (j *job) search(ctx context.Context, ech chan error) error { } // idx is the modulo, which takes between <0, len(vecs)-1>. idx := (iter - 1) % len(vecs) - if len(vecs[idx]) != j.dimension { - log.Warn("len(vecs) ", len(vecs[iter]), "is not matched with ", j.dimension) - return nil - } res, err := j.client.Search(egctx, &payload.Search_Request{ Vector: vecs[idx], Config: cfg, @@ -121,10 +117,6 @@ func (j *job) search(ctx context.Context, ech chan error) error { log.Debugf("[benchmark job] Start linear search: iter = %d", iter) // idx is the modulo, which takes between <0, len(vecs)-1>. idx := (iter - 1) % len(vecs) - if len(vecs[idx]) != j.dimension { - log.Warn("len(vecs) ", len(vecs[idx]), "is not matched with ", j.dimension) - return nil - } res, err := j.client.LinearSearch(egctx, &payload.Search_Request{ Vector: vecs[idx], Config: cfg, diff --git a/pkg/tools/benchmark/job/usecase/benchmarkd.go b/pkg/tools/benchmark/job/usecase/benchmarkd.go index 173edf6d8b..1036125b2e 100644 --- a/pkg/tools/benchmark/job/usecase/benchmarkd.go +++ b/pkg/tools/benchmark/job/usecase/benchmarkd.go @@ -91,7 +91,6 @@ func New(cfg *config.Config) (r runner.Runner, err error) { service.WithValdClient(vcli), service.WithDataset(cfg.Job.Dataset), service.WithJobTypeByString(cfg.Job.JobType), - service.WithDimension(cfg.Job.Dimension), service.WithInsertConfig(cfg.Job.InsertConfig), service.WithUpdateConfig(cfg.Job.UpdateConfig), service.WithUpsertConfig(cfg.Job.UpsertConfig), From 630bab0bc351127a89f6aa19804afb34b33cce95 Mon Sep 17 00:00:00 2001 From: vankichi Date: Wed, 21 Feb 2024 09:30:14 +0900 Subject: [PATCH 09/11] :pencil: Add sample benchmark manifest Signed-off-by: vankichi --- docs/performance/continuos-benchmatk.md | 4 +- example/helm/benchmark/job-values.yaml | 46 +++++ example/helm/benchmark/scenario-values.yaml | 183 ++++++++++++++++++++ 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 example/helm/benchmark/job-values.yaml create mode 100644 example/helm/benchmark/scenario-values.yaml diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index 41bdab43ee..be67043f25 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -271,8 +271,10 @@ For more details of the configuration of `vald-benchmark-operator-release`, plea kubectl apply -f vbor.yaml ``` +## Running Continuous Benchmarks + After deploy the benchmark operator, you can execute continuous benchmark by applying `ValdBenchmarkScenarioRelease` or `ValdBenchmarkJobRelease`. Please configure designed benchmark and apply by `kubectl` command. -The sample manifests are [here](https://github.com/vdaas/vald/tree/main/charts/vald-benchmark-operator/values). +The sample manifests are [here](https://github.com/vdaas/vald/tree/main/example/helm/benchmark). diff --git a/example/helm/benchmark/job-values.yaml b/example/helm/benchmark/job-values.yaml new file mode 100644 index 0000000000..3df5553bcd --- /dev/null +++ b/example/helm/benchmark/job-values.yaml @@ -0,0 +1,46 @@ +apiVersion: vald.vdaas.org/v1 +kind: ValdBenchmarkJob +metadata: + name: sample-job +spec: + # @schema {"name": "dataset", "type": "object"} + # dataset -- dataset information + dataset: + # @schema {"name": "dataset.name", "type": "string" } + # dataset.name -- the name of dataset + name: "fashion-mnist" + # @schema {"name": "dataset.indexes", "type": "integer"} + # dataset.indexes -- the amount of indexes + indexes: 60000 + # @schema {"name": "dataset.group", "type": "string"} + # dataset.group -- the hdf5 group name of dataset + group: "train" + # @schema {"name": "dataset.range", "type": "object"} + # dataset.range -- the data range of indexes + range: + # @schema {"name": "dataset.range.start", "type": "integer"} + # dataset.range.start -- start index number + start: 1 + # @schema {"name": "dataset.range.end", "type": "integer"} + # dataset.range.end -- end index number + end: 60000 + url: "http://ann-benchmarks.com/fashion-mnist-784-euclidean.hdf5" + job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + client_config: + health_check_duration: "10s" + rps: 2000 + insert_config: + skip_strict_exist_check: false + upsert_config: + skip_strict_exist_check: false + target: + host: "vald-lb-gateway.default.svc.cluster.local" + port: 8081 + global_config: + version: v0.0.1 + time_zone: JST + logging: + level: debug diff --git a/example/helm/benchmark/scenario-values.yaml b/example/helm/benchmark/scenario-values.yaml new file mode 100644 index 0000000000..14a3ea051c --- /dev/null +++ b/example/helm/benchmark/scenario-values.yaml @@ -0,0 +1,183 @@ +apiVersion: vald.vdaas.org/v1 +kind: ValdBenchmarkScenario +metadata: + name: insert-search +spec: + # @schema {"name": "dataset", "type": "object"} + # dataset -- dataset information + dataset: + # @schema {"name": "dataset.name", "type": "string" } + # dataset.name -- the name of dataset + name: "fashion-mnist" + # @schema {"name": "dataset.indexes", "type": "integer"} + # dataset.indexes -- the amount of indexes + indexes: 100000 + # @schema {"name": "dataset.group", "type": "string"} + # dataset.group -- the hdf5 group name of dataset + group: "train" + # @schema {"name": "dataset.range", "type": "object"} + # dataset.range -- the data range of indexes + range: + # @schema {"name": "dataset.range.start", "type": "integer"} + # dataset.range.start -- start index number + start: 1 + # @schema {"name": "dataset.range.end", "type": "integer"} + # dataset.range.end -- end index number + end: 100000 + # @schema {"name": "jobs", "type": "array", "items": {"type": "object"}} + # jobs -- benchmark jobs + jobs: + - job_type: "insert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 10000 + group: "train" + range: + start: 1 + end: 10000 + insert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + upsert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "search" + repetition: 1 + replica: 1 + rules: [] + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 10 + timeout: "1m" + enable_linear_search: false + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "upsert" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 30000 + group: "train" + range: + start: 10001 + end: 40000 + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 5 + timeout: "1m" + upsert_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 2000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "search" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "test" + range: + start: 1 + end: 20000 + search_config: + epsilon: 0.1 + radius: -1 + num: 10 + min_num: 10 + timeout: "1m" + enable_linear_search: false + client_config: + health_check_duration: "10s" + rps: 4000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "exists" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "getobject" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 20000 + group: "train" + range: + start: 1 + end: 20000 + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + - job_type: "remove" + repetition: 1 + replica: 1 + rules: [] + dataset: + name: "fashion-mnist" + indexes: 40000 + group: "train" + range: + start: 1 + end: 40000 + remove_config: + skip_strict_exist_check: true + client_config: + health_check_duration: "10s" + rps: 1000 + concurrency_limit: 150 + ttl_seconds_after_finished: 100 + # @schema {"name": "target", "type": "array", "items": {"type": "object"}} + # target -- target cluster host&port + target: + host: "vald-lb-gateway.default.svc.cluster.local" + port: 8081 From 323aa1e3b7fd154155c716f9ef22389226cf3ca2 Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Wed, 21 Feb 2024 10:52:49 +0900 Subject: [PATCH 10/11] Update docs/performance/continuos-benchmatk.md Co-authored-by: Yusuke Kato Signed-off-by: Kiichiro YUKAWA --- docs/performance/continuos-benchmatk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index be67043f25..e87f9c40fa 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -2,7 +2,7 @@ ## What is the Continuous Benchmark Tool? -Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/365. +Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/7. Assumed use case is: From d0cb8f82db03485c7c2150715220036d7b8e95bf Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Wed, 21 Feb 2024 11:48:36 +0900 Subject: [PATCH 11/11] Update docs/performance/continuos-benchmatk.md Co-authored-by: Hiroto Funakoshi Signed-off-by: Kiichiro YUKAWA --- docs/performance/continuos-benchmatk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/performance/continuos-benchmatk.md b/docs/performance/continuos-benchmatk.md index e87f9c40fa..d5bbdae4ed 100644 --- a/docs/performance/continuos-benchmatk.md +++ b/docs/performance/continuos-benchmatk.md @@ -6,7 +6,7 @@ Continuous Benchmark Tool allows you to get benchmark of Vald cluster in 24/7. Assumed use case is: -- Verification with workload close to production environment +- Verification with workload close to the production environment - Verification before service installation when Vald version up ## Architecture