diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py index ba0fc5b60..77e963921 100644 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ b/benchmarks/perf-tool/okpt/test/steps/factory.py @@ -10,7 +10,7 @@ from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, \ - QueryStep, QueryWithFilterStep, GetStatsStep + QueryStep, QueryWithFilterStep, GetStatsStep, WarmupStep def create_step(step_config: StepConfig) -> Step: @@ -40,5 +40,7 @@ def create_step(step_config: StepConfig) -> Step: return ClearCacheStep(step_config) elif step_config.step_name == GetStatsStep.label: return GetStatsStep(step_config) + elif step_config.step_name == WarmupStep.label: + return WarmupStep(step_config) raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index cc1773330..a8ee36c05 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -162,6 +162,23 @@ def _action(self): def _get_measures(self) -> List[str]: return ['took'] +class WarmupStep(OpenSearchStep): + """See base class.""" + + label = 'warmup_operation' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + self.index_name = parse_string_param('index_name', step_config.config, {}, + None) + + def _action(self): + """Performs warmup operation on an index.""" + warmup_operation(self.endpoint, self.port, self.index_name) + return {} + + def _get_measures(self) -> List[str]: + return ['took'] class TrainModelStep(OpenSearchStep): """See base class.""" @@ -690,6 +707,23 @@ def delete_model(endpoint, port, model_id): return response.json() +def warmup_operation(endpoint, port, index): + """ + Performs warmup operation on index to load native library files + of that index to reduce query latencies. + Args: + endpoint: Endpoint OpenSearch is running on + port: Port OpenSearch is running on + index: index name + Returns: + Deleted model response + """ + response = requests.get('http://' + endpoint + ':' + str(port) + + '/_plugins/_knn/warmup/' + index, + headers={'content-type': 'application/json'}) + return response.json() + + def get_opensearch_client(endpoint: str, port: int, timeout=60): """ Get an opensearch client from an endpoint and port diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw-sqfp16/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw-sqfp16/test.yml index efc737711..50460f60b 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw-sqfp16/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw-sqfp16/test.yml @@ -1,4 +1,4 @@ -endpoint: localhost +endpoint: "localhost" test_name: "Faiss HNSW SQfp16 Test" test_id: "Faiss HNSW SQfp16 Test" num_runs: 10 @@ -20,6 +20,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1