From 6bb7c63e98056f09ed795335d031277159507566 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 4 Dec 2024 12:55:41 +0800 Subject: [PATCH] change name and reformat logic Signed-off-by: xinyual --- .../org/opensearch/ml/common/CommonValue.java | 2 +- .../{tools => utils}/AgentModelsSearcher.java | 6 +- .../models/DeleteModelTransportAction.java | 89 +++++++-------- .../ml/plugin/MachineLearningPlugin.java | 2 +- .../DeleteModelTransportActionTests.java | 102 ++++++------------ 5 files changed, 80 insertions(+), 121 deletions(-) rename ml-algorithms/src/main/java/org/opensearch/ml/engine/{tools => utils}/AgentModelsSearcher.java (83%) diff --git a/common/src/main/java/org/opensearch/ml/common/CommonValue.java b/common/src/main/java/org/opensearch/ml/common/CommonValue.java index 60852dcb55..85dc16b7d7 100644 --- a/common/src/main/java/org/opensearch/ml/common/CommonValue.java +++ b/common/src/main/java/org/opensearch/ml/common/CommonValue.java @@ -81,7 +81,7 @@ public class CommonValue { public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words"; public static final Set stopWordsIndices = ImmutableSet.of(".plugins-ml-stop-words"); public static final Integer ML_MEMORY_MESSAGE_INDEX_SCHEMA_VERSION = 1; - public static final String TOOL_MODEL_RELATED_FIELD_PREFIX = "tools.parameters."; + public static final String TOOL_PARAMETERS_PREFIX = "tools.parameters."; public static final String USER_FIELD_MAPPING = " \"" + CommonValue.USER + "\": {\n" diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/tools/AgentModelsSearcher.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/utils/AgentModelsSearcher.java similarity index 83% rename from ml-algorithms/src/main/java/org/opensearch/ml/engine/tools/AgentModelsSearcher.java rename to ml-algorithms/src/main/java/org/opensearch/ml/engine/utils/AgentModelsSearcher.java index 800fcb56b8..0403184bcb 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/tools/AgentModelsSearcher.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/utils/AgentModelsSearcher.java @@ -1,7 +1,7 @@ -package org.opensearch.ml.engine.tools; +package org.opensearch.ml.engine.utils; import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX; -import static org.opensearch.ml.common.CommonValue.TOOL_MODEL_RELATED_FIELD_PREFIX; +import static org.opensearch.ml.common.CommonValue.TOOL_PARAMETERS_PREFIX; import java.util.HashSet; import java.util.Map; @@ -28,7 +28,7 @@ public SearchRequest constructQueryRequest(String candidateModelId) { SearchRequest searchRequest = new SearchRequest(ML_AGENT_INDEX); BoolQueryBuilder shouldQuery = QueryBuilders.boolQuery(); for (String keyField : relatedModelIdSet) { - shouldQuery.should(QueryBuilders.termsQuery(TOOL_MODEL_RELATED_FIELD_PREFIX + keyField, candidateModelId)); + shouldQuery.should(QueryBuilders.termsQuery(TOOL_PARAMETERS_PREFIX + keyField, candidateModelId)); } searchRequest.source(new SearchSourceBuilder().query(shouldQuery)); return searchRequest; diff --git a/plugin/src/main/java/org/opensearch/ml/action/models/DeleteModelTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/models/DeleteModelTransportAction.java index e79ecee40b..564fbb7af6 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/models/DeleteModelTransportAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/models/DeleteModelTransportAction.java @@ -28,11 +28,13 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; +import java.util.function.Supplier; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.OpenSearchStatusException; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; @@ -66,7 +68,8 @@ import org.opensearch.ml.common.transport.model.MLModelDeleteAction; import org.opensearch.ml.common.transport.model.MLModelDeleteRequest; import org.opensearch.ml.common.transport.model.MLModelGetRequest; -import org.opensearch.ml.engine.tools.AgentModelsSearcher; +import org.opensearch.ml.common.utils.StringUtils; +import org.opensearch.ml.engine.utils.AgentModelsSearcher; import org.opensearch.ml.helper.ModelAccessControlHelper; import org.opensearch.ml.utils.RestActionUtils; import org.opensearch.search.SearchHit; @@ -284,48 +287,27 @@ private void checkAgentBeforeDeleteModel(String modelId, ActionListener } private void checkIngestPipelineBeforeDeleteModel(String modelId, ActionListener actionListener) { - GetPipelineRequest getPipelineRequest = new GetPipelineRequest(); - client.execute(GetPipelineAction.INSTANCE, getPipelineRequest, ActionListener.wrap(ingestPipelineResponse -> { - List allDependentPipelineIds = findDependentPipelines( - ingestPipelineResponse.pipelines(), - modelId, - org.opensearch.ingest.PipelineConfiguration::getConfigAsMap, - org.opensearch.ingest.PipelineConfiguration::getId - ); - if (allDependentPipelineIds.isEmpty()) { - actionListener.onResponse(true); - } else { - actionListener - .onFailure( - new OpenSearchStatusException( - String - .format( - Locale.ROOT, - "%d ingest pipelines are still using this model, please delete or update the pipelines first: %s", - allDependentPipelineIds.size(), - Arrays.toString(allDependentPipelineIds.toArray(new String[0])) - ), - RestStatus.CONFLICT - ) - ); - } - }, e -> { - log.error("Failed to delete ML Model: " + modelId, e); - actionListener.onFailure(e); - - })); + checkPipelineBeforeDeleteModel(modelId, actionListener, "ingest", GetPipelineRequest::new, GetPipelineAction.INSTANCE); } private void checkSearchPipelineBeforeDeleteModel(String modelId, ActionListener actionListener) { - GetSearchPipelineRequest getSearchPipelineRequest = new GetSearchPipelineRequest(); - client.execute(GetSearchPipelineAction.INSTANCE, getSearchPipelineRequest, ActionListener.wrap(searchPipelineResponse -> { - List allDependentPipelineIds = findDependentPipelines( - searchPipelineResponse.pipelines(), - modelId, - org.opensearch.search.pipeline.PipelineConfiguration::getConfigAsMap, - org.opensearch.search.pipeline.PipelineConfiguration::getId - ); + checkPipelineBeforeDeleteModel(modelId, actionListener, "search", GetSearchPipelineRequest::new, GetSearchPipelineAction.INSTANCE); + + } + + private void checkPipelineBeforeDeleteModel( + String modelId, + ActionListener actionListener, + String pipelineType, + Supplier requestSupplier, + ActionType actionType + ) { + ActionRequest request = requestSupplier.get(); + client.execute(actionType, request, ActionListener.wrap(pipelineResponse -> { + String responseString = pipelineResponse.toString(); + Map allConfigMap = StringUtils.fromJson(pipelineResponse.toString(), ""); + List allDependentPipelineIds = findDependentPipelinesEasy(allConfigMap, modelId); if (allDependentPipelineIds.isEmpty()) { actionListener.onResponse(true); } else { @@ -335,8 +317,9 @@ private void checkSearchPipelineBeforeDeleteModel(String modelId, ActionListener String .format( Locale.ROOT, - "%d search pipelines are still using this model, please delete or update the pipelines first: %s", + "%d %s pipelines are still using this model, please delete or update the pipelines first: %s", allDependentPipelineIds.size(), + pipelineType, Arrays.toString(allDependentPipelineIds.toArray(new String[0])) ), RestStatus.CONFLICT @@ -479,6 +462,18 @@ private Boolean isModelNotDeployed(MLModelState mlModelState) { && !mlModelState.equals(MLModelState.PARTIALLY_DEPLOYED); } + private List findDependentPipelinesEasy(Map allConfigMap, String candidateModelId) { + List dependentPipelineConfigurations = new ArrayList<>(); + for (Map.Entry entry : allConfigMap.entrySet()) { + String id = entry.getKey(); + Map config = (Map) entry.getValue(); + if (searchThroughConfig(config, candidateModelId)) { + dependentPipelineConfigurations.add(id); + } + } + return dependentPipelineConfigurations; + } + private List findDependentPipelines( List pipelineConfigurations, String candidateModelId, @@ -533,24 +528,22 @@ private Boolean searchThroughConfig(Object searchCandidate, String candidateId) } private String formatAgentErrorMessage(SearchHit[] hits) { - boolean isHidden = false; List agentIds = new ArrayList<>(); for (SearchHit hit : hits) { Map sourceAsMap = hit.getSourceAsMap(); - isHidden = isHidden || Boolean.parseBoolean((String) sourceAsMap.getOrDefault(MLAgent.IS_HIDDEN_FIELD, false)); - agentIds.add(hit.getId()); - } - if (isHidden) { - return String - .format(Locale.ROOT, "%d agents are still using this model, please delete or update the agents first", hits.length); + Boolean isHidden = (Boolean) sourceAsMap.getOrDefault(MLAgent.IS_HIDDEN_FIELD, false); + if (!isHidden) { + agentIds.add(hit.getId()); + } } return String .format( Locale.ROOT, - "%d agents are still using this model, please delete or update the agents first: %s", + "%d agents are still using this model, please delete or update the agents first, all visible agents are: %s", hits.length, Arrays.toString(agentIds.toArray(new String[0])) ); + } // this method is only to stub static method. diff --git a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java index 26cd1a4b75..34e8012da9 100644 --- a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java +++ b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java @@ -178,7 +178,6 @@ import org.opensearch.ml.engine.indices.MLInputDatasetHandler; import org.opensearch.ml.engine.memory.ConversationIndexMemory; import org.opensearch.ml.engine.memory.MLMemoryManager; -import org.opensearch.ml.engine.tools.AgentModelsSearcher; import org.opensearch.ml.engine.tools.AgentTool; import org.opensearch.ml.engine.tools.CatIndexTool; import org.opensearch.ml.engine.tools.ConnectorTool; @@ -186,6 +185,7 @@ import org.opensearch.ml.engine.tools.MLModelTool; import org.opensearch.ml.engine.tools.SearchIndexTool; import org.opensearch.ml.engine.tools.VisualizationsTool; +import org.opensearch.ml.engine.utils.AgentModelsSearcher; import org.opensearch.ml.helper.ConnectorAccessControlHelper; import org.opensearch.ml.helper.ModelAccessControlHelper; import org.opensearch.ml.memory.ConversationalMemoryHandler; diff --git a/plugin/src/test/java/org/opensearch/ml/action/models/DeleteModelTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/models/DeleteModelTransportActionTests.java index 47fe9eeb6d..10519a1de3 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/models/DeleteModelTransportActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/models/DeleteModelTransportActionTests.java @@ -21,10 +21,8 @@ import static org.opensearch.ml.common.CommonValue.ML_MODEL_INDEX; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -57,9 +55,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -72,7 +68,8 @@ import org.opensearch.ml.common.agent.MLAgent; import org.opensearch.ml.common.model.MLModelState; import org.opensearch.ml.common.transport.model.MLModelDeleteRequest; -import org.opensearch.ml.engine.tools.AgentModelsSearcher; +import org.opensearch.ml.common.utils.StringUtils; +import org.opensearch.ml.engine.utils.AgentModelsSearcher; import org.opensearch.ml.helper.ModelAccessControlHelper; import org.opensearch.ml.model.MLModelManager; import org.opensearch.search.SearchHit; @@ -194,21 +191,18 @@ public void testDeleteModel_Success() throws IOException { } public void testDeleteModel_BlockedBySearchPipelineAndIngestionPipeline() throws IOException { - when(searchPipelineConfiguration.getId()).thenReturn("search_1"); - when(searchPipelineConfiguration.getConfigAsMap()).thenReturn(configDataMap); - when(getSearchPipelineResponse.pipelines()).thenReturn(List.of(searchPipelineConfiguration)); + when(getSearchPipelineResponse.toString()).thenReturn( + StringUtils.toJson(Map.of("search_1", configDataMap)) + ); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getSearchPipelineResponse); return null; }).when(client).execute(eq(GetSearchPipelineAction.INSTANCE), any(), any()); - org.opensearch.ingest.PipelineConfiguration ingestPipelineConfiguration = new org.opensearch.ingest.PipelineConfiguration( - "ingest_1", - new BytesArray("{\"model_id\": \"test_id\"}".getBytes(StandardCharsets.UTF_8)), - MediaTypeRegistry.JSON + when(getIngestionPipelineResponse.toString()).thenReturn( + StringUtils.toJson(Map.of("ingest_1", Map.of("model_id", "test_id"))) ); - when(getIngestionPipelineResponse.pipelines()).thenReturn(List.of(ingestPipelineConfiguration)); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getIngestionPipelineResponse); @@ -227,15 +221,10 @@ public void testDeleteModel_BlockedBySearchPipelineAndIngestionPipeline() throws public void testDeleteModel_BlockedBySearchPipeline() throws IOException { //org.opensearch.search.pipeline.PipelineConfiguration pipelineConfiguration = new PipelineConfiguration(); - when(searchPipelineConfiguration.getId()).thenReturn("1"); - when(searchPipelineConfiguration.getConfigAsMap()).thenReturn(configDataMap); + when(getSearchPipelineResponse.toString()).thenReturn( + StringUtils.toJson(Map.of("search_1", configDataMap, "indenpendent_search", Map.of("nothing", "nothinh"))) + ); - org.opensearch.search.pipeline.PipelineConfiguration independentSearchPipelineConfiguration = mock(org.opensearch.search.pipeline.PipelineConfiguration.class); - Map independentConfigMap = new HashMap<>(); - independentConfigMap.put("nothing", "nothing"); - when(independentSearchPipelineConfiguration.getConfigAsMap()).thenReturn(independentConfigMap); - when(independentSearchPipelineConfiguration.getId()).thenReturn("2"); - when(getSearchPipelineResponse.pipelines()).thenReturn(List.of(searchPipelineConfiguration, independentSearchPipelineConfiguration)); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getSearchPipelineResponse); @@ -245,24 +234,15 @@ public void testDeleteModel_BlockedBySearchPipeline() throws IOException { deleteModelTransportAction.doExecute(null, mlModelDeleteRequest, actionListener); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); verify(actionListener).onFailure(argumentCaptor.capture()); - assertEquals("1 search pipelines are still using this model, please delete or update the pipelines first: [1]", argumentCaptor.getValue().getMessage()); + assertEquals("1 search pipelines are still using this model, please delete or update the pipelines first: [search_1]", argumentCaptor.getValue().getMessage()); } public void testDeleteModel_BlockedBySearchPipelineSingleModelId() throws IOException { Map configDataMapWithSingleModelId = Map .of("model_id", "test_id", "list_model_id", List.of("test_id"), "test_map_id", Map.of("map_model_id", "test_id")); - when(searchPipelineConfiguration.getId()).thenReturn("search_1"); - when(searchPipelineConfiguration.getConfigAsMap()).thenReturn(configDataMapWithSingleModelId); - org.opensearch.search.pipeline.PipelineConfiguration independentSearchPipelineConfiguration = mock( - org.opensearch.search.pipeline.PipelineConfiguration.class - ); - Map independentConfigMap = new HashMap<>(); - independentConfigMap.put("nothing", "nothing"); - when(independentSearchPipelineConfiguration.getConfigAsMap()).thenReturn(independentConfigMap); - when(independentSearchPipelineConfiguration.getId()).thenReturn("2"); - when(getSearchPipelineResponse.pipelines()) - .thenReturn(List.of(searchPipelineConfiguration, independentSearchPipelineConfiguration)); + when(getSearchPipelineResponse.toString()).thenReturn(StringUtils.toJson(Map.of("search_1", configDataMapWithSingleModelId))); + doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getSearchPipelineResponse); @@ -279,20 +259,9 @@ public void testDeleteModel_BlockedBySearchPipelineSingleModelId() throws IOExce } public void testDeleteModel_BlockedBySearchPipelineListModelId() throws IOException { - Map configDataMapWithSingleModelId = Map + Map configDataMapWithListModelId = Map .of("single_model_id", "test_id", "model_id", List.of("test_id"), "test_map_id", Map.of("map_model_id", "test_id")); - when(searchPipelineConfiguration.getId()).thenReturn("search_1"); - when(searchPipelineConfiguration.getConfigAsMap()).thenReturn(configDataMapWithSingleModelId); - - org.opensearch.search.pipeline.PipelineConfiguration independentSearchPipelineConfiguration = mock( - org.opensearch.search.pipeline.PipelineConfiguration.class - ); - Map independentConfigMap = new HashMap<>(); - independentConfigMap.put("nothing", "nothing"); - when(independentSearchPipelineConfiguration.getConfigAsMap()).thenReturn(independentConfigMap); - when(independentSearchPipelineConfiguration.getId()).thenReturn("2"); - when(getSearchPipelineResponse.pipelines()) - .thenReturn(List.of(searchPipelineConfiguration, independentSearchPipelineConfiguration)); + when(getSearchPipelineResponse.toString()).thenReturn(StringUtils.toJson(Map.of("search_1", configDataMapWithListModelId))); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getSearchPipelineResponse); @@ -309,21 +278,12 @@ public void testDeleteModel_BlockedBySearchPipelineListModelId() throws IOExcept } public void testDeleteModel_BlockedByIngestPipeline() throws IOException { - org.opensearch.ingest.PipelineConfiguration ingestPipelineConfiguration = new org.opensearch.ingest.PipelineConfiguration( - "ingest_1", - new BytesArray("{\"model_id\": \"test_id\"}".getBytes(StandardCharsets.UTF_8)), - MediaTypeRegistry.JSON - ); - - org.opensearch.ingest.PipelineConfiguration independentIngestPipelineConfiguration = - new org.opensearch.ingest.PipelineConfiguration( - "2", - new BytesArray("{\"nothing\": \"test_id\"}".getBytes(StandardCharsets.UTF_8)), - MediaTypeRegistry.JSON - ); - - when(getIngestionPipelineResponse.pipelines()) - .thenReturn(List.of(ingestPipelineConfiguration, independentIngestPipelineConfiguration)); + Map ingestPipelineConfig1 = Map.of("model_id", "test_id"); + Map ingestPipelineConfig2 = Map.of("nothing", "test_id"); + when(getIngestionPipelineResponse.toString()) + .thenReturn(StringUtils.toJson(Map.of("ingest_1", ingestPipelineConfig1, "ingest_2", ingestPipelineConfig2))); + // when(getIngestionPipelineResponse.pipelines()) + // .thenReturn(List.of(ingestPipelineConfiguration, independentIngestPipelineConfiguration)); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getIngestionPipelineResponse); @@ -357,7 +317,7 @@ public void testDeleteModel_BlockedByAgent() throws IOException { ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); verify(actionListener).onFailure(argumentCaptor.capture()); assertEquals( - "1 agents are still using this model, please delete or update the agents first: [1]", + "1 agents are still using this model, please delete or update the agents first, all visible agents are: [1]", argumentCaptor.getValue().getMessage() ); } @@ -365,10 +325,15 @@ public void testDeleteModel_BlockedByAgent() throws IOException { public void testDeleteModel_BlockedByHiddenAgent() throws IOException { XContentBuilder content = XContentBuilder.builder(XContentType.JSON.xContent()); content.startObject(); - content.field(MLAgent.IS_HIDDEN_FIELD, "true"); + content.field(MLAgent.IS_HIDDEN_FIELD, true); content.endObject(); SearchHit hit = new SearchHit(1, "1", null, null).sourceRef(BytesReference.bytes(content)); - SearchHits searchHits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); + XContentBuilder content2 = XContentBuilder.builder(XContentType.JSON.xContent()); + content2.startObject(); + content2.field(MLAgent.IS_HIDDEN_FIELD, false); + content2.endObject(); + SearchHit hit2 = new SearchHit(2, "2", null, null).sourceRef(BytesReference.bytes(content2)); + SearchHits searchHits = new SearchHits(new SearchHit[] { hit, hit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); when(searchResponse.getHits()).thenReturn(searchHits); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(1); @@ -380,7 +345,7 @@ public void testDeleteModel_BlockedByHiddenAgent() throws IOException { ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); verify(actionListener).onFailure(argumentCaptor.capture()); assertEquals( - "1 agents are still using this model, please delete or update the agents first", + "2 agents are still using this model, please delete or update the agents first, all visible agents are: [2]", argumentCaptor.getValue().getMessage() ); } @@ -818,8 +783,8 @@ private void prepare() throws IOException { emptyBulkByScrollResponse = new BulkByScrollResponse(new ArrayList<>(), null); SearchHits hits = new SearchHits(new SearchHit[] {}, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0.0f); when(searchResponse.getHits()).thenReturn(hits); - when(getIngestionPipelineResponse.pipelines()).thenReturn(List.of()); - + // when(getIngestionPipelineResponse.pipelines()).thenReturn(List.of()); + when(getIngestionPipelineResponse.toString()).thenReturn(StringUtils.toJson(Map.of())); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(1); listener.onResponse(searchResponse); @@ -838,7 +803,8 @@ private void prepare() throws IOException { return null; }).when(client).execute(eq(GetPipelineAction.INSTANCE), any(), any()); - when(getSearchPipelineResponse.pipelines()).thenReturn(List.of()); + // when(getSearchPipelineResponse.pipelines()).thenReturn(List.of()); + when(getSearchPipelineResponse.toString()).thenReturn(StringUtils.toJson(Map.of())); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(2); listener.onResponse(getSearchPipelineResponse);