diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java index bbdd06fe1..1ba9a1825 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java @@ -99,6 +99,7 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (acknowledgedResponse.isAcknowledged() == false) { log.warn("Upserting component template not ack'd!"); } + boolean updateConflictingTemplate = false; // Find template which matches input index best String templateName = MetadataIndexTemplateService.findV2Template( @@ -106,12 +107,39 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { normalizeIndexName(indexName), false ); + // If we find conflicting templates(regardless of priority) and that template was created by us, + // we will silently update index_pattern of that template. + // Otherwise, we will fail since we don't want to change index_pattern of user created index template + Map> conflictingTemplates = + MetadataIndexTemplateService.findConflictingV2Templates( + state, + computeIndexTemplateName(indexName), + List.of(computeIndexPattern(indexName)) + ); + + // If there is 1 conflict with our own template, we will update that template's index_pattern field + if (conflictingTemplates.size() == 1) { + String conflictingTemplateName = conflictingTemplates.keySet().iterator().next(); + if (conflictingTemplateName.startsWith(OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX)) { + templateName = conflictingTemplateName; + updateConflictingTemplate = true; + } + } + + if (templateName == null && conflictingTemplates.size() > 0) { + String errorMessage = "Found conflicting templates: [" + + String.join(", ", conflictingTemplates.keySet()) + "]"; + log.error(errorMessage); + actionListener.onFailure(SecurityAnalyticsException.wrap(new IllegalStateException(errorMessage))); + return; + } + String componentName = computeComponentTemplateName(indexName); ComposableIndexTemplate template; if (templateName == null) { template = new ComposableIndexTemplate( - List.of(indexName.endsWith("*") == false ? indexName + "*": indexName), + List.of(computeIndexPattern(indexName)), null, List.of(componentName), null, @@ -123,10 +151,18 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { template = state.metadata().templatesV2().get(templateName); // Check if we need to append our component to composedOf list if (template.composedOf().contains(componentName) == false) { - List newComposedOf = new ArrayList<>(template.composedOf()); - newComposedOf.add(componentName); + List newComposedOf; + List indexPatterns; + if (updateConflictingTemplate) { + newComposedOf = new ArrayList<>(template.composedOf()); + newComposedOf.add(componentName); + indexPatterns = List.of(computeIndexPattern(indexName)); + } else { + newComposedOf = List.of(componentName); + indexPatterns = template.indexPatterns(); + } template = new ComposableIndexTemplate( - template.indexPatterns(), + indexPatterns, template.template(), newComposedOf, template.priority(), @@ -155,6 +191,10 @@ public void onFailure(Exception e) { } + private String computeIndexPattern(String indexName) { + return indexName.endsWith("*") == false ? indexName + "*" : indexName; + } + private void upsertIndexTemplate( IndicesAdminClient indicesClient, boolean create, diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index d27494942..553905565 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -248,7 +248,7 @@ public void getMappingAction(String indexName, ActionListener() { @Override public void onResponse(String concreteIndex) { - doGetMappingAction(concreteIndex, actionListener); + doGetMappingAction(indexName, concreteIndex, actionListener); } @Override @@ -263,14 +263,13 @@ public void onFailure(Exception e) { } } - public void doGetMappingAction(String indexName, ActionListener actionListener) { - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(indexName); + public void doGetMappingAction(String indexName, String concreteIndexName, ActionListener actionListener) { + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(concreteIndexName); indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override public void onResponse(GetMappingsResponse getMappingsResponse) { try { - // Extract indexName and MappingMetadata - String indexName = getMappingsResponse.mappings().iterator().next().key; + // Extract MappingMetadata MappingMetadata mappingMetadata = getMappingsResponse.mappings().iterator().next().value; // List of all found applied aliases on index List appliedAliases = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index dfc691030..c096fa7c0 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -1386,6 +1386,10 @@ protected void createComponentTemplateWithMappings(String componentTemplateName, } protected void createComposableIndexTemplate(String templateName, List indexPatterns, String componentTemplateName, boolean isDatastream) throws IOException { + createComposableIndexTemplate(templateName, indexPatterns, componentTemplateName, isDatastream, 0); + } + + protected void createComposableIndexTemplate(String templateName, List indexPatterns, String componentTemplateName, boolean isDatastream, int priority) throws IOException { String body = "{\n" + (isDatastream ? "\"data_stream\": { }," : "") + @@ -1393,7 +1397,8 @@ protected void createComposableIndexTemplate(String templateName, List i indexPatterns.stream().collect( Collectors.joining(",", "\"", "\"")) + " ]," + - "\"composed_of\": [\"" + componentTemplateName + "\"]" + + "\"composed_of\": [\"" + componentTemplateName + "\"]," + + "\"priority\":" + priority + "}"; Response response = makeRequest( client(), diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index d639e070b..c518a11a2 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -4,8 +4,10 @@ */ package org.opensearch.securityanalytics.mapper; +import java.util.ArrayList; import java.util.Collections; import java.util.Optional; +import java.util.Set; import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; @@ -34,9 +36,30 @@ import java.util.Map; import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; + public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { + public void testGetMappingSuccess() throws IOException { + String testIndexName1 = "my_index_1"; + String testIndexName2 = "my_index_2"; + String testIndexPattern = "my_index*"; + + createSampleIndex(testIndexName1); + createSampleIndex(testIndexName2); + + createMappingsAPI(testIndexName2, "netflow"); + + Request request = new Request("GET", MAPPER_BASE_URI + "?index_name=" + testIndexPattern); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response); + // Assert that indexName returned is one passed by user + assertTrue(respMap.containsKey(testIndexPattern)); + } + public void testCreateMappingSuccess() throws IOException { String testIndexName = "my_index"; @@ -270,7 +293,7 @@ public void testGetMappingsViewSuccess() throws IOException { createSampleIndex(testIndexName); - // Execute CreateMappingsAction to add alias mapping for index + // Execute GetMappingsViewAction to add alias mapping for index Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); // both req params and req body are supported request.addParameter("index_name", testIndexName); @@ -293,11 +316,311 @@ public void testGetMappingsViewSuccess() throws IOException { assertEquals(3, unmappedFieldAliases.size()); } + public void testCreateMappings_withDatastream_success() throws IOException { + String datastream = "test_datastream"; + + String datastreamMappings = "\"properties\": {" + + " \"@timestamp\":{ \"type\": \"date\" }," + + " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + + " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + + "}"; + + createSampleDatastream(datastream, datastreamMappings); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(datastream, "netflow"); + + // Verify mappings + Map props = getIndexMappingsAPIFlat(datastream); + assertEquals(5, props.size()); + assertTrue(props.containsKey("@timestamp")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + + // Verify that index template applied mappings + Response response = makeRequest(client(), "POST", datastream + "/_rollover", Collections.emptyMap(), null); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + // Insert doc to index to add additional fields to mapping + String sampleDoc = "{" + + " \"@timestamp\":\"2023-01-06T00:05:00\"," + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(datastream, "2", sampleDoc); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(datastream, "netflow"); + + String writeIndex = getDatastreamWriteIndex(datastream); + + // Verify mappings + props = getIndexMappingsAPIFlat(writeIndex); + assertEquals(9, props.size()); + assertTrue(props.containsKey("@timestamp")); + assertTrue(props.containsKey("netflow.source_ipv4_address")); + assertTrue(props.containsKey("netflow.source_transport_port")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("source.port")); + + // Get applied mappings + props = getIndexMappingsSAFlat(datastream); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("source.port")); + + deleteDatastreamAPI(datastream); + } + + public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexName3 = "test_index_3"; + + String indexPattern = "test_index*"; + + String componentTemplateMappings = "\"properties\": {" + + " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + + " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + + "}"; + + // Setup index_template + createComponentTemplateWithMappings( + IndexTemplateManager.computeComponentTemplateName(indexPattern), + componentTemplateMappings + ); + + createComposableIndexTemplate( + IndexTemplateManager.computeIndexTemplateName(indexPattern), + List.of(indexPattern), + IndexTemplateManager.computeComponentTemplateName(indexPattern), + false + ); + + createIndex(indexName1, Settings.EMPTY, null); + + // Execute CreateMappingsAction to apply alias mappings - index template should be updated + createMappingsAPI(indexPattern, "netflow"); + + // Create new index to verify that index template is updated + createIndex(indexName2, Settings.EMPTY, null); + + // Verify that template applied mappings + Map props = getIndexMappingsAPIFlat(indexName2); + assertEquals(4, props.size()); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + + // Verify our GetIndexMappings -- applied mappings + props = getIndexMappingsSAFlat(indexPattern); + assertEquals(2, props.size()); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + + + // Insert doc to index to add additional fields to mapping + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(indexName2, "1", sampleDoc); + + // Call CreateMappings API and expect index template to be updated with 2 additional aliases + createMappingsAPI(indexPattern, "netflow"); + + // Create new index to verify that index template was updated correctly + createIndex(indexName3, Settings.EMPTY, null); + + // Verify mappings + props = getIndexMappingsAPIFlat(indexName3); + assertEquals(8, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("netflow.source_transport_port")); + assertTrue(props.containsKey("netflow.source_ipv4_address")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + // Verify our GetIndexMappings -- applied mappings + props = getIndexMappingsSAFlat(indexPattern); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + } + + public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexPattern, "netflow"); + } + + public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexName3 = "test_index_3"; + String indexName4 = "test_index_4"; + + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc1 = "{" + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + "}"; + + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc1); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexPattern, "netflow"); + + // Verify that index template is up + createIndex(indexName3, Settings.EMPTY, null); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("GET", indexName3 + "/_mapping"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response).get(indexName3); + + MappingsTraverser mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); + Map flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); + // Verify mappings + Map props = (Map) flatMappings.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + String sampleDoc2 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(indexName3, "1", sampleDoc2); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexPattern, "netflow"); + + // Verify that index template is updated + createIndex(indexName4, Settings.EMPTY, null); + + // Execute CreateMappingsAction to add alias mapping for index + request = new Request("GET", indexName4 + "/_mapping"); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + respMap = (Map) responseAsMap(response).get(indexName4); + + mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); + flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); + // Verify mappings + props = (Map) flatMappings.get("properties"); + assertEquals(8, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("netflow.source_transport_port")); + assertTrue(props.containsKey("netflow.source_ipv4_address")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + // Verify applied mappings + props = getIndexMappingsSAFlat(indexName4); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + } + + public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + try { + createMappingsAPI(indexPattern, "netflow"); + fail("expected 500 failure!"); + } catch (ResponseException e) { + assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getResponse().getStatusLine().getStatusCode()); + } + + } + public void testGetMappingsView_index_pattern_two_indices_Success() throws IOException { - String testIndexName1 = "get_mappings_view_index11"; - String testIndexName2 = "get_mappings_view_index22"; - String indexPattern = "get_mappings_view_index*"; + String testIndexName1 = "get_mappings_view_index111"; + String testIndexName2 = "get_mappings_view_index122"; + String testIndexName3 = "get_mappings_view_index"; + + String indexPattern = "get_mappings_view_index1*"; + String indexPattern2 = "get_mappings_view_index*"; + createSampleIndex(testIndexName1); createSampleIndex(testIndexName2); indexDoc(testIndexName2, "987654", "{ \"extra_field\": 12345 }"); @@ -519,83 +842,139 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } - public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; + public void testCreateMappings_withIndexPattern_conflictingTemplates_success() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); client().performRequest(new Request("POST", "_refresh")); - // Insert sample docs - String sampleDoc1 = "{" + + // Insert sample doc + String sampleDoc = "{" + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + + " \"netflow.destination_transport_port\":1234" + "}"; + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction with first index pattern + createMappingsAPI(indexPattern1, "netflow"); + + createIndex(indexName3, Settings.EMPTY, null); + + // Insert sample doc String sampleDoc2 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + + " \"netflow.source_transport_port\":4444" + "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); - client().performRequest(new Request("POST", "_refresh")); + indexDoc(indexName3, "1", sampleDoc2); - // Execute CreateMappingsAction to add alias mapping for index - Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - request.setJsonEntity( - "{ \"index_name\":\"" + indexPattern + "\"," + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true" + - "}" - ); - Response response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + // Execute CreateMappingsAction with conflicting index pattern - expect template to be updated + createMappingsAPI(indexPattern2, "netflow"); + + createIndex(indexName4, Settings.EMPTY, null); + // Verify with GET _mapping + Map props = getIndexMappingsAPIFlat(indexName4); + assertEquals(8, props.size()); + // Verify with SA's GetIndexMappings + props = getIndexMappingsSAFlat(indexName4); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); } - public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; + public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; createIndex(indexName1, Settings.EMPTY, null); createIndex(indexName2, Settings.EMPTY, null); client().performRequest(new Request("POST", "_refresh")); - // Insert sample docs - String sampleDoc1 = "{" + + // Insert sample doc + String sampleDoc = "{" + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + + " \"netflow.destination_transport_port\":1234" + "}"; - String sampleDoc2 = "{" + - " \"netflow11.destination33_transport_port\":1234," + - " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction with first index pattern + createMappingsAPI(indexPattern1, "netflow"); + + // User-create template with conflicting pattern but higher priority + createComponentTemplateWithMappings("user_component_template", "\"properties\": { \"some_field\": { \"type\": \"long\" } }"); + createComposableIndexTemplate("user_custom_template", List.of("test_index_111111*"), "user_component_template", false, 100); + + // Execute CreateMappingsAction and expect 2 conflicting templates and failure + try { + createMappingsAPI(indexPattern2, "netflow"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Found conflicting templates: [user_custom_template, .opensearch-sap-alias-mappings-index-template-test_index_1]")); + } + } + + public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234" + "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); client().performRequest(new Request("POST", "_refresh")); - // Execute CreateMappingsAction to add alias mapping for index - Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - request.setJsonEntity( - "{ \"index_name\":\"" + indexPattern + "\"," + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true" + - "}" - ); - Response response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + // User-create template with conflicting pattern but higher priority + createComponentTemplateWithMappings("user_component_template", "\"properties\": { \"some_field\": { \"type\": \"long\" } }"); + createComposableIndexTemplate("user_custom_template", List.of("test_index_111111*"), "user_component_template", false, 100); + + // Execute CreateMappingsAction and expect conflict with 1 user template + try { + createMappingsAPI(indexPattern2, "netflow"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Found conflicting templates: [user_custom_template]")); + } } - public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { + + public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { String indexName1 = "test_index_1"; String indexName2 = "test_index_2"; String indexPattern = "test_index*"; @@ -611,7 +990,12 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I " \"netflow.destination_transport_port\":1234," + " \"netflow.source_transport_port\":4444" + "}"; + String sampleDoc2 = "{" + + " \"netflow11.destination33_transport_port\":1234," + + " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + + "}"; indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); client().performRequest(new Request("POST", "_refresh")); @@ -624,13 +1008,8 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I " \"partial\":true" + "}" ); - try { - client().performRequest(request); - fail("expected 500 failure!"); - } catch (ResponseException e) { - assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getResponse().getStatusLine().getStatusCode()); - } - + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } private void createSampleIndex(String indexName) throws IOException { @@ -991,4 +1370,88 @@ public void testCreateDNSMapping() throws IOException{ assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + + public void testTraverseAndCopy() { + + try { + String indexName = "my_test_index"; + + String indexMappingJSON = "" + + " \"properties\": {" + + " \"netflow.event_data.SourceAddress\": {" + + " \"type\": \"ip\"" + + " }," + + " \"type\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event_data.DestinationPort\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event.stop\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event.start\": {" + + " \"type\": \"long\"" + + " }," + + " \"plain1\": {" + + " \"type\": \"integer\"" + + " }," + + " \"user\":{" + + " \"type\":\"nested\"," + + " \"properties\":{" + + " \"first\":{" + + " \"type\":\"long\"" + + " }," + + " \"last\":{" + + " \"type\":\"text\"," + + " \"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"; + + createIndex(indexName, Settings.EMPTY, indexMappingJSON); + + Map mappings = getIndexMappingsAPI(indexName); + + MappingsTraverser mappingsTraverser; + + mappingsTraverser = new MappingsTraverser(mappings, Set.of()); + + // Copy specific paths from mappings + Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter( + List.of("netflow.event_data.SourceAddress", "netflow.event.stop", "plain1", "user.first", "user.last") + ); + + // Now traverse filtered mapppings to confirm only copied paths are present + List paths = new ArrayList<>(); + mappingsTraverser = new MappingsTraverser(filteredMappings, Set.of()); + mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() { + @Override + public void onLeafVisited(MappingsTraverser.Node node) { + paths.add(node.currentPath); + } + + @Override + public void onError(String error) { + fail("Failed traversing valid mappings"); + } + }); + mappingsTraverser.traverse(); + assertEquals(5, paths.size()); + assertTrue(paths.contains("user.first")); + assertTrue(paths.contains("user.last")); + assertTrue(paths.contains("plain1")); + assertTrue(paths.contains("netflow.event.stop")); + assertTrue(paths.contains("netflow.event_data.SourceAddress")); + + } catch (IOException e) { + fail("Error instantiating MappingsTraverser with JSON string as mappings"); + } + } }