Merge branch 'main' into support_rollover

Signed-off-by: kkewwei <[email protected]>

CHANGELOG.md

@@ -5,14 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
-- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
 - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993))
 - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111))
 - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795))
 - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471))
 - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284))
 
 ### Dependencies
+- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502))
 
 ### Changed
 
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294))
 - Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403))
+- Ensure index templates are not applied to system indices ([#16418](https://github.com/opensearch-project/OpenSearch/pull/16418))
 - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
 
 ### Security

plugins/discovery-gce/build.gradle

@@ -18,7 +18,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0"
+  api "com.google.apis:google-api-services-compute:v1-rev20241015-2.0.0"
   api "com.google.api-client:google-api-client:1.35.2"
   api "com.google.oauth-client:google-oauth-client:1.36.0"
   api "com.google.http-client:google-http-client:${versions.google_http_client}"

plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1

@@ -0,0 +1 @@
+83d293916d59ced480e48fd8c0aefb643e27566c

qa/smoke-test-http/src/test/java/org/opensearch/http/SystemIndexRestIT.java

@@ -123,6 +123,76 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception {
         }
     }
 
+    public void testSystemIndexCreatedWithoutAnyTemplates() throws Exception {
+        // create template
+        {
+            Request templateRequest = new Request("POST", "_component_template/error_mapping_test_template");
+            String jsonBody = "{\n" +
+                "  \"template\": {\n" +
+                "    \"mappings\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"error\" : {\n" +
+                "          \"type\": \"nested\",\n" +
+                "          \"properties\": {\n" +
+                "            \"message\": {\n" +
+                "              \"type\": \"text\"\n" +
+                "            },\n" +
+                "            \"status\": {\n" +
+                "              \"type\": \"integer\"\n" +
+                "            }\n" +
+                "          }\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+
+            templateRequest.setJsonEntity(jsonBody);
+            Response resp = getRestClient().performRequest(templateRequest);
+            assertThat(resp.getStatusLine().getStatusCode(), equalTo(200));
+        }
+
+
+        // apply template to indices
+        {
+            Request applyTemplateRequest = new Request("POST", "_index_template/match_all_test_template");
+            String jsonBody = "{\n" +
+                "  \"index_patterns\": [\n" +
+                "    \"*system-idx*\"\n" +
+                "  ],\n" +
+                "  \"template\": {\n" +
+                "    \"settings\": {}\n" +
+                "  },\n" +
+                "  \"priority\": 10,\n" +
+                "  \"composed_of\": [\n" +
+                "    \"error_mapping_test_template\"\n" +
+                "  ],\n" +
+                "  \"version\": 1\n" +
+                "}";
+
+            applyTemplateRequest.setJsonEntity(jsonBody);
+            Response resp = getRestClient().performRequest(applyTemplateRequest);
+            assertThat(resp.getStatusLine().getStatusCode(), equalTo(200));
+        }
+
+        // create system index - success
+        {
+            Request indexRequest = new Request("PUT", "/" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+            String jsonBody = "{\n" +
+                "  \"mappings\": {\n" +
+                "    \"properties\": {\n" +
+                "      \"error\": {\n" +
+                "        \"type\": \"text\"\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
+            indexRequest.setJsonEntity(jsonBody);
+            Response resp = getRestClient().performRequest(indexRequest);
+            assertThat(resp.getStatusLine().getStatusCode(), equalTo(200));
+        }
+    }
+
     private void assertDeprecationWarningOnAccess(String queryPattern, String warningIndexName) throws IOException {
         String expectedWarning = "this request accesses system indices: [" + warningIndexName + "], but in a " +
             "future major version, direct access to system indices will be prevented by default";

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -434,15 +434,21 @@ public ClusterState applyCreateIndexRequest(
             // in which case templates don't apply, so create the index from the source metadata
             return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata, metadataTransformer);
         } else {
+            // The backing index may have a different name or prefix than the data stream name.
+            final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index();
+
+            // Do not apply any templates to system indices
+            if (systemIndices.isSystemIndex(name)) {
+                return applyCreateIndexRequestWithNoTemplates(currentState, request, silent, metadataTransformer);
+            }
+
             // Hidden indices apply templates slightly differently (ignoring wildcard '*'
             // templates), so we need to check to see if the request is creating a hidden index
             // prior to resolving which templates it matches
             final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings())
                 ? IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings())
                 : null;
 
-            // The backing index may have a different name or prefix than the data stream name.
-            final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index();
             // Check to see if a v2 template matched
             final String v2Template = MetadataIndexTemplateService.findV2Template(
                 currentState.metadata(),
@@ -676,6 +682,17 @@ public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, bo
         tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
     }
 
+    private ClusterState applyCreateIndexRequestWithNoTemplates(
+        final ClusterState currentState,
+        final CreateIndexClusterStateUpdateRequest request,
+        final boolean silent,
+        final BiConsumer<Metadata.Builder, IndexMetadata> metadataTransformer
+    ) throws Exception {
+        // Using applyCreateIndexRequestWithV1Templates with empty list instead of applyCreateIndexRequestWithV2Template
+        // with null template as applyCreateIndexRequestWithV2Template has assertions when template is null
+        return applyCreateIndexRequestWithV1Templates(currentState, request, silent, Collections.emptyList(), metadataTransformer);
+    }
+
     private ClusterState applyCreateIndexRequestWithV1Templates(
         final ClusterState currentState,
         final CreateIndexClusterStateUpdateRequest request,