Add/fix serialization

nielsbauman · Apr 26, 2024 · 79b3b07 · 79b3b07
1 parent 78c79df
commit 79b3b07
Show file tree

Hide file tree

Showing 13 changed files with 232 additions and 143 deletions.
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java
@@ -25,6 +25,7 @@
 public class DataStreamFeatures implements FeatureSpecification {
 
     public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
+    public static final NodeFeature DATA_STREAM_INDICES_EXTRACTION = new NodeFeature("data_stream.indices_extraction");
 
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
@@ -37,7 +38,8 @@ public Set<NodeFeature> getFeatures() {
             DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE,  // Added in 8.12
             LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER,                    // Added in 8.13
             DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE,
-            DataStreamGlobalRetention.GLOBAL_RETENTION                      // Added in 8.14
+            DataStreamGlobalRetention.GLOBAL_RETENTION,                      // Added in 8.14
+            DATA_STREAM_INDICES_EXTRACTION                                   // Added in 8.15
         );
     }
 }
diff --git a/...reams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/...reams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
@@ -159,8 +159,9 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 );
 
                 if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-                    List<Object> failureStoresRepresentation = (List<Object>) dataStreamMap.get(
-                        DataStream.FAILURE_INDICES_FIELD.getPreferredName()
+                    var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
+                    List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(
+                        DataStream.INDICES_FIELD.getPreferredName()
                     );
                     Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
                     assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));
@@ -251,8 +252,9 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 );
 
                 if (DataStream.isFailureStoreFeatureFlagEnabled()) {
-                    List<Object> failureStoresRepresentation = (List<Object>) dataStreamMap.get(
-                        DataStream.FAILURE_INDICES_FIELD.getPreferredName()
+                    var failureStore = (Map<String, Object>) dataStreamMap.get(DataStream.FAILURE_STORE_FIELD.getPreferredName());
+                    List<Object> failureStoresRepresentation = (List<Object>) failureStore.get(
+                        DataStream.INDICES_FIELD.getPreferredName()
                     );
                     Map<String, Object> failureStoreRepresentation = (Map<String, Object>) failureStoresRepresentation.get(0);
                     assertThat(failureStoreRepresentation.get("index_name"), is(failureStoreIndex.getName()));

diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml
@@ -210,8 +210,8 @@ setup:
 ---
 "Create data stream with failure store":
   - requires:
-      cluster_features: ["gte_v8.11.0"]
-      reason: "data stream failure stores only creatable in 8.11+"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
 
   - do:
       allowed_warnings:
@@ -248,9 +248,9 @@ setup:
   - match: { data_streams.0.status: 'GREEN' }
   - match: { data_streams.0.template: 'my-template4' }
   - match: { data_streams.0.hidden: false }
-  - match: { data_streams.0.failure_store: true }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/'}
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/'}
 
   - match: { data_streams.1.name: failure-data-stream2 }
   - match: { data_streams.1.timestamp_field.name: '@timestamp' }
@@ -259,15 +259,15 @@ setup:
   - match: { data_streams.1.indices.0.index_name: '/\.ds-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
   - match: { data_streams.1.template: 'my-template4' }
   - match: { data_streams.1.hidden: false }
-  - match: { data_streams.1.failure_store: true }
-  - length: { data_streams.1.failure_indices: 1 }
-  - match: { data_streams.1.failure_indices.0.index_name: '/\.fs-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.1.failure_store.enabled: true }
+  - length: { data_streams.1.failure_store.indices: 1 }
+  - match: { data_streams.1.failure_store.indices.0.index_name: '/\.fs-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
 
   # save the backing index names for later use
   - set: { data_streams.0.indices.0.index_name: idx0name }
-  - set: { data_streams.0.failure_indices.0.index_name: fsidx0name }
+  - set: { data_streams.0.failure_store.indices.0.index_name: fsidx0name }
   - set: { data_streams.1.indices.0.index_name: idx1name }
-  - set: { data_streams.1.failure_indices.0.index_name: fsidx1name }
+  - set: { data_streams.1.failure_store.indices.0.index_name: fsidx1name }
 
   - do:
       indices.get_mapping:
@@ -538,8 +538,8 @@ setup:
 ---
 "Delete data stream with failure stores":
   - requires:
-      cluster_features: ["gte_v8.12.0"]
-      reason: "data stream failure stores only supported in 8.12+"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
 
   - do:
       allowed_warnings:
@@ -570,7 +570,7 @@ setup:
         name: failure-data-stream1
 
   - set: { data_streams.0.indices.0.index_name: idx0name }
-  - set: { data_streams.0.failure_indices.0.index_name: fs0name }
+  - set: { data_streams.0.failure_store.indices.0.index_name: fs0name }
 
   - do:
       indices.get:
@@ -586,8 +586,8 @@ setup:
   - match: { data_streams.0.generation: 1 }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
 
   - do:
       indices.delete_data_stream:

diff --git a/...eams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/...eams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml
@@ -92,8 +92,8 @@
 ---
 "Modify a data stream's failure store":
   - requires:
-      cluster_features: [ "gte_v8.14.0" ]
-      reason: "this API was released in 8.14.0"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
       test_runner_features: [ "allowed_warnings" ]
 
   - do:
@@ -128,14 +128,14 @@
       indices.get_data_stream:
         name: data-stream-for-modification
   - set: { data_streams.0.indices.0.index_name: write_index }
-  - set: { data_streams.0.failure_indices.0.index_name: first_failure_index }
-  - set: { data_streams.0.failure_indices.1.index_name: write_failure_index }
+  - set: { data_streams.0.failure_store.indices.0.index_name: first_failure_index }
+  - set: { data_streams.0.failure_store.indices.1.index_name: write_failure_index }
 
   - do:
       indices.get_data_stream:
         name: data-stream-for-modification2
   - set: { data_streams.0.indices.0.index_name: second_write_index }
-  - set: { data_streams.0.failure_indices.0.index_name: second_write_failure_index }
+  - set: { data_streams.0.failure_store.indices.0.index_name: second_write_failure_index }
 
   - do:
       index:
@@ -170,11 +170,11 @@
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - match: { data_streams.0.generation: 3 }
   - length: { data_streams.0.indices: 1 }
-  - length: { data_streams.0.failure_indices: 3 }
+  - length: { data_streams.0.failure_store.indices: 3 }
   - match: { data_streams.0.indices.0.index_name: $write_index }
-  - match: { data_streams.0.failure_indices.0.index_name: 'test_index1' }
-  - match: { data_streams.0.failure_indices.1.index_name: $first_failure_index }
-  - match: { data_streams.0.failure_indices.2.index_name: $write_failure_index }
+  - match: { data_streams.0.failure_store.indices.0.index_name: 'test_index1' }
+  - match: { data_streams.0.failure_store.indices.1.index_name: $first_failure_index }
+  - match: { data_streams.0.failure_store.indices.2.index_name: $write_failure_index }
 
   # An index that has an alias is not allowed to be added to failure store
   - do:
@@ -269,10 +269,10 @@
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - match: { data_streams.0.generation: 4 }
   - length: { data_streams.0.indices: 1 }
-  - length: { data_streams.0.failure_indices: 2 }
+  - length: { data_streams.0.failure_store.indices: 2 }
   - match: { data_streams.0.indices.0.index_name: $write_index }
-  - match: { data_streams.0.failure_indices.0.index_name: $first_failure_index }
-  - match: { data_streams.0.failure_indices.1.index_name: $write_failure_index }
+  - match: { data_streams.0.failure_store.indices.0.index_name: $first_failure_index }
+  - match: { data_streams.0.failure_store.indices.1.index_name: $write_failure_index }
 
   - do:
       indices.delete_data_stream:

diff --git a/...c/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/...c/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
@@ -23,8 +23,8 @@ teardown:
 ---
 "Redirect ingest failure in data stream to failure store":
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "data stream failure stores only redirect ingest failures in 8.13+"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
       test_runner_features: [allowed_warnings, contains]
 
   - do:
@@ -74,9 +74,9 @@ teardown:
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - match: { data_streams.0.failure_store: true }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
 
   - do:
       search:
@@ -152,9 +152,9 @@ teardown:
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - match: { data_streams.0.failure_store: true }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
 
   - do:
       search:

diff --git a/.../src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/.../src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
@@ -1,8 +1,8 @@
 ---
 setup:
   - requires:
-      cluster_features: ["gte_v8.14.0"]
-      reason: "data stream failure store rollover only supported in 8.14+"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
       test_runner_features: allowed_warnings
 
   - do:
@@ -48,9 +48,9 @@ setup:
   - match: { data_streams.0.generation: 2 }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - length: { data_streams.0.failure_indices: 2 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - match: { data_streams.0.failure_indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - length: { data_streams.0.failure_store.indices: 2 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
 
 ---
 "Roll over a data stream's failure store with conditions":
@@ -86,9 +86,9 @@ setup:
   - match: { data_streams.0.generation: 2 }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - length: { data_streams.0.failure_indices: 2 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - match: { data_streams.0.failure_indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - length: { data_streams.0.failure_store.indices: 2 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
 
 ---
 "Don't roll over a data stream's failure store when conditions aren't met":
@@ -112,5 +112,5 @@ setup:
   - match: { data_streams.0.generation: 1 }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
diff --git a/.../src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml b/.../src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml
@@ -50,8 +50,8 @@
 ---
 "Put index template with failure store":
   - requires:
-      cluster_features: ["gte_v8.11.0"]
-      reason: "data stream failure stores only creatable in 8.11+"
+      cluster_features: ["data_stream.indices_extraction"]
+      reason: "data stream failure stores REST structure changed in 8.15+"
       test_runner_features: allowed_warnings
 
   - do:
@@ -91,9 +91,9 @@
   - match: { data_streams.0.timestamp_field.name: '@timestamp' }
   - length: { data_streams.0.indices: 1 }
   - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
-  - match: { data_streams.0.failure_store: true }
-  - length: { data_streams.0.failure_indices: 1 }
-  - match: { data_streams.0.failure_indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
 
   - do:
       indices.delete_data_stream:

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -184,6 +184,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION = def(8_643_00_0);
     public static final TransportVersion ESQL_MV_ORDERING_SORTED_ASCENDING = def(8_644_00_0);
     public static final TransportVersion ESQL_PAGE_MAPPING_TO_ITERATOR = def(8_645_00_0);
+    public static final TransportVersion DATA_STREAM_INDICES_EXTRACTION = def(8_646_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,