Deprecate indices without soft-deletes

docs/reference/index-modules/history-retention.asciidoc

@@ -65,6 +65,9 @@ there>>. {ccr-cap} will not function if soft deletes are disabled.
   configured at index creation and only on indices created on or after 6.5.0.
   The default value is `true`.
 
+  deprecated::[7.6, Creating indices with soft-deletes disabled is
+  deprecated and will be removed in future Elasticsearch versions.]
+
 `index.soft_deletes.retention_lease.period`::
 
   The maximum length of time to retain a shard history retention lease before

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -281,7 +282,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
         }
     }
 
-    public void testRecoveryWithSoftDeletes() throws Exception {
+    public void testRecovery() throws Exception {
         final String index = "recover_with_soft_deletes";
         if (CLUSTER_TYPE == ClusterType.OLD) {
             Settings.Builder settings = Settings.builder()
@@ -294,7 +295,7 @@ public void testRecoveryWithSoftDeletes() throws Exception {
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
             if (randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             }
             createIndex(index, settings.build());
             int numDocs = randomInt(10);
@@ -325,7 +326,7 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
                 .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             createIndex(index, settings.build());
             int numDocs = randomInt(10);
             indexDocs(index, 0, numDocs);
@@ -348,7 +349,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                     .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
                 createIndex(index, settings.build());
                 int numDocs = randomInt(10);
                 indexDocs(index, 0, numDocs);
@@ -740,6 +741,26 @@ public void testAutoExpandIndicesDuringRollingUpgrade() throws Exception {
         }
     }
 
+    public void testSoftDeletesDisabledWarning() throws Exception {
+        final String indexName = "test_soft_deletes_disabled_warning";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            boolean softDeletesEnabled = true;
+            Settings.Builder settings = Settings.builder();
+            if (randomBoolean()) {
+                softDeletesEnabled = randomBoolean();
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
+            }
+            Request request = new Request("PUT", "/" + indexName);
+            request.setJsonEntity("{\"settings\": " + Strings.toString(settings.build()) + "}");
+            if (softDeletesEnabled == false) {
+                expectSoftDeletesWarning(request, indexName);
+            }
+            client().performRequest(request);
+        }
+        ensureGreen(indexName);
+        indexDocs(indexName, randomInt(100), randomInt(100));
+    }
+
     @SuppressWarnings("unchecked")
     private Map<String, Object> getIndexSettingsAsMap(String index) throws IOException {
         Map<String, Object> indexSettings = getIndexSettings(index);

rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml

@@ -119,3 +119,20 @@
             properties:
               "":
                 type:     keyword
+
+---
+"Create index without soft deletes":
+  - skip:
+      version:  " - 7.9.99"
+      reason:   "indices without soft deletes are deprecated in 8.0"
+      features: "warnings"
+
+  - do:
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            soft_deletes.enabled: false

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml

@@ -1,11 +1,19 @@
 ---
 "Translog retention without soft_deletes":
+  - skip:
+      version:  " - 7.9.99"
+      reason:   "indices without soft deletes are deprecated in 8.0"
+      features: "warnings"
+
   - do:
       indices.create:
         index: test
         body:
           settings:
             soft_deletes.enabled: false
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
   - do:
       cluster.health:
         wait_for_no_initializing_shards: true
@@ -68,8 +76,8 @@
 ---
 "Translog retention with soft_deletes":
   - skip:
-      version: " - 7.9.99"
-      reason:  "start ignoring translog retention policy with soft-deletes enabled in 8.0"
+      version: " - 7.3.99"
+      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
   - do:
       indices.create:
         index: test
@@ -130,15 +138,20 @@
 ---
 "Translog stats on closed indices without soft-deletes":
   - skip:
-      version: " - 7.2.99"
-      reason:  "closed indices have translog stats starting version 7.3.0"
+      version: " - 7.9.99"
+      reason:  "indices without soft deletes are deprecated in 8.0"
+      features: "warnings"
 
   - do:
       indices.create:
         index: test
         body:
           settings:
             soft_deletes.enabled: false
+      warnings:
+        - Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
+          Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
+
   - do:
       cluster.health:
         wait_for_no_initializing_shards: true
@@ -184,8 +197,8 @@
 ---
 "Translog stats on closed indices with soft-deletes":
   - skip:
-      version: " - 7.9.99"
-      reason:  "start ignoring translog retention policy with soft-deletes enabled in 8.0"
+      version: " - 7.3.99"
+      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
   - do:
       indices.create:
         index: test

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -53,6 +53,7 @@
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -62,6 +63,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -102,6 +104,7 @@
  */
 public class MetaDataCreateIndexService {
     private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class);
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
@@ -434,6 +437,11 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
          * that will be used to create this index.
          */
         MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState);
+        if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
+            DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled",
+                "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
+                "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
+        }
         return indexSettings;
     }
 

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -828,6 +828,21 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() {
         assertThat(targetRoutingNumberOfShards, is(6));
     }
 
+    public void testSoftDeletesDisabledDeprecation() {
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build());
+        aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(),
+            null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. "
+            + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test].");
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        if (randomBoolean()) {
+            request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
+        }
+        aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(),
+            null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+    }
+
     private IndexTemplateMetaData addMatchingTemplate(Consumer<IndexTemplateMetaData.Builder> configurator) {
         IndexTemplateMetaData.Builder builder = templateMetaDataBuilder("template1", "te*");
         configurator.accept(builder);

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -52,6 +52,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotState;
@@ -964,23 +965,27 @@ protected static void ensureNoInitializingShards() throws IOException {
     }
 
     protected static void createIndex(String name, Settings settings) throws IOException {
-        Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + "}");
-        client().performRequest(request);
+        createIndex(name, settings, null);
     }
 
     protected static void createIndex(String name, Settings settings, String mapping) throws IOException {
-        Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
-                + ", \"mappings\" : {" + mapping + "} }");
-        client().performRequest(request);
+        createIndex(name, settings, mapping, null);
     }
 
     protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException {
         Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
-            + ", \"mappings\" : {" + mapping + "}"
-            + ", \"aliases\": {" + aliases + "} }");
+        String entity = "{\"settings\": " + Strings.toString(settings);
+        if (mapping != null) {
+            entity += ",\"mappings\" : {" + mapping + "}";
+        }
+        if (aliases != null) {
+            entity += ",\"aliases\": {" + aliases + "}";
+        }
+        entity += "}";
+        if (settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
+            expectSoftDeletesWarning(request, name);
+        }
+        request.setJsonEntity(entity);
         client().performRequest(request);
     }
 
@@ -999,6 +1004,19 @@ private static void updateIndexSettings(String index, Settings settings) throws
         client().performRequest(request);
     }
 
+    protected static void expectSoftDeletesWarning(Request request, String indexName) {
+        final List<String> expectedWarnings = List.of(
+            "Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
+            "Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "].");
+        if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_8_0_0))) {
+            request.setOptions(RequestOptions.DEFAULT.toBuilder()
+                .setWarningsHandler(warnings -> warnings.equals(expectedWarnings) == false));
+        } else if (nodeVersions.stream().anyMatch(version -> version.onOrAfter(Version.V_8_0_0))) {
+            request.setOptions(RequestOptions.DEFAULT.toBuilder()
+                .setWarningsHandler(warnings -> warnings.isEmpty() == false && warnings.equals(expectedWarnings) == false));
+        }
+    }
+
     protected static Map<String, Object> getIndexSettings(String index) throws IOException {
         Request request = new Request("GET", "/" + index + "/_settings");
         request.addParameter("flat_settings", "true");