From 65903abdfa15326420932108aec60e8010d43fa2 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 8 Oct 2020 14:58:28 -0600 Subject: [PATCH 001/107] Add plugin name to SystemIndexPlugin --- .../org/elasticsearch/kibana/KibanaPlugin.java | 5 +++++ .../org/elasticsearch/http/SystemIndexRestIT.java | 5 +++++ .../cluster/ClusterInfoServiceIT.java | 5 +++++ .../org/elasticsearch/indices/SystemIndices.java | 2 +- .../elasticsearch/plugins/SystemIndexPlugin.java | 2 ++ .../elasticsearch/indices/SystemIndicesTests.java | 2 +- .../xpack/async/AsyncResultsIndexPlugin.java | 5 +++++ .../core/LocalStateCompositeXPackPlugin.java | 4 +--- .../elasticsearch/xpack/enrich/EnrichPlugin.java | 5 +++++ .../elasticsearch/xpack/logstash/Logstash.java | 5 +++++ .../elasticsearch/xpack/ml/MachineLearning.java | 5 +++++ .../LocalStateSearchableSnapshots.java | 8 +++++++- ...earchableSnapshotsSystemIndicesIntegTests.java | 5 +++++ .../searchablesnapshots/SearchableSnapshots.java | 5 +++++ .../elasticsearch/xpack/security/Security.java | 15 ++++++++++----- .../elasticsearch/xpack/transform/Transform.java | 5 +++++ .../org/elasticsearch/xpack/watcher/Watcher.java | 5 +++++ 17 files changed, 77 insertions(+), 11 deletions(-) diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 186da3f827fcb..8fe6913a3782e 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -75,6 +75,11 @@ public Collection getSystemIndexDescriptors(Settings sett .collect(Collectors.toUnmodifiableList()); } + @Override + public String getPluginName() { + return "kibana"; + } + @Override public List getRestHandlers( Settings settings, diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java index 12bbd12485853..5088614abc700 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -143,6 +143,11 @@ public Collection getSystemIndexDescriptors(Settings sett return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); } + @Override + public String getPluginName() { + return SystemIndexRestIT.class.getSimpleName(); + } + public static class AddDocRestHandler extends BaseRestHandler { @Override public boolean allowSystemIndexAccessByDefault() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index eda8de8c41c1a..c9f99704b6473 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -92,6 +92,11 @@ public List getActionFilters() { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']')); } + + @Override + public String getPluginName() { + return ClusterInfoServiceIT.class.getSimpleName(); + } } public static class BlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple { diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index ae4a64111a588..78ea911e8ca34 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -48,7 +48,7 @@ */ public class SystemIndices { private static final Map> SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( - TaskResultsService.class.getName(), List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index")) + TaskResultsService.class.getSimpleName(), List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index")) ); private final CharacterRunAutomaton runAutomaton; diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 821f141b3eabc..5007da185c3b3 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -40,4 +40,6 @@ public interface SystemIndexPlugin extends ActionPlugin { default Collection getSystemIndexDescriptors(Settings settings) { return Collections.emptyList(); } + + String getPluginName(); } diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java index 439df84e184b2..83420fc703e2c 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java @@ -95,7 +95,7 @@ public void testBuiltInSystemIndices() { public void testPluginCannotOverrideBuiltInSystemIndex() { Map> pluginMap = Map.of( - TaskResultsService.class.getName(), List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) + TaskResultsService.class.getSimpleName(), List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SystemIndices(pluginMap)); assertThat(e.getMessage(), containsString("plugin or module attempted to define the same source")); diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java index faa1628cfbd68..18f2db149769c 100644 --- a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -48,6 +48,11 @@ public Collection getSystemIndexDescriptors(Settings sett return Collections.singletonList(new SystemIndexDescriptor(XPackPlugin.ASYNC_RESULTS_INDEX, this.getClass().getSimpleName())); } + @Override + public String getPluginName() { + return "async_search"; + } + @Override public Collection createComponents( Client client, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 54555c2639968..dc6e3a59a7ce4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -64,7 +64,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestController; @@ -102,8 +101,7 @@ import static java.util.stream.Collectors.toList; public class LocalStateCompositeXPackPlugin extends XPackPlugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, - ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin, IndexStorePlugin, - SystemIndexPlugin { + ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin, IndexStorePlugin { private XPackLicenseState licenseState; private SSLService sslService; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 239681a6cd0c0..1f85735016290 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -236,4 +236,9 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(ENRICH_INDEX_PATTERN, "Contains data to support enrich ingest processors.") ); } + + @Override + public String getPluginName() { + return "enrich"; + } } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 1491fff3313a2..5351206089d1c 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -100,4 +100,9 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management") ); } + + @Override + public String getPluginName() { + return "logstash_management"; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 552225a41eb7a..6919615458e7f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1094,6 +1094,11 @@ public Collection getSystemIndexDescriptors(Settings sett ); } + @Override + public String getPluginName() { + return "machine_learning"; + } + @Override public BreakerSettings getCircuitBreaker(Settings settings) { return BreakerSettings.updateFromSettings( diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java index 8140663b3f856..89df64ca42223 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java @@ -9,12 +9,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.nio.file.Path; import java.util.Collection; -public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin { +public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin implements SystemIndexPlugin { private final SearchableSnapshots plugin; @@ -35,4 +36,9 @@ protected XPackLicenseState getLicenseState() { public Collection getSystemIndexDescriptors(Settings settings) { return plugin.getSystemIndexDescriptors(settings); } + + @Override + public String getPluginName() { + return plugin.getPluginName(); + } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java index 34ea6e15464ba..a84cd78809e10 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java @@ -91,5 +91,10 @@ public static class TestSystemIndexPlugin extends Plugin implements SystemIndexP public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new SystemIndexDescriptor(INDEX_NAME, "System index for [" + getTestClass().getName() + ']')); } + + @Override + public String getPluginName() { + return SearchableSnapshotsSystemIndicesIntegTests.class.getSimpleName(); + } } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index a67c14b521c32..22edef5e037d6 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -222,6 +222,11 @@ public Collection getSystemIndexDescriptors(Settings sett return List.of(new SystemIndexDescriptor(SNAPSHOT_BLOB_CACHE_INDEX, "Contains cached data of blob store repositories")); } + @Override + public String getPluginName() { + return "searchable_snapshots"; + } + @Override public Map getDirectoryFactories() { return Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 4474e401ecbaf..13b4178d70f1e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -104,9 +104,9 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction; +import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlInvalidateSessionAction; import org.elasticsearch.xpack.core.security.action.saml.SamlLogoutAction; -import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutAction; import org.elasticsearch.xpack.core.security.action.saml.SamlPrepareAuthenticationAction; import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction; @@ -170,9 +170,9 @@ import org.elasticsearch.xpack.security.action.rolemapping.TransportGetRoleMappingsAction; import org.elasticsearch.xpack.security.action.rolemapping.TransportPutRoleMappingAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlAuthenticateAction; +import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlLogoutAction; -import org.elasticsearch.xpack.security.action.saml.TransportSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.action.saml.TransportSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.action.token.TransportCreateTokenAction; import org.elasticsearch.xpack.security.action.token.TransportInvalidateTokenAction; @@ -214,8 +214,8 @@ import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; -import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction; +import org.elasticsearch.xpack.security.rest.action.apikey.RestClearApiKeyCacheAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction; import org.elasticsearch.xpack.security.rest.action.apikey.RestGrantApiKeyAction; @@ -239,9 +239,9 @@ import org.elasticsearch.xpack.security.rest.action.rolemapping.RestGetRoleMappingsAction; import org.elasticsearch.xpack.security.rest.action.rolemapping.RestPutRoleMappingAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlAuthenticateAction; +import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlInvalidateSessionAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlLogoutAction; -import org.elasticsearch.xpack.security.rest.action.saml.RestSamlCompleteLogoutAction; import org.elasticsearch.xpack.security.rest.action.saml.RestSamlPrepareAuthenticationAction; import org.elasticsearch.xpack.security.rest.action.user.RestChangePasswordAction; import org.elasticsearch.xpack.security.rest.action.user.RestDeleteUserAction; @@ -250,8 +250,8 @@ import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction; import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction; import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction; -import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry; +import org.elasticsearch.xpack.security.support.ExtensionComponents; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.support.SecurityStatusChangeListener; import org.elasticsearch.xpack.security.transport.SecurityHttpSettings; @@ -1145,4 +1145,9 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data") ); } + + @Override + public String getPluginName() { + return "security"; + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 37b09cc04cf5c..2ae982b376457 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -374,4 +374,9 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data") ); } + + @Override + public String getPluginName() { + return "transform"; + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 36c3902c24c10..3cb1a697a4ee0 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -703,4 +703,9 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution") ); } + + @Override + public String getPluginName() { + return "watcher"; + } } From 6d32a6522a3dfda380ee8329429ee76672a5afad Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 8 Oct 2020 15:02:22 -0600 Subject: [PATCH 002/107] Use plugin name for system index descriptor map --- server/src/main/java/org/elasticsearch/node/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index f08a6ebcf24d4..80f64c9dd9a60 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -480,7 +480,7 @@ protected Node(final Environment initialEnvironment, .filterPlugins(SystemIndexPlugin.class) .stream() .collect(Collectors.toUnmodifiableMap( - plugin -> plugin.getClass().getSimpleName(), + plugin -> plugin.getPluginName(), plugin -> plugin.getSystemIndexDescriptors(settings))); final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap); From 37e4f4b4a79ee8fca501c662f252a541b3db1f08 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 8 Oct 2020 16:02:27 -0600 Subject: [PATCH 003/107] Add pluginStates to create and restore snapshot requests --- .../create/CreateSnapshotRequest.java | 48 +++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 48 ++++++++++++++++++- .../create/CreateSnapshotRequestTests.java | 12 +++++ .../restore/RestoreSnapshotRequestTests.java | 12 +++++ 4 files changed, 119 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 01a30bb819f8e..63a834ce18439 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -64,6 +64,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest MAXIMUM_METADATA_BYTES) { validationException = addValidationError("metadata must be smaller than 1024 bytes, but was [" + metadataSize + "]", @@ -348,6 +360,28 @@ public CreateSnapshotRequest userMetadata(Map userMetadata) { return this; } + /** + * @return Which plugin states should be included in the snapshot + */ + public String[] pluginStates() { + return pluginStates; + } + + /** + * @param pluginStates The plugin states to be included in the snapshot + */ + public CreateSnapshotRequest pluginStates(String[] pluginStates) { + this.pluginStates = pluginStates; + return this; + } + + /** + * @param pluginStates The plugin states to be included in the snapshot + */ + public CreateSnapshotRequest pluginStates(List pluginStates) { + return pluginStates(pluginStates.toArray(new String[pluginStates.size()])); + } + /** * Parses snapshot definition. * @@ -366,6 +400,12 @@ public CreateSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } + } else if (name.equals("plugin_states")) { + if (entry.getValue() instanceof List) { + pluginStates((List) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed plugin_states section, should be an array of strings"); + } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); } else if (name.equals("include_global_state")) { @@ -391,6 +431,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(index); } builder.endArray(); + builder.startArray("plugin_states"); + for (String plugin : pluginStates) { + builder.value(plugin); + } + builder.endArray(); builder.field("partial", partial); builder.field("include_global_state", includeGlobalState); if (indicesOptions != null) { @@ -418,6 +463,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && + Arrays.equals(pluginStates, that.pluginStates) && Objects.equals(masterNodeTimeout, that.masterNodeTimeout) && Objects.equals(userMetadata, that.userMetadata); } @@ -427,6 +473,7 @@ public int hashCode() { int result = Objects.hash(snapshot, repository, indicesOptions, partial, includeGlobalState, waitForCompletion, userMetadata); result = 31 * result + Arrays.hashCode(indices); + result = 31 * result + Arrays.hashCode(pluginStates); return result; } @@ -437,6 +484,7 @@ public String toString() { ", repository='" + repository + '\'' + ", indices=" + (indices == null ? null : Arrays.asList(indices)) + ", indicesOptions=" + indicesOptions + + ", pluginStates=" + Arrays.asList(pluginStates) + ", partial=" + partial + ", includeGlobalState=" + includeGlobalState + ", waitForCompletion=" + waitForCompletion + diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index bd1e84cdff58b..e68103d1d1444 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -39,6 +39,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.PLUGIN_STATES_VERSION; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; @@ -53,6 +54,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest pluginStates) { + return pluginStates(pluginStates.toArray(new String[pluginStates.size()])); + } + /** * Parses restore definition * @@ -455,7 +488,7 @@ public String snapshotUuid() { * @return this request */ @SuppressWarnings("unchecked") - public RestoreSnapshotRequest source(Map source) { + public RestoreSnapshotRequest source(Map source) { // GWB> Handle pluginStates NOCOMMIT for (Map.Entry entry : source.entrySet()) { String name = entry.getKey(); if (name.equals("indices")) { @@ -466,6 +499,12 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } + } else if (name.equals("plugin_states")) { + if (entry.getValue() instanceof List) { + pluginStates((List) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed plugin_states section, should be an array of strings"); + } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); } else if (name.equals("include_global_state")) { @@ -524,6 +563,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } + builder.startArray("plugin_states"); + for (String plugin : pluginStates) { + builder.value(plugin); + } + builder.endArray(); builder.field("include_global_state", includeGlobalState); builder.field("partial", partial); builder.field("include_aliases", includeAliases); @@ -561,6 +605,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && + Arrays.equals(pluginStates, that.pluginStates) && Objects.equals(renamePattern, that.renamePattern) && Objects.equals(renameReplacement, that.renameReplacement) && Objects.equals(indexSettings, that.indexSettings) && @@ -574,6 +619,7 @@ public int hashCode() { includeGlobalState, partial, includeAliases, indexSettings, snapshotUuid); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); + result = 31 * result + Arrays.hashCode(pluginStates); return result; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 3214bad7e0b7a..930bd7a167c51 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -66,6 +66,18 @@ public void testToXContent() throws IOException { original.indices(indices); } + if (randomBoolean()) { + List plugins = new ArrayList<>(); + int count = randomInt(3) + 1; + + for (int i = 0; i < count; ++i) { + plugins.add(randomAlphaOfLength(randomInt(3) + 2)); + } + + original.pluginStates(plugins); + } + + if (randomBoolean()) { original.partial(randomBoolean()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index f78607061df49..5aea0a0c88c4b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -52,6 +52,18 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { instance.indices(indices); } + + if (randomBoolean()) { + List plugins = new ArrayList<>(); + int count = randomInt(3) + 1; + + for (int i = 0; i < count; ++i) { + plugins.add(randomAlphaOfLength(randomInt(3) + 2)); + } + + instance.pluginStates(plugins); + } + if (randomBoolean()) { instance.renamePattern(randomUnicodeOfLengthBetween(1, 100)); } From 1fefb26832bdc9db7b16a29155011d2a7bd2b4ad Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 8 Oct 2020 16:03:32 -0600 Subject: [PATCH 004/107] Plumb system index descriptor map into SnapshotsService --- server/src/main/java/org/elasticsearch/node/Node.java | 3 ++- .../org/elasticsearch/snapshots/SnapshotsService.java | 9 +++++++-- .../elasticsearch/snapshots/SnapshotResiliencyTests.java | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 80f64c9dd9a60..0328110a69de9 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -568,7 +568,8 @@ protected Node(final Environment initialEnvironment, RepositoriesService repositoryService = repositoriesModule.getRepositoryService(); repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService(settings, clusterService, - clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters()); + clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters(), + systemIndexDescriptorMap); SnapshotShardsService snapshotShardsService = new SnapshotShardsService(settings, clusterService, repositoryService, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index e408ee99f317d..9eff940a50596 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -47,7 +47,6 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; import org.elasticsearch.cluster.SnapshotsInProgress.ShardState; import org.elasticsearch.cluster.SnapshotsInProgress.State; @@ -79,12 +78,14 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -170,6 +171,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final OngoingRepositoryOperations repositoryOperations = new OngoingRepositoryOperations(); + private final Map> systemIndexDescriptorMap; + /** * Setting that specifies the maximum number of allowed concurrent snapshot create and delete operations in the * cluster state. The number of concurrent operations in a cluster state is defined as the sum of the sizes of @@ -181,7 +184,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private volatile int maxConcurrentOperations; public SnapshotsService(Settings settings, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, - RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters) { + RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters, + Map> systemIndexDescriptorMap) { this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.repositoriesService = repositoriesService; @@ -198,6 +202,7 @@ public SnapshotsService(Settings settings, ClusterService clusterService, IndexN clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, i -> maxConcurrentOperations = i); } + this.systemIndexDescriptorMap = systemIndexDescriptorMap; } /** diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index fbe84963b71ed..ab6b0990e9390 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1487,7 +1487,7 @@ public void onFailure(final Exception e) { ); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService(settings, clusterService, indexNameExpressionResolver, repositoriesService, - transportService, actionFilters); + transportService, actionFilters, null); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); From 1869e915cbb56693c40e387c3526e8ad68f76c2e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 8 Oct 2020 16:06:37 -0600 Subject: [PATCH 005/107] Remove unnecessary comment --- .../admin/cluster/snapshots/restore/RestoreSnapshotRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index e68103d1d1444..e5d402f85e5c5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -488,7 +488,7 @@ public RestoreSnapshotRequest pluginStates(List pluginStates) { * @return this request */ @SuppressWarnings("unchecked") - public RestoreSnapshotRequest source(Map source) { // GWB> Handle pluginStates NOCOMMIT + public RestoreSnapshotRequest source(Map source) { for (Map.Entry entry : source.entrySet()) { String name = entry.getKey(); if (name.equals("indices")) { From febcdd778d22c12df9e2f543462bac761e3f84ba Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 12 Oct 2020 16:47:52 -0600 Subject: [PATCH 006/107] Only include pluginStates in request XContent if non-empty --- .../snapshots/create/CreateSnapshotRequest.java | 10 ++++++---- .../snapshots/restore/RestoreSnapshotRequest.java | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 63a834ce18439..ac0d67c57b882 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -431,11 +431,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(index); } builder.endArray(); - builder.startArray("plugin_states"); - for (String plugin : pluginStates) { - builder.value(plugin); + if (pluginStates != null && pluginStates.length != 0) { + builder.startArray("plugin_states"); + for (String plugin : pluginStates) { + builder.value(plugin); + } + builder.endArray(); } - builder.endArray(); builder.field("partial", partial); builder.field("include_global_state", includeGlobalState); if (indicesOptions != null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index e5d402f85e5c5..357a1c8316f52 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -563,11 +563,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } - builder.startArray("plugin_states"); - for (String plugin : pluginStates) { - builder.value(plugin); + if (pluginStates != null && pluginStates.length != 0) { + builder.startArray("plugin_states"); + for (String plugin : pluginStates) { + builder.value(plugin); + } + builder.endArray(); } - builder.endArray(); builder.field("include_global_state", includeGlobalState); builder.field("partial", partial); builder.field("include_aliases", includeAliases); From c16e472c26582a49babf4c2444a725650e6aef51 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 13 Oct 2020 16:33:21 -0600 Subject: [PATCH 007/107] Add plugin states to SnapshotInfo --- .../get/TransportGetSnapshotsAction.java | 3 +- .../elasticsearch/snapshots/SnapshotInfo.java | 136 +++++++++++++++--- .../snapshots/SnapshotsService.java | 6 +- .../create/CreateSnapshotResponseTests.java | 5 +- .../get/GetSnapshotsResponseTests.java | 5 +- .../BlobStoreRepositoryRestoreTests.java | 18 ++- .../snapshots/SnapshotInfoTests.java | 60 ++++---- ...ckEventuallyConsistentRepositoryTests.java | 6 +- .../AbstractSnapshotIntegTestCase.java | 2 +- .../xpack/ccr/repository/CcrRepository.java | 7 +- .../SourceOnlySnapshotShardTests.java | 6 +- .../SnapshotRetentionConfigurationTests.java | 27 +--- .../xpack/slm/SnapshotLifecycleTaskTests.java | 9 +- .../xpack/slm/SnapshotRetentionTaskTests.java | 41 +++--- 14 files changed, 226 insertions(+), 105 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 1656253378242..f06a610218137 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -289,7 +289,8 @@ private static List buildSimpleSnapshotInfos(final Set for (SnapshotId snapshotId : toResolve) { final List indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList()); CollectionUtil.timSort(indices); - snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), repositoryData.getSnapshotState(snapshotId))); + snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), null, + repositoryData.getSnapshotState(snapshotId))); } CollectionUtil.timSort(snapshotInfos); return Collections.unmodifiableList(snapshotInfos); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index b76bd48193f38..5e866e3c1e863 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.common.Nullable; @@ -29,8 +30,10 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; @@ -80,6 +83,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; private static final String USER_METADATA = "metadata"; + private static final String PLUGIN_STATES = "plugin_states"; private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); @@ -173,7 +177,7 @@ public SnapshotInfo build() { } return new SnapshotInfo(snapshotId, indices, dataStreams, snapshotState, reason, version, startTime, endTime, - totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata); + totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata, null); } } @@ -236,6 +240,8 @@ int getSuccessfulShards() { private final List dataStreams; + private final List pluginStates; + private final long startTime; private final long endTime; @@ -255,30 +261,35 @@ int getSuccessfulShards() { private final List shardFailures; - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state) { - this(snapshotId, indices, dataStreams, state, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null); + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + SnapshotState state) { + this(snapshotId, indices, dataStreams, state, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null, pluginStates); } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state, Version version) { - this(snapshotId, indices, dataStreams, state, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null); + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + Version version, SnapshotState state) { + this(snapshotId, indices, dataStreams, state, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null, pluginStates); } public SnapshotInfo(SnapshotsInProgress.Entry entry) { this(entry.snapshot().getSnapshotId(), entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), SnapshotState.IN_PROGRESS, - null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata()); + null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), + null); // TODO: Add pluginStates to SnapshotInProgress.Entry } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, long startTime, String reason, - long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, - Map userMetadata) { + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + String reason, long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, + Map userMetadata, long startTime) { this(snapshotId, indices, dataStreams, snapshotState(reason, shardFailures), reason, Version.CURRENT, - startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata); + startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata, + pluginStates); } SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state, String reason, - Version version, long startTime, long endTime, int totalShards, int successfulShards, - List shardFailures, Boolean includeGlobalState, Map userMetadata) { + Version version, long startTime, long endTime, int totalShards, int successfulShards, + List shardFailures, Boolean includeGlobalState, Map userMetadata, + List pluginStates) { this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.dataStreams = Collections.unmodifiableList(Objects.requireNonNull(dataStreams)); @@ -292,6 +303,7 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da this.shardFailures = Objects.requireNonNull(shardFailures); this.includeGlobalState = includeGlobalState; this.userMetadata = userMetadata; + this.pluginStates = pluginStates; } /** @@ -311,6 +323,11 @@ public SnapshotInfo(final StreamInput in) throws IOException { includeGlobalState = in.readOptionalBoolean(); userMetadata = in.readMap(); dataStreams = in.readStringList(); + if (in.getVersion().before(CreateSnapshotRequest.PLUGIN_STATES_VERSION)) { + pluginStates = null; + } else { + pluginStates = in.readList(SnapshotPluginInfo::new); + } } /** @@ -318,7 +335,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { * all information stripped out except the snapshot id, state, and indices. */ public SnapshotInfo basic() { - return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), state); + return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), null, state); } /** @@ -473,6 +490,7 @@ public String toString() { ", includeGlobalState=" + includeGlobalState + ", version=" + version + ", shardFailures=" + shardFailures + + ", pluginStates=" + pluginStates + '}'; } @@ -551,6 +569,13 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SUCCESSFUL, successfulShards); builder.endObject(); } + if (verbose || pluginStates != null) { + builder.startArray(PLUGIN_STATES); + for (SnapshotPluginInfo snapshotPluginInfo : pluginStates) { + builder.value(snapshotPluginInfo); + } + builder.endArray(); + } builder.endObject(); return builder; } @@ -588,6 +613,12 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final shardFailure.toXContent(builder, params); } builder.endArray(); + builder.startArray(PLUGIN_STATES); + for (SnapshotPluginInfo snapshotPluginInfo : pluginStates) { + builder.value(snapshotPluginInfo); + } + builder.endArray(); + builder.endObject(); return builder; } @@ -612,6 +643,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr Boolean includeGlobalState = null; Map userMetadata = null; List shardFailures = Collections.emptyList(); + List pluginStates = null; if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -666,6 +698,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser)); } shardFailures = Collections.unmodifiableList(shardFailureArrayList); + } else if (PLUGIN_STATES.equals(currentFieldName)) { + ArrayList snapshotPluginInfoArrayList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + snapshotPluginInfoArrayList.add(SnapshotPluginInfo.fromXContent(parser)); + } + pluginStates = Collections.unmodifiableList(snapshotPluginInfoArrayList); } else { // It was probably created by newer version - ignoring parser.skipChildren(); @@ -697,7 +735,8 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr successfulShards, shardFailures, includeGlobalState, - userMetadata); + userMetadata, + pluginStates); } @Override @@ -725,6 +764,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalBoolean(includeGlobalState); out.writeMap(userMetadata); out.writeStringCollection(dataStreams); + if (pluginStates != null) { + out.writeList(pluginStates); + } } private static SnapshotState snapshotState(final String reason, final List shardFailures) { @@ -756,12 +798,74 @@ public boolean equals(Object o) { Objects.equals(includeGlobalState, that.includeGlobalState) && Objects.equals(version, that.version) && Objects.equals(shardFailures, that.shardFailures) && - Objects.equals(userMetadata, that.userMetadata); + Objects.equals(userMetadata, that.userMetadata) && + Objects.equals(pluginStates, that.pluginStates); } @Override public int hashCode() { return Objects.hash(snapshotId, state, reason, indices, dataStreams, startTime, endTime, - totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata); + totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata, + pluginStates); + } + + private static class SnapshotPluginInfo implements Writeable, ToXContentObject { + final String pluginName; + final List indices; + + static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = + new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { + String pluginName = (String) a[0]; + List indices = (List) a[1]; + return new SnapshotPluginInfo(pluginName, indices); + }); + + static { + SNAPSHOT_PLUGIN_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); + SNAPSHOT_PLUGIN_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); + } + + private SnapshotPluginInfo(String pluginName, List indices) { + this.pluginName = pluginName; + this.indices = indices; + } + + SnapshotPluginInfo(final StreamInput in) throws IOException { + this.pluginName = in.readString(); + this.indices = in.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(pluginName); + out.writeStringCollection(indices); + } + + public static SnapshotPluginInfo fromXContent(XContentParser parser) throws IOException { + return SNAPSHOT_PLUGIN_INFO_PARSER.parse(parser, null); + } + + public String getPluginName() { + return pluginName; + } + + public List getIndices() { + return indices; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("plugin_name", pluginName); + builder.startArray("indices"); + for (String index : indices) { + builder.value(index); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d2fb37890cc8f..232aceb68f5af 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1155,9 +1155,9 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), - entry.startTime(), failure, threadPool.absoluteTimeInMillis(), - entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, - entry.includeGlobalState(), entry.userMetadata()); + null, failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), + shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime() + ); final StepListener metadataListener = new StepListener<>(); final Repository repo = repositoriesService.repository(snapshot.getRepository()); if (entry.isClone()) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java index 520f1dcce9434..d862cf6def9e1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java @@ -70,8 +70,9 @@ protected CreateSnapshotResponse createTestInstance() { boolean globalState = randomBoolean(); return new CreateSnapshotResponse( - new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, - globalState, SnapshotInfoTests.randomUserMetadata())); + new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, endTime, totalShards, shardFailures, globalState, + SnapshotInfoTests.randomUserMetadata(), startTime + )); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 7aea18bb6305e..d7f03d9fdc35e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -82,8 +82,9 @@ private List createSnapshotInfos() { ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), - System.currentTimeMillis(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), - SnapshotInfoTests.randomUserMetadata())); + null, reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), + SnapshotInfoTests.randomUserMetadata(), System.currentTimeMillis() + )); } return snapshots; diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 078647dc52de5..4612054890376 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -182,9 +182,21 @@ public void testSnapshotWithConflictingName() throws Exception { shardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), - new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream() - .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, 6, - Collections.emptyList(), true, Collections.emptyMap()), + new SnapshotInfo( + snapshot.getSnapshotId(), + shardGenerations.indices().stream() + .map(IndexId::getName) + .collect(Collectors.toList()), + Collections.emptyList(), + null, + null, + 1L, + 6, + Collections.emptyList(), + true, + Collections.emptyMap(), + 0L + ), Version.CURRENT, Function.identity(), f)); IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java index 8c5eb8c695e0b..999cb33c5ac63 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java @@ -59,8 +59,9 @@ protected SnapshotInfo createTestInstance() { Map userMetadata = randomUserMetadata(); - return new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, - includeGlobalState, userMetadata); + return new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, endTime, totalShards, shardFailures, includeGlobalState, + userMetadata, startTime + ); } @Override @@ -75,29 +76,36 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { SnapshotId snapshotId = new SnapshotId( randomValueOtherThan(instance.snapshotId().getName(), () -> randomAlphaOfLength(5)), randomValueOtherThan(instance.snapshotId().getUUID(), () -> randomAlphaOfLength(5))); - return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), instance.startTime(), instance.reason(), + return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + instance.userMetadata(), instance.startTime() + ); case 1: int indicesSize = randomValueOtherThan(instance.indices().size(), () -> randomIntBetween(1, 10)); List indices = Arrays.asList(randomArray(indicesSize, indicesSize, String[]::new, () -> randomAlphaOfLengthBetween(2, 20))); - return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), instance.startTime(), instance.reason(), + return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + instance.userMetadata(), instance.startTime() + ); case 2: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - randomValueOtherThan(instance.startTime(), ESTestCase::randomNonNegativeLong), instance.reason(), - instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + instance.includeGlobalState(), instance.userMetadata(), randomValueOtherThan(instance.startTime(), + ESTestCase::randomNonNegativeLong) + ); case 3: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, randomValueOtherThan(instance.reason(), () -> randomAlphaOfLengthBetween(5, 15)), instance.endTime(), - instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata()); + instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), + instance.startTime() + ); case 4: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - instance.startTime(), instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), - instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata()); + null, instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), + instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), + instance.startTime() + ); case 5: int totalShards = randomValueOtherThan(instance.totalShards(), () -> randomIntBetween(0, 100)); int failedShards = randomIntBetween(0, totalShards); @@ -110,23 +118,27 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { return new SnapshotShardFailure(randomAlphaOfLengthBetween(5, 10), shardId, randomAlphaOfLengthBetween(5, 10)); })); - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), - instance.reason(), instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), - instance.userMetadata()); + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), + instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), instance.userMetadata(), + instance.startTime() + ); case 6: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), - instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), - Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata()); + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), + instance.endTime(), instance.totalShards(), instance.shardFailures(), + Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata(), instance.startTime() + ); case 7: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), - instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata)); + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), + instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), + randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata), instance.startTime() + ); case 8: List dataStreams = randomValueOtherThan(instance.dataStreams(), () -> Arrays.asList(randomArray(1, 10, String[]::new, () -> randomAlphaOfLengthBetween(2, 20)))); return new SnapshotInfo(instance.snapshotId(), instance.indices(), dataStreams, - instance.startTime(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), - instance.includeGlobalState(), instance.userMetadata()); + null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + instance.includeGlobalState(), instance.userMetadata(), instance.startTime() + ); default: throw new IllegalArgumentException("invalid randomization case"); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java index cef84c6161b0a..c90f0e9dd14f5 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java @@ -159,7 +159,7 @@ blobStoreContext, random())) { // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), + null, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. @@ -167,7 +167,7 @@ blobStoreContext, random())) { () -> PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), + null, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f))); assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>")); @@ -176,7 +176,7 @@ blobStoreContext, random())) { PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), + null, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index a5f413330c377..f81ec51980b3c 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -461,7 +461,7 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Mapget(f -> repo.finalizeSnapshot( ShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), snapshotInfo, SnapshotsService.OLD_SNAPSHOT_FORMAT, Function.identity(), f)); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index d0d13404d4c9b..93ba65b8e4246 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -64,9 +64,9 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.blobstore.FileRestoreContext; import org.elasticsearch.snapshots.SnapshotId; @@ -180,8 +180,9 @@ public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) { ArrayList indices = new ArrayList<>(indicesMap.size()); indicesMap.keysIt().forEachRemaining(indices::add); - return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), SnapshotState.SUCCESS, - response.getState().getNodes().getMaxNodeVersion()); + return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), null, + response.getState().getNodes().getMaxNodeVersion(), SnapshotState.SUCCESS + ); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index dc13646ff5504..8c5eea810824f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -228,9 +228,9 @@ public void testRestoreMinmal() throws IOException { Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshotId, shardGenerations.indices().stream() - .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, - shardGenerations.totalShards(), - Collections.emptyList(), true, Collections.emptyMap()), + .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), null, null, 1L, + shardGenerations.totalShards(), Collections.emptyList(), true, Collections.emptyMap(), 0L + ), Version.CURRENT, Function.identity(), finFuture); finFuture.actionGet(); }); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java index 462fc2aa09c66..5b2469b469859 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java @@ -259,13 +259,8 @@ private SnapshotInfo makeInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-" + randomAlphaOfLength(3), "uuid"), Collections.singletonList("foo"), Collections.singletonList("bar"), - startTime, - null, - startTime + between(1, 10000), - totalShards, - new ArrayList<>(), - false, - meta); + null, null, startTime + between(1, 10000), totalShards, new ArrayList<>(), false, meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.SUCCESS)); return snapInfo; } @@ -291,13 +286,8 @@ private SnapshotInfo makeFailureInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - startTime, - "forced-failure", - startTime + between(1, 10000), - totalShards, - failures, - randomBoolean(), - meta); + null, "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED)); return snapInfo; } @@ -315,13 +305,8 @@ private SnapshotInfo makePartialInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - startTime, - null, - startTime + between(1, 10000), - totalShards, - failures, - randomBoolean(), - meta); + null, null, startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.PARTIAL)); return snapInfo; } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java index e44f335bf3dcb..7cc96ea982c90 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java @@ -244,14 +244,9 @@ public void testPartialFailureSnapshot() throws Exception { new SnapshotId(req.snapshot(), "uuid"), Arrays.asList(req.indices()), Collections.emptyList(), - startTime, - "snapshot started", - endTime, - 3, - Collections.singletonList( + null, "snapshot started", endTime, 3, Collections.singletonList( new SnapshotShardFailure("nodeId", new ShardId("index", "uuid", 0), "forced failure")), - req.includeGlobalState(), - req.userMetadata() + req.includeGlobalState(), req.userMetadata(), startTime )); })) { final AtomicBoolean historyStoreCalled = new AtomicBoolean(false); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index 8f2a1a3394150..458cf67813594 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -125,35 +125,37 @@ public void testSnapshotEligibleForDeletion() { // Test when user metadata is null SnapshotInfo info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(),0L, null, 1L, 1, Collections.emptyList(), true, null); + Collections.emptyList(), null, null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test when no retention is configured info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, null); + null, null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyWithNoRetentionMap), equalTo(false)); // Test when user metadata is a map that doesn't contain "policy" info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar")); + null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test with an ancient snapshot that should be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy")); + null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a snapshot that's start date is old enough to be expunged (but the finish date is not) long time = System.currentTimeMillis() - TimeValue.timeValueDays(30).millis() - 1; info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - time, null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), - true, Collections.singletonMap("policy", "policy")); + null, null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), true, + Collections.singletonMap("policy", "policy"), time + ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a fresh snapshot that should not be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - System.currentTimeMillis(), null, System.currentTimeMillis() + 1, - 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy")); + null, null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), + System.currentTimeMillis() + ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); } @@ -179,10 +181,12 @@ private void retentionTaskTest(final boolean deletionSuccess) throws Exception { ClusterServiceUtils.setState(clusterService, state); final SnapshotInfo eligibleSnapshot = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(), 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 0L); final SnapshotInfo ineligibleSnapshot = new SnapshotInfo(new SnapshotId("name2", "uuid2"), Collections.singletonList("index"), - Collections.emptyList(), System.currentTimeMillis(), null, System.currentTimeMillis() + 1, 1, - Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", policyId), System.currentTimeMillis() + ); Set deleted = ConcurrentHashMap.newKeySet(); Set deletedSnapshotsInHistory = ConcurrentHashMap.newKeySet(); @@ -267,15 +271,20 @@ private void timeBoundedDeletion(final boolean deletionSuccess) throws Exception ClusterServiceUtils.setState(clusterService, state); final SnapshotInfo snap1 = new SnapshotInfo(new SnapshotId("name1", "uuid1"), Collections.singletonList("index"), - Collections.emptyList(), 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 0L); final SnapshotInfo snap2 = new SnapshotInfo(new SnapshotId("name2", "uuid2"), Collections.singletonList("index"), - Collections.emptyList(), 1L, null, 2L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 2L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 1L); final SnapshotInfo snap3 = new SnapshotInfo(new SnapshotId("name3", "uuid3"), Collections.singletonList("index"), - Collections.emptyList(), 2L, null, 3L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 3L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 2L); final SnapshotInfo snap4 = new SnapshotInfo(new SnapshotId("name4", "uuid4"), Collections.singletonList("index"), - Collections.emptyList(), 3L, null, 4L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 4L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 3L); final SnapshotInfo snap5 = new SnapshotInfo(new SnapshotId("name5", "uuid5"), Collections.singletonList("index"), - Collections.emptyList(), 4L, null, 5L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), null, null, 5L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), + 4L); final Set deleted = ConcurrentHashMap.newKeySet(); // We're expected two deletions before they hit the "taken too long" test, so have a latch of 2 From 0c53a229ab9f6f9bfd5f7e2eb01e706ae581e4ce Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 14 Oct 2020 16:14:37 -0600 Subject: [PATCH 008/107] pluginStates->featureStates --- .../create/CreateSnapshotRequest.java | 50 ++++++------ .../restore/RestoreSnapshotRequest.java | 48 ++++++------ .../elasticsearch/snapshots/SnapshotInfo.java | 77 ++++++++++--------- .../create/CreateSnapshotRequestTests.java | 6 +- .../restore/RestoreSnapshotRequestTests.java | 2 +- .../AbstractSnapshotIntegTestCase.java | 6 +- 6 files changed, 96 insertions(+), 93 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index ac0d67c57b882..0ceebdef32c24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -64,7 +64,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest MAXIMUM_METADATA_BYTES) { @@ -363,23 +363,23 @@ public CreateSnapshotRequest userMetadata(Map userMetadata) { /** * @return Which plugin states should be included in the snapshot */ - public String[] pluginStates() { - return pluginStates; + public String[] featureStates() { + return featureStates; } /** - * @param pluginStates The plugin states to be included in the snapshot + * @param featureStates The plugin states to be included in the snapshot */ - public CreateSnapshotRequest pluginStates(String[] pluginStates) { - this.pluginStates = pluginStates; + public CreateSnapshotRequest featureStates(String[] featureStates) { + this.featureStates = featureStates; return this; } /** - * @param pluginStates The plugin states to be included in the snapshot + * @param featureStates The plugin states to be included in the snapshot */ - public CreateSnapshotRequest pluginStates(List pluginStates) { - return pluginStates(pluginStates.toArray(new String[pluginStates.size()])); + public CreateSnapshotRequest featureStates(List featureStates) { + return featureStates(featureStates.toArray(new String[featureStates.size()])); } /** @@ -400,11 +400,11 @@ public CreateSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } - } else if (name.equals("plugin_states")) { + } else if (name.equals("feature_states")) { if (entry.getValue() instanceof List) { - pluginStates((List) entry.getValue()); + featureStates((List) entry.getValue()); } else { - throw new IllegalArgumentException("malformed plugin_states section, should be an array of strings"); + throw new IllegalArgumentException("malformed feature_states section, should be an array of strings"); } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); @@ -431,9 +431,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(index); } builder.endArray(); - if (pluginStates != null && pluginStates.length != 0) { - builder.startArray("plugin_states"); - for (String plugin : pluginStates) { + if (featureStates != null && featureStates.length != 0) { + builder.startArray("feature_states"); + for (String plugin : featureStates) { builder.value(plugin); } builder.endArray(); @@ -465,7 +465,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && - Arrays.equals(pluginStates, that.pluginStates) && + Arrays.equals(featureStates, that.featureStates) && Objects.equals(masterNodeTimeout, that.masterNodeTimeout) && Objects.equals(userMetadata, that.userMetadata); } @@ -475,7 +475,7 @@ public int hashCode() { int result = Objects.hash(snapshot, repository, indicesOptions, partial, includeGlobalState, waitForCompletion, userMetadata); result = 31 * result + Arrays.hashCode(indices); - result = 31 * result + Arrays.hashCode(pluginStates); + result = 31 * result + Arrays.hashCode(featureStates); return result; } @@ -486,7 +486,7 @@ public String toString() { ", repository='" + repository + '\'' + ", indices=" + (indices == null ? null : Arrays.asList(indices)) + ", indicesOptions=" + indicesOptions + - ", pluginStates=" + Arrays.asList(pluginStates) + + ", featureStates=" + Arrays.asList(featureStates) + ", partial=" + partial + ", includeGlobalState=" + includeGlobalState + ", waitForCompletion=" + waitForCompletion + diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 357a1c8316f52..d15dfff0c696c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -39,7 +39,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.PLUGIN_STATES_VERSION; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; @@ -54,7 +54,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest pluginStates) { - return pluginStates(pluginStates.toArray(new String[pluginStates.size()])); + public RestoreSnapshotRequest featureStates(List featureStates) { + return featureStates(featureStates.toArray(new String[featureStates.size()])); } /** @@ -499,11 +499,11 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } - } else if (name.equals("plugin_states")) { + } else if (name.equals("feature_states")) { if (entry.getValue() instanceof List) { - pluginStates((List) entry.getValue()); + featureStates((List) entry.getValue()); } else { - throw new IllegalArgumentException("malformed plugin_states section, should be an array of strings"); + throw new IllegalArgumentException("malformed feature_states section, should be an array of strings"); } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); @@ -563,9 +563,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } - if (pluginStates != null && pluginStates.length != 0) { - builder.startArray("plugin_states"); - for (String plugin : pluginStates) { + if (featureStates != null && featureStates.length != 0) { + builder.startArray("feature_states"); + for (String plugin : featureStates) { builder.value(plugin); } builder.endArray(); @@ -607,7 +607,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && - Arrays.equals(pluginStates, that.pluginStates) && + Arrays.equals(featureStates, that.featureStates) && Objects.equals(renamePattern, that.renamePattern) && Objects.equals(renameReplacement, that.renameReplacement) && Objects.equals(indexSettings, that.indexSettings) && @@ -621,7 +621,7 @@ public int hashCode() { includeGlobalState, partial, includeAliases, indexSettings, snapshotUuid); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); - result = 31 * result + Arrays.hashCode(pluginStates); + result = 31 * result + Arrays.hashCode(featureStates); return result; } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 5e866e3c1e863..7fc9554b082af 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -83,7 +83,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; private static final String USER_METADATA = "metadata"; - private static final String PLUGIN_STATES = "plugin_states"; + private static final String FEATURE_STATES = "feature_states"; private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); @@ -176,8 +176,9 @@ public SnapshotInfo build() { shardFailures = new ArrayList<>(); } - return new SnapshotInfo(snapshotId, indices, dataStreams, snapshotState, reason, version, startTime, endTime, - totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata, null); + return new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, version, startTime, endTime, totalShards, + successfulShards, shardFailures, includeGlobalState, userMetadata, snapshotState + ); } } @@ -240,7 +241,7 @@ int getSuccessfulShards() { private final List dataStreams; - private final List pluginStates; + private final List featureStates; private final long startTime; @@ -261,35 +262,36 @@ int getSuccessfulShards() { private final List shardFailures; - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, SnapshotState state) { - this(snapshotId, indices, dataStreams, state, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null, pluginStates); + this(snapshotId, indices, dataStreams, featureStates, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, Version version, SnapshotState state) { - this(snapshotId, indices, dataStreams, state, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null, pluginStates); + this(snapshotId, indices, dataStreams, featureStates, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } public SnapshotInfo(SnapshotsInProgress.Entry entry) { this(entry.snapshot().getSnapshotId(), - entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), SnapshotState.IN_PROGRESS, - null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), - null); // TODO: Add pluginStates to SnapshotInProgress.Entry + entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), null, null, Version.CURRENT, + entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), + SnapshotState.IN_PROGRESS + ); // TODO: Add featureStates to SnapshotInProgress.Entry } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List pluginStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, String reason, long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, Map userMetadata, long startTime) { - this(snapshotId, indices, dataStreams, snapshotState(reason, shardFailures), reason, Version.CURRENT, - startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata, - pluginStates); + this(snapshotId, indices, dataStreams, featureStates, reason, Version.CURRENT, startTime, endTime, totalShards, + totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata, snapshotState(reason, shardFailures) + ); } - SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state, String reason, - Version version, long startTime, long endTime, int totalShards, int successfulShards, + SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + String reason, Version version, long startTime, long endTime, int totalShards, int successfulShards, List shardFailures, Boolean includeGlobalState, Map userMetadata, - List pluginStates) { + SnapshotState state) { this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.dataStreams = Collections.unmodifiableList(Objects.requireNonNull(dataStreams)); @@ -303,7 +305,7 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da this.shardFailures = Objects.requireNonNull(shardFailures); this.includeGlobalState = includeGlobalState; this.userMetadata = userMetadata; - this.pluginStates = pluginStates; + this.featureStates = featureStates; } /** @@ -323,10 +325,10 @@ public SnapshotInfo(final StreamInput in) throws IOException { includeGlobalState = in.readOptionalBoolean(); userMetadata = in.readMap(); dataStreams = in.readStringList(); - if (in.getVersion().before(CreateSnapshotRequest.PLUGIN_STATES_VERSION)) { - pluginStates = null; + if (in.getVersion().before(CreateSnapshotRequest.FEATURE_STATES_VERSION)) { + featureStates = null; } else { - pluginStates = in.readList(SnapshotPluginInfo::new); + featureStates = in.readList(SnapshotPluginInfo::new); } } @@ -490,7 +492,7 @@ public String toString() { ", includeGlobalState=" + includeGlobalState + ", version=" + version + ", shardFailures=" + shardFailures + - ", pluginStates=" + pluginStates + + ", featureStates=" + featureStates + '}'; } @@ -569,9 +571,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SUCCESSFUL, successfulShards); builder.endObject(); } - if (verbose || pluginStates != null) { - builder.startArray(PLUGIN_STATES); - for (SnapshotPluginInfo snapshotPluginInfo : pluginStates) { + if (verbose || featureStates != null) { + builder.startArray(FEATURE_STATES); + for (SnapshotPluginInfo snapshotPluginInfo : featureStates) { builder.value(snapshotPluginInfo); } builder.endArray(); @@ -613,8 +615,8 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final shardFailure.toXContent(builder, params); } builder.endArray(); - builder.startArray(PLUGIN_STATES); - for (SnapshotPluginInfo snapshotPluginInfo : pluginStates) { + builder.startArray(FEATURE_STATES); + for (SnapshotPluginInfo snapshotPluginInfo : featureStates) { builder.value(snapshotPluginInfo); } builder.endArray(); @@ -643,7 +645,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr Boolean includeGlobalState = null; Map userMetadata = null; List shardFailures = Collections.emptyList(); - List pluginStates = null; + List featureStates = null; if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -698,12 +700,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser)); } shardFailures = Collections.unmodifiableList(shardFailureArrayList); - } else if (PLUGIN_STATES.equals(currentFieldName)) { + } else if (FEATURE_STATES.equals(currentFieldName)) { ArrayList snapshotPluginInfoArrayList = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { snapshotPluginInfoArrayList.add(SnapshotPluginInfo.fromXContent(parser)); } - pluginStates = Collections.unmodifiableList(snapshotPluginInfoArrayList); + featureStates = Collections.unmodifiableList(snapshotPluginInfoArrayList); } else { // It was probably created by newer version - ignoring parser.skipChildren(); @@ -726,7 +728,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr return new SnapshotInfo(new SnapshotId(name, uuid), indices, dataStreams, - state, + featureStates, reason, version, startTime, @@ -736,7 +738,8 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr shardFailures, includeGlobalState, userMetadata, - pluginStates); + state + ); } @Override @@ -764,8 +767,8 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalBoolean(includeGlobalState); out.writeMap(userMetadata); out.writeStringCollection(dataStreams); - if (pluginStates != null) { - out.writeList(pluginStates); + if (featureStates != null) { + out.writeList(featureStates); } } @@ -799,14 +802,14 @@ public boolean equals(Object o) { Objects.equals(version, that.version) && Objects.equals(shardFailures, that.shardFailures) && Objects.equals(userMetadata, that.userMetadata) && - Objects.equals(pluginStates, that.pluginStates); + Objects.equals(featureStates, that.featureStates); } @Override public int hashCode() { return Objects.hash(snapshotId, state, reason, indices, dataStreams, startTime, endTime, totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata, - pluginStates); + featureStates); } private static class SnapshotPluginInfo implements Writeable, ToXContentObject { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 930bd7a167c51..f1d265992e2a8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -67,14 +67,14 @@ public void testToXContent() throws IOException { } if (randomBoolean()) { - List plugins = new ArrayList<>(); + List featureStates = new ArrayList<>(); int count = randomInt(3) + 1; for (int i = 0; i < count; ++i) { - plugins.add(randomAlphaOfLength(randomInt(3) + 2)); + featureStates.add(randomAlphaOfLength(randomInt(3) + 2)); } - original.pluginStates(plugins); + original.featureStates(featureStates); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index 5aea0a0c88c4b..55e0ff1039ede 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -61,7 +61,7 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { plugins.add(randomAlphaOfLength(randomInt(3) + 2)); } - instance.pluginStates(plugins); + instance.featureStates(plugins); } if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index f81ec51980b3c..396549acdc97f 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -459,9 +459,9 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Map adding old version FAILED snapshot [{}] to repository [{}]", snapshotId, repoName); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - SnapshotState.FAILED, "failed on purpose", - SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L,0L, 0, 0, Collections.emptyList(), - randomBoolean(), metadata, null); + null, "failed on purpose", SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L, 0L, 0, 0, Collections.emptyList(), randomBoolean(), + metadata, SnapshotState.FAILED + ); PlainActionFuture.get(f -> repo.finalizeSnapshot( ShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), snapshotInfo, SnapshotsService.OLD_SNAPSHOT_FORMAT, Function.identity(), f)); From f0ef426f106bc050443ead6072592f248cbeb224 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 14 Oct 2020 16:20:36 -0600 Subject: [PATCH 009/107] More plugin->feature --- .../elasticsearch/snapshots/SnapshotInfo.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 7fc9554b082af..4b99b846d1ee3 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -241,7 +241,7 @@ int getSuccessfulShards() { private final List dataStreams; - private final List featureStates; + private final List featureStates; private final long startTime; @@ -262,12 +262,12 @@ int getSuccessfulShards() { private final List shardFailures; - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, SnapshotState state) { this(snapshotId, indices, dataStreams, featureStates, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, Version version, SnapshotState state) { this(snapshotId, indices, dataStreams, featureStates, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } @@ -280,7 +280,7 @@ public SnapshotInfo(SnapshotsInProgress.Entry entry) { ); // TODO: Add featureStates to SnapshotInProgress.Entry } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, String reason, long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, Map userMetadata, long startTime) { this(snapshotId, indices, dataStreams, featureStates, reason, Version.CURRENT, startTime, endTime, totalShards, @@ -288,7 +288,7 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da ); } - SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, String reason, Version version, long startTime, long endTime, int totalShards, int successfulShards, List shardFailures, Boolean includeGlobalState, Map userMetadata, SnapshotState state) { @@ -328,7 +328,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { if (in.getVersion().before(CreateSnapshotRequest.FEATURE_STATES_VERSION)) { featureStates = null; } else { - featureStates = in.readList(SnapshotPluginInfo::new); + featureStates = in.readList(SnapshotFeatureInfo::new); } } @@ -573,8 +573,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa } if (verbose || featureStates != null) { builder.startArray(FEATURE_STATES); - for (SnapshotPluginInfo snapshotPluginInfo : featureStates) { - builder.value(snapshotPluginInfo); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); } builder.endArray(); } @@ -616,8 +616,8 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final } builder.endArray(); builder.startArray(FEATURE_STATES); - for (SnapshotPluginInfo snapshotPluginInfo : featureStates) { - builder.value(snapshotPluginInfo); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); } builder.endArray(); @@ -645,7 +645,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr Boolean includeGlobalState = null; Map userMetadata = null; List shardFailures = Collections.emptyList(); - List featureStates = null; + List featureStates = null; if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -701,11 +701,11 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr } shardFailures = Collections.unmodifiableList(shardFailureArrayList); } else if (FEATURE_STATES.equals(currentFieldName)) { - ArrayList snapshotPluginInfoArrayList = new ArrayList<>(); + ArrayList snapshotFeatureInfoArrayList = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - snapshotPluginInfoArrayList.add(SnapshotPluginInfo.fromXContent(parser)); + snapshotFeatureInfoArrayList.add(SnapshotFeatureInfo.fromXContent(parser)); } - featureStates = Collections.unmodifiableList(snapshotPluginInfoArrayList); + featureStates = Collections.unmodifiableList(snapshotFeatureInfoArrayList); } else { // It was probably created by newer version - ignoring parser.skipChildren(); @@ -812,15 +812,15 @@ public int hashCode() { featureStates); } - private static class SnapshotPluginInfo implements Writeable, ToXContentObject { + private static class SnapshotFeatureInfo implements Writeable, ToXContentObject { final String pluginName; final List indices; - static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = + static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { String pluginName = (String) a[0]; List indices = (List) a[1]; - return new SnapshotPluginInfo(pluginName, indices); + return new SnapshotFeatureInfo(pluginName, indices); }); static { @@ -828,12 +828,12 @@ private static class SnapshotPluginInfo implements Writeable, ToXContentObject { SNAPSHOT_PLUGIN_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); } - private SnapshotPluginInfo(String pluginName, List indices) { + private SnapshotFeatureInfo(String pluginName, List indices) { this.pluginName = pluginName; this.indices = indices; } - SnapshotPluginInfo(final StreamInput in) throws IOException { + SnapshotFeatureInfo(final StreamInput in) throws IOException { this.pluginName = in.readString(); this.indices = in.readStringList(); } @@ -844,7 +844,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(indices); } - public static SnapshotPluginInfo fromXContent(XContentParser parser) throws IOException { + public static SnapshotFeatureInfo fromXContent(XContentParser parser) throws IOException { return SNAPSHOT_PLUGIN_INFO_PARSER.parse(parser, null); } From 1f4ecfe6641fbbc2b9ebc505414f05dd1c3fc403 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 14 Oct 2020 16:31:58 -0600 Subject: [PATCH 010/107] Break out SnapshotFeatureInfo, add tests and equals/hashcode --- .../snapshots/SnapshotFeatureInfo.java | 113 ++++++++++++++++++ .../elasticsearch/snapshots/SnapshotInfo.java | 61 ---------- .../snapshots/SnapshotFeatureInfoTests.java | 58 +++++++++ 3 files changed, 171 insertions(+), 61 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java create mode 100644 server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java new file mode 100644 index 0000000000000..c2015c4ddba86 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -0,0 +1,113 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +class SnapshotFeatureInfo implements Writeable, ToXContentObject { + final String pluginName; + final List indices; + + static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = + new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { + String pluginName = (String) a[0]; + List indices = (List) a[1]; + return new SnapshotFeatureInfo(pluginName, indices); + }); + + static { + SNAPSHOT_PLUGIN_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); + SNAPSHOT_PLUGIN_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); + } + + SnapshotFeatureInfo(String pluginName, List indices) { + this.pluginName = pluginName; + this.indices = indices; + } + + SnapshotFeatureInfo(final StreamInput in) throws IOException { + this.pluginName = in.readString(); + this.indices = in.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(pluginName); + out.writeStringCollection(indices); + } + + public static SnapshotFeatureInfo fromXContent(XContentParser parser) throws IOException { + return SNAPSHOT_PLUGIN_INFO_PARSER.parse(parser, null); + } + + public String getPluginName() { + return pluginName; + } + + public List getIndices() { + return indices; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("plugin_name", pluginName); + builder.startArray("indices"); + for (String index : indices) { + builder.value(index); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SnapshotFeatureInfo)) return false; + SnapshotFeatureInfo that = (SnapshotFeatureInfo) o; + return getPluginName().equals(that.getPluginName()) && + getIndices().equals(that.getIndices()); + } + + @Override + public int hashCode() { + return Objects.hash(getPluginName(), getIndices()); + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 4b99b846d1ee3..7e79cec53c81e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -30,10 +30,8 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; @@ -812,63 +810,4 @@ public int hashCode() { featureStates); } - private static class SnapshotFeatureInfo implements Writeable, ToXContentObject { - final String pluginName; - final List indices; - - static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = - new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { - String pluginName = (String) a[0]; - List indices = (List) a[1]; - return new SnapshotFeatureInfo(pluginName, indices); - }); - - static { - SNAPSHOT_PLUGIN_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); - SNAPSHOT_PLUGIN_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); - } - - private SnapshotFeatureInfo(String pluginName, List indices) { - this.pluginName = pluginName; - this.indices = indices; - } - - SnapshotFeatureInfo(final StreamInput in) throws IOException { - this.pluginName = in.readString(); - this.indices = in.readStringList(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(pluginName); - out.writeStringCollection(indices); - } - - public static SnapshotFeatureInfo fromXContent(XContentParser parser) throws IOException { - return SNAPSHOT_PLUGIN_INFO_PARSER.parse(parser, null); - } - - public String getPluginName() { - return pluginName; - } - - public List getIndices() { - return indices; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.field("plugin_name", pluginName); - builder.startArray("indices"); - for (String index : indices) { - builder.value(index); - } - builder.endArray(); - } - builder.endObject(); - return builder; - } - } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java new file mode 100644 index 0000000000000..13f1508af8ba2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +public class SnapshotFeatureInfoTests extends AbstractSerializingTestCase { + @Override + protected SnapshotFeatureInfo doParseInstance(XContentParser parser) throws IOException { + return SnapshotFeatureInfo.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return SnapshotFeatureInfo::new; + } + + @Override + protected SnapshotFeatureInfo createTestInstance() { + String feature = randomAlphaOfLengthBetween(5,20); + List indices = randomList(1, 10, () -> randomAlphaOfLengthBetween(5, 20)); + return new SnapshotFeatureInfo(feature, indices); + } + + @Override + protected SnapshotFeatureInfo mutateInstance(SnapshotFeatureInfo instance) throws IOException { + if (randomBoolean()) { + return new SnapshotFeatureInfo(randomValueOtherThan(instance.getPluginName(), () -> randomAlphaOfLengthBetween(5, 20)), + instance.getIndices()); + } else { + return new SnapshotFeatureInfo(instance.getPluginName(), + randomList(1, 10, () -> randomValueOtherThanMany(instance.getIndices()::contains, + () -> randomAlphaOfLengthBetween(5, 20)))); + } + } +} From 040fda6b5906ae4772866a58330600f109888512 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 15 Oct 2020 15:48:03 -0600 Subject: [PATCH 011/107] Null guard for `featureStates` in `toXContent` --- .../java/org/elasticsearch/snapshots/SnapshotInfo.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 7e79cec53c81e..6823e51d19b8c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -613,11 +613,13 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final shardFailure.toXContent(builder, params); } builder.endArray(); - builder.startArray(FEATURE_STATES); - for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { - builder.value(snapshotFeatureInfo); + if (featureStates != null) { + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); + } + builder.endArray(); } - builder.endArray(); builder.endObject(); return builder; From 9120ffadb267c991ebdb25df605f237427119e66 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 15 Oct 2020 16:14:06 -0600 Subject: [PATCH 012/107] Another null guard for `featureStates` in `toXContent` --- .../org/elasticsearch/snapshots/SnapshotInfo.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 6823e51d19b8c..8439f6f373580 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -570,11 +570,15 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.endObject(); } if (verbose || featureStates != null) { - builder.startArray(FEATURE_STATES); - for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { - builder.value(snapshotFeatureInfo); + if (featureStates != null) { + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); + } + builder.endArray(); + } else { + builder.field(FEATURE_STATES, (Object) null); } - builder.endArray(); } builder.endObject(); return builder; From 26e94f603804bfef8efe6d464cc375a968b8056f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 15 Oct 2020 16:37:58 -0600 Subject: [PATCH 013/107] writeTo version guard --- .../java/org/elasticsearch/snapshots/SnapshotInfo.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 8439f6f373580..fe0f1b2a10032 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -20,7 +20,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.common.Nullable; @@ -49,6 +48,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; + /** * Information about a snapshot */ @@ -323,7 +324,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { includeGlobalState = in.readOptionalBoolean(); userMetadata = in.readMap(); dataStreams = in.readStringList(); - if (in.getVersion().before(CreateSnapshotRequest.FEATURE_STATES_VERSION)) { + if (in.getVersion().before(FEATURE_STATES_VERSION)) { featureStates = null; } else { featureStates = in.readList(SnapshotFeatureInfo::new); @@ -771,7 +772,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalBoolean(includeGlobalState); out.writeMap(userMetadata); out.writeStringCollection(dataStreams); - if (featureStates != null) { + if (out.getVersion().before(FEATURE_STATES_VERSION)) { out.writeList(featureStates); } } From df3defff3bbc3ccd55f5ed690c90e74f66b2ee5b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 15 Oct 2020 16:52:40 -0600 Subject: [PATCH 014/107] ACTUAL writeTo version guard --- .../src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index fe0f1b2a10032..1f3ef61a11c68 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -772,7 +772,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalBoolean(includeGlobalState); out.writeMap(userMetadata); out.writeStringCollection(dataStreams); - if (out.getVersion().before(FEATURE_STATES_VERSION)) { + if (out.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { out.writeList(featureStates); } } From ce24b74b34753e15c96b0f3804451e26a3e8b591 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 19 Oct 2020 10:19:57 -0600 Subject: [PATCH 015/107] SnapshotInfo featureStates now non-nullable --- .../get/TransportGetSnapshotsAction.java | 3 +- .../elasticsearch/snapshots/SnapshotInfo.java | 25 ++++++++++----- .../snapshots/SnapshotsService.java | 3 +- .../create/CreateSnapshotResponseTests.java | 6 ++-- .../get/GetSnapshotsResponseTests.java | 3 +- .../BlobStoreRepositoryRestoreTests.java | 2 +- .../snapshots/SnapshotInfoTests.java | 31 ++++++++++--------- ...ckEventuallyConsistentRepositoryTests.java | 6 ++-- .../AbstractSnapshotIntegTestCase.java | 4 +-- .../xpack/ccr/repository/CcrRepository.java | 2 +- .../SourceOnlySnapshotShardTests.java | 2 +- .../SnapshotRetentionConfigurationTests.java | 6 ++-- .../xpack/slm/SnapshotLifecycleTaskTests.java | 2 +- .../xpack/slm/SnapshotRetentionTaskTests.java | 20 ++++++------ 14 files changed, 65 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index f06a610218137..982f4c9682746 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -289,7 +289,8 @@ private static List buildSimpleSnapshotInfos(final Set for (SnapshotId snapshotId : toResolve) { final List indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList()); CollectionUtil.timSort(indices); - snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), null, + // NOCOMMIT is emptyList the correct featureStates? + snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), Collections.emptyList(), repositoryData.getSnapshotState(snapshotId))); } CollectionUtil.timSort(snapshotInfos); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 1f3ef61a11c68..707eeabcffeaf 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -94,6 +94,7 @@ public static final class SnapshotInfoBuilder { private String reason = null; private List indices = null; private List dataStreams = null; + private List featureStates = null; private long startTime = 0L; private long endTime = 0L; private ShardStatsBuilder shardStatsBuilder = null; @@ -126,6 +127,10 @@ private void setDataStreams(List dataStreams) { this.dataStreams = dataStreams; } + private void setFeatureStates(List featureStates) { + this.featureStates = featureStates; + } + private void setStartTime(long startTime) { this.startTime = startTime; } @@ -165,6 +170,10 @@ public SnapshotInfo build() { dataStreams = Collections.emptyList(); } + if (featureStates == null) { + featureStates = Collections.emptyList(); + } + SnapshotState snapshotState = state == null ? null : SnapshotState.valueOf(state); Version version = this.version == -1 ? Version.CURRENT : Version.fromId(this.version); @@ -175,7 +184,7 @@ public SnapshotInfo build() { shardFailures = new ArrayList<>(); } - return new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, version, startTime, endTime, totalShards, + return new SnapshotInfo(snapshotId, indices, dataStreams, featureStates, reason, version, startTime, endTime, totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata, snapshotState ); } @@ -273,8 +282,8 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da public SnapshotInfo(SnapshotsInProgress.Entry entry) { this(entry.snapshot().getSnapshotId(), - entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), null, null, Version.CURRENT, - entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), + entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), Collections.emptyList(), + null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), SnapshotState.IN_PROGRESS ); // TODO: Add featureStates to SnapshotInProgress.Entry } @@ -294,6 +303,7 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.dataStreams = Collections.unmodifiableList(Objects.requireNonNull(dataStreams)); + this.featureStates = Collections.unmodifiableList(Objects.requireNonNull(featureStates)); this.state = state; this.reason = reason; this.version = version; @@ -304,7 +314,6 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da this.shardFailures = Objects.requireNonNull(shardFailures); this.includeGlobalState = includeGlobalState; this.userMetadata = userMetadata; - this.featureStates = featureStates; } /** @@ -325,9 +334,9 @@ public SnapshotInfo(final StreamInput in) throws IOException { userMetadata = in.readMap(); dataStreams = in.readStringList(); if (in.getVersion().before(FEATURE_STATES_VERSION)) { - featureStates = null; + featureStates = Collections.emptyList(); } else { - featureStates = in.readList(SnapshotFeatureInfo::new); + featureStates = Collections.unmodifiableList(in.readList(SnapshotFeatureInfo::new)); } } @@ -336,7 +345,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { * all information stripped out except the snapshot id, state, and indices. */ public SnapshotInfo basic() { - return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), null, state); + return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), featureStates, state); } /** @@ -650,7 +659,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr Boolean includeGlobalState = null; Map userMetadata = null; List shardFailures = Collections.emptyList(); - List featureStates = null; + List featureStates = Collections.emptyList(); if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 232aceb68f5af..009551f8c5aee 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1155,7 +1155,8 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), - null, failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), + Collections.emptyList(), failure, threadPool.absoluteTimeInMillis(), + entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime() ); final StepListener metadataListener = new StepListener<>(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java index d862cf6def9e1..dc23c69cfadbe 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.function.Predicate; @@ -70,8 +71,9 @@ protected CreateSnapshotResponse createTestInstance() { boolean globalState = randomBoolean(); return new CreateSnapshotResponse( - new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, endTime, totalShards, shardFailures, globalState, - SnapshotInfoTests.randomUserMetadata(), startTime + // NOCOMMIT generate actual feature states list here + new SnapshotInfo(snapshotId, indices, dataStreams, Collections.emptyList(), reason, endTime, totalShards, shardFailures, + globalState, SnapshotInfoTests.randomUserMetadata(), startTime )); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index d7f03d9fdc35e..3d8397822cf16 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -81,8 +81,9 @@ private List createSnapshotInfos() { String reason = randomBoolean() ? null : "reason"; ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); + // NOCOMMIT generate actual feature states here snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), - null, reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), + Collections.emptyList(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), SnapshotInfoTests.randomUserMetadata(), System.currentTimeMillis() )); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 4612054890376..76a43365ee3cd 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -188,7 +188,7 @@ public void testSnapshotWithConflictingName() throws Exception { .map(IndexId::getName) .collect(Collectors.toList()), Collections.emptyList(), - null, + Collections.emptyList(), null, 1L, 6, diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java index 999cb33c5ac63..313fad974ec8d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,8 +60,8 @@ protected SnapshotInfo createTestInstance() { Map userMetadata = randomUserMetadata(); - return new SnapshotInfo(snapshotId, indices, dataStreams, null, reason, endTime, totalShards, shardFailures, includeGlobalState, - userMetadata, startTime + return new SnapshotInfo(snapshotId, indices, dataStreams, Collections.emptyList(), reason, endTime, totalShards, shardFailures, + includeGlobalState, userMetadata, startTime ); } @@ -76,7 +77,7 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { SnapshotId snapshotId = new SnapshotId( randomValueOtherThan(instance.snapshotId().getName(), () -> randomAlphaOfLength(5)), randomValueOtherThan(instance.snapshotId().getUUID(), () -> randomAlphaOfLength(5))); - return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), null, instance.reason(), + return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), instance.startTime() ); @@ -84,25 +85,25 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { int indicesSize = randomValueOtherThan(instance.indices().size(), () -> randomIntBetween(1, 10)); List indices = Arrays.asList(randomArray(indicesSize, indicesSize, String[]::new, () -> randomAlphaOfLengthBetween(2, 20))); - return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), null, instance.reason(), + return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), instance.startTime() ); case 2: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), randomValueOtherThan(instance.startTime(), ESTestCase::randomNonNegativeLong) ); case 3: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), randomValueOtherThan(instance.reason(), () -> randomAlphaOfLengthBetween(5, 15)), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), instance.startTime() ); case 4: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - null, instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), + Collections.emptyList(), instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), instance.startTime() ); @@ -118,25 +119,25 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { return new SnapshotShardFailure(randomAlphaOfLengthBetween(5, 10), shardId, randomAlphaOfLengthBetween(5, 10)); })); - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), - instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), instance.userMetadata(), - instance.startTime() + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), + instance.reason(), instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), + instance.userMetadata(), instance.startTime() ); case 6: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), - instance.endTime(), instance.totalShards(), instance.shardFailures(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), + instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata(), instance.startTime() ); case 7: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), null, instance.reason(), - instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), + instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata), instance.startTime() ); case 8: List dataStreams = randomValueOtherThan(instance.dataStreams(), () -> Arrays.asList(randomArray(1, 10, String[]::new, () -> randomAlphaOfLengthBetween(2, 20)))); return new SnapshotInfo(instance.snapshotId(), instance.indices(), dataStreams, - null, instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), instance.startTime() ); default: diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java index c90f0e9dd14f5..126ee61d9176c 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java @@ -159,7 +159,7 @@ blobStoreContext, random())) { // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - null, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), + Collections.emptyList(), null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. @@ -167,7 +167,7 @@ blobStoreContext, random())) { () -> PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - null, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap(), 0L), + Collections.emptyList(), null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f))); assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>")); @@ -176,7 +176,7 @@ blobStoreContext, random())) { PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - null, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), + Collections.emptyList(), null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 3af4a02a6df83..2212a4be7944b 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -410,8 +410,8 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Map adding old version FAILED snapshot [{}] to repository [{}]", snapshotId, repoName); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - null, "failed on purpose", SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L, 0L, 0, 0, Collections.emptyList(), randomBoolean(), - metadata, SnapshotState.FAILED + Collections.emptyList(), "failed on purpose", SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L, 0L, 0, 0, Collections.emptyList(), + randomBoolean(), metadata, SnapshotState.FAILED ); PlainActionFuture.get(f -> repo.finalizeSnapshot( ShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), snapshotInfo, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 93ba65b8e4246..ef0b47363e6b1 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -180,7 +180,7 @@ public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) { ArrayList indices = new ArrayList<>(indicesMap.size()); indicesMap.keysIt().forEachRemaining(indices::add); - return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), null, + return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), Collections.emptyList(), response.getState().getNodes().getMaxNodeVersion(), SnapshotState.SUCCESS ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index 8c5eea810824f..d5d012aae2ff2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -228,7 +228,7 @@ public void testRestoreMinmal() throws IOException { Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshotId, shardGenerations.indices().stream() - .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), null, null, 1L, + .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), Collections.emptyList(), null, 1L, shardGenerations.totalShards(), Collections.emptyList(), true, Collections.emptyMap(), 0L ), Version.CURRENT, Function.identity(), finFuture); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java index 5b2469b469859..58e007c22c3b1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java @@ -259,7 +259,7 @@ private SnapshotInfo makeInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-" + randomAlphaOfLength(3), "uuid"), Collections.singletonList("foo"), Collections.singletonList("bar"), - null, null, startTime + between(1, 10000), totalShards, new ArrayList<>(), false, meta, startTime + Collections.emptyList(), null, startTime + between(1, 10000), totalShards, new ArrayList<>(), false, meta, startTime ); assertThat(snapInfo.state(), equalTo(SnapshotState.SUCCESS)); return snapInfo; @@ -286,7 +286,7 @@ private SnapshotInfo makeFailureInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - null, "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + Collections.emptyList(), "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime ); assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED)); return snapInfo; @@ -305,7 +305,7 @@ private SnapshotInfo makePartialInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - null, null, startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + Collections.emptyList(), null, startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime ); assertThat(snapInfo.state(), equalTo(SnapshotState.PARTIAL)); return snapInfo; diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java index 7cc96ea982c90..1983b919a5151 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java @@ -244,7 +244,7 @@ public void testPartialFailureSnapshot() throws Exception { new SnapshotId(req.snapshot(), "uuid"), Arrays.asList(req.indices()), Collections.emptyList(), - null, "snapshot started", endTime, 3, Collections.singletonList( + Collections.emptyList(), "snapshot started", endTime, 3, Collections.singletonList( new SnapshotShardFailure("nodeId", new ShardId("index", "uuid", 0), "forced failure")), req.includeGlobalState(), req.userMetadata(), startTime )); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index 89a9c56304e24..4a3d746772cda 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -110,36 +110,36 @@ public void testSnapshotEligibleForDeletion() { // Test when user metadata is null SnapshotInfo info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(), null, null, 1L, 1, Collections.emptyList(), true, null, 0L); + Collections.emptyList(), Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test when no retention is configured info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - null, null, 1L, 1, Collections.emptyList(), true, null, 0L); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyWithNoRetentionMap), equalTo(false)); // Test when user metadata is a map that doesn't contain "policy" info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar"), 0L); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test with an ancient snapshot that should be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), 0L); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a snapshot that's start date is old enough to be expunged (but the finish date is not) long time = System.currentTimeMillis() - TimeValue.timeValueDays(30).millis() - 1; info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - null, null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), true, + Collections.emptyList(), null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), time ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a fresh snapshot that should not be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - null, null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), - System.currentTimeMillis() + Collections.emptyList(), null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", "policy"), System.currentTimeMillis() ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); } @@ -166,10 +166,10 @@ private void retentionTaskTest(final boolean deletionSuccess) throws Exception { ClusterServiceUtils.setState(clusterService, state); final SnapshotInfo eligibleSnapshot = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(), null, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), - 0L); + Collections.emptyList(), Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", policyId), 0L); final SnapshotInfo ineligibleSnapshot = new SnapshotInfo(new SnapshotId("name2", "uuid2"), Collections.singletonList("index"), - Collections.emptyList(), null, null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, + Collections.emptyList(), Collections.emptyList(), null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId), System.currentTimeMillis() ); From 9382885bdfffb01a684074e5aae8d0011bba2224 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 19 Oct 2020 10:28:30 -0600 Subject: [PATCH 016/107] Test featureStates in CreateSnapshotResponseTests & fix parser --- .../elasticsearch/snapshots/SnapshotFeatureInfo.java | 10 +++++----- .../java/org/elasticsearch/snapshots/SnapshotInfo.java | 2 ++ .../snapshots/create/CreateSnapshotResponseTests.java | 9 ++++++--- .../snapshots/SnapshotFeatureInfoTests.java | 4 ++++ 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index c2015c4ddba86..1daebc7a334cd 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -33,11 +33,11 @@ import java.util.List; import java.util.Objects; -class SnapshotFeatureInfo implements Writeable, ToXContentObject { +public class SnapshotFeatureInfo implements Writeable, ToXContentObject { final String pluginName; final List indices; - static final ConstructingObjectParser SNAPSHOT_PLUGIN_INFO_PARSER = + static final ConstructingObjectParser SNAPSHOT_FEATURE_INFO_PARSER = new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { String pluginName = (String) a[0]; List indices = (List) a[1]; @@ -45,8 +45,8 @@ class SnapshotFeatureInfo implements Writeable, ToXContentObject { }); static { - SNAPSHOT_PLUGIN_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); - SNAPSHOT_PLUGIN_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); + SNAPSHOT_FEATURE_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); + SNAPSHOT_FEATURE_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); } SnapshotFeatureInfo(String pluginName, List indices) { @@ -66,7 +66,7 @@ public void writeTo(StreamOutput out) throws IOException { } public static SnapshotFeatureInfo fromXContent(XContentParser parser) throws IOException { - return SNAPSHOT_PLUGIN_INFO_PARSER.parse(parser, null); + return SNAPSHOT_FEATURE_INFO_PARSER.parse(parser, null); } public String getPluginName() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 707eeabcffeaf..89beddb402b61 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -224,6 +224,8 @@ int getSuccessfulShards() { SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setReason, new ParseField(REASON)); SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setIndices, new ParseField(INDICES)); SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setDataStreams, new ParseField(DATA_STREAMS)); + SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setFeatureStates, SnapshotFeatureInfo.SNAPSHOT_FEATURE_INFO_PARSER, + new ParseField(FEATURE_STATES)); SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setStartTime, new ParseField(START_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java index dc23c69cfadbe..ca440f0c791ff 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java @@ -21,6 +21,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; +import org.elasticsearch.snapshots.SnapshotFeatureInfoTests; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotInfoTests; @@ -29,7 +31,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.function.Predicate; @@ -56,6 +57,9 @@ protected CreateSnapshotResponse createTestInstance() { List dataStreams = new ArrayList<>(); dataStreams.add("test0"); dataStreams.add("test1"); + + List featureStates = randomList(5, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo); + String reason = "reason"; long startTime = System.currentTimeMillis(); long endTime = startTime + 10000; @@ -71,8 +75,7 @@ protected CreateSnapshotResponse createTestInstance() { boolean globalState = randomBoolean(); return new CreateSnapshotResponse( - // NOCOMMIT generate actual feature states list here - new SnapshotInfo(snapshotId, indices, dataStreams, Collections.emptyList(), reason, endTime, totalShards, shardFailures, + new SnapshotInfo(snapshotId, indices, dataStreams, featureStates, reason, endTime, totalShards, shardFailures, globalState, SnapshotInfoTests.randomUserMetadata(), startTime )); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java index 13f1508af8ba2..061116b4dc76d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java @@ -39,6 +39,10 @@ protected Writeable.Reader instanceReader() { @Override protected SnapshotFeatureInfo createTestInstance() { + return randomSnapshotFeatureInfo(); + } + + public static SnapshotFeatureInfo randomSnapshotFeatureInfo() { String feature = randomAlphaOfLengthBetween(5,20); List indices = randomList(1, 10, () -> randomAlphaOfLengthBetween(5, 20)); return new SnapshotFeatureInfo(feature, indices); From 261f5c0d1a27a2cc958b3cf337a5705b2510c5f1 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 19 Oct 2020 10:28:47 -0600 Subject: [PATCH 017/107] NOCOMMIT -> GWB todos --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 2 +- .../admin/cluster/snapshots/get/GetSnapshotsResponseTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 982f4c9682746..86efe4147fee3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -289,7 +289,7 @@ private static List buildSimpleSnapshotInfos(final Set for (SnapshotId snapshotId : toResolve) { final List indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList()); CollectionUtil.timSort(indices); - // NOCOMMIT is emptyList the correct featureStates? + // GWB-> is emptyList the correct featureStates? snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), Collections.emptyList(), repositoryData.getSnapshotState(snapshotId))); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 3d8397822cf16..ad6b270600913 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -81,7 +81,7 @@ private List createSnapshotInfos() { String reason = randomBoolean() ? null : "reason"; ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); - // NOCOMMIT generate actual feature states here + // GWB-> generate actual feature states here snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), Collections.emptyList(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), SnapshotInfoTests.randomUserMetadata(), System.currentTimeMillis() From 89d8e19b81f440884b0892d4c6bf050fc431d5e5 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 19 Oct 2020 10:31:38 -0600 Subject: [PATCH 018/107] Line length --- .../xpack/core/slm/SnapshotRetentionConfigurationTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java index 58e007c22c3b1..80cda1d3ce54f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java @@ -286,7 +286,8 @@ private SnapshotInfo makeFailureInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - Collections.emptyList(), "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + Collections.emptyList(), // GWB-> Actually generate some feature states + "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime ); assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED)); return snapInfo; From 08406bd6f88ddcab78333c3aebf43d5283932295 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 19 Oct 2020 13:31:34 -0600 Subject: [PATCH 019/107] Add `feature_states` to expected responses in the docs --- .../reference/snapshot-restore/apis/create-snapshot-api.asciidoc | 1 + docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index e6e05f83bf3be..9be9eee83bc87 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -156,6 +156,7 @@ The API returns the following response: "version": , "indices": [], "data_streams": [], + "feature_states": [], "include_global_state": false, "metadata": { "taken_by": "user123", diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 5ff01f6a201fe..8a9cb6017b408 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -211,6 +211,7 @@ The API returns the following response: "version": , "indices": [], "data_streams": [], + "feature_states": [], "include_global_state": true, "state": "SUCCESS", "start_time": "2020-07-06T21:55:18.129Z", From 93af2218917b25b3359074ea612ceeb097774c90 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 20 Oct 2020 14:26:06 -0600 Subject: [PATCH 020/107] Add system indices during snapshot creation (first draft) --- .../cluster/ClusterStateDiffIT.java | 4 +- .../cluster/SnapshotsInProgress.java | 75 +++++++++++++------ .../metadata/IndexNameExpressionResolver.java | 5 ++ .../snapshots/SnapshotFeatureInfo.java | 4 +- .../elasticsearch/snapshots/SnapshotInfo.java | 2 +- .../snapshots/SnapshotsService.java | 24 +++++- .../MetadataDeleteIndexServiceTests.java | 4 +- .../MetadataIndexStateServiceTests.java | 4 +- ...SnapshotsInProgressSerializationTests.java | 6 +- .../snapshots/SnapshotsServiceTests.java | 2 +- .../DeleteDataStreamTransportActionTests.java | 6 +- 11 files changed, 96 insertions(+), 40 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index c04ad12ecd955..cde9928687b77 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -720,12 +720,12 @@ public ClusterState.Custom randomCreate(String name) { SnapshotsInProgressSerializationTests.randomState(ImmutableOpenMap.of()), Collections.emptyList(), Collections.emptyList(), - Math.abs(randomLong()), + Collections.emptyList(), randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), - randomVersion(random())))); + randomVersion(random()), Math.abs(randomLong())))); case 1: return new RestoreInProgress.Builder().add( new RestoreInProgress.Entry( diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 5b286bb14e4a2..d0bec5f11930a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -35,10 +35,11 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.RepositoryOperation; +import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.snapshots.InFlightShardSnapshotStates; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotsService; @@ -52,6 +53,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; + /** * Meta data about snapshots that are currently executing */ @@ -92,12 +95,12 @@ public String toString() { * will be in state {@link State#SUCCESS} right away otherwise it will be in state {@link State#STARTED}. */ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, boolean partial, List indices, - List dataStreams, long startTime, long repositoryStateId, + List dataStreams, List featureStates, long repositoryStateId, ImmutableOpenMap shards, Map userMetadata, - Version version) { + Version version, long startTime) { return new SnapshotsInProgress.Entry(snapshot, includeGlobalState, partial, completed(shards.values()) ? State.SUCCESS : State.STARTED, - indices, dataStreams, startTime, repositoryStateId, shards, null, userMetadata, version); + indices, dataStreams, featureStates, repositoryStateId, shards, null, userMetadata, version, startTime); } /** @@ -113,9 +116,10 @@ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, */ public static Entry startClone(Snapshot snapshot, SnapshotId source, List indices, long startTime, long repositoryStateId, Version version) { + // GWB-> Is featureStates right here? return new SnapshotsInProgress.Entry(snapshot, true, false, State.STARTED, indices, Collections.emptyList(), - startTime, repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, - ImmutableOpenMap.of()); + Collections.emptyList(), repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, + ImmutableOpenMap.of(), startTime); } public static class Entry implements Writeable, ToXContent, RepositoryOperation { @@ -129,6 +133,7 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation private final ImmutableOpenMap shards; private final List indices; private final List dataStreams; + private final List featureStates; private final long startTime; private final long repositoryStateId; // see #useShardGenerations @@ -151,24 +156,25 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation // visible for testing, use #startedEntry and copy constructors in production code public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, long startTime, long repositoryStateId, + List dataStreams, List featureStates, long repositoryStateId, ImmutableOpenMap shards, String failure, Map userMetadata, - Version version) { - this(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, failure, - userMetadata, version, null, ImmutableOpenMap.of()); + Version version, long startTime) { + this(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, repositoryStateId, shards, failure, + userMetadata, version, null, ImmutableOpenMap.of(), startTime); } private Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, long startTime, long repositoryStateId, - ImmutableOpenMap shards, String failure, Map userMetadata, - Version version, @Nullable SnapshotId source, - @Nullable ImmutableOpenMap clones) { + List dataStreams, List featureStates, long repositoryStateId, + ImmutableOpenMap shards, String failure, Map userMetadata, + Version version, @Nullable SnapshotId source, + @Nullable ImmutableOpenMap clones, long startTime) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; this.partial = partial; this.indices = indices; this.dataStreams = dataStreams; + this.featureStates = Collections.unmodifiableList(featureStates); this.startTime = startTime; this.shards = shards; this.repositoryStateId = repositoryStateId; @@ -205,6 +211,11 @@ private Entry(StreamInput in) throws IOException { source = null; clones = ImmutableOpenMap.of(); } + if (in.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { + featureStates = Collections.unmodifiableList(in.readList(SnapshotFeatureInfo::new)); + } else { + featureStates = Collections.emptyList(); + } } private static boolean assertShardsConsistent(SnapshotId source, State state, List indices, @@ -239,8 +250,8 @@ assert hasFailures(clones) == false || state == State.FAILED public Entry withRepoGen(long newRepoGen) { assert newRepoGen > repositoryStateId : "Updated repository generation [" + newRepoGen + "] must be higher than current generation [" + repositoryStateId + "]"; - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, newRepoGen, shards, failure, - userMetadata, version, source, clones); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, newRepoGen, shards, failure, + userMetadata, version, source, clones, startTime); } public Entry withClones(ImmutableOpenMap updatedClones) { @@ -249,8 +260,8 @@ public Entry withClones(ImmutableOpenMap } return new Entry(snapshot, includeGlobalState, partial, completed(updatedClones.values()) ? (hasFailures(updatedClones) ? State.FAILED : State.SUCCESS) : - state, indices, dataStreams, startTime, repositoryStateId, shards, failure, userMetadata, version, source, - updatedClones); + state, indices, dataStreams, featureStates, repositoryStateId, shards, failure, userMetadata, version, source, + updatedClones, startTime); } /** @@ -286,8 +297,8 @@ public Entry abort() { } public Entry fail(ImmutableOpenMap shards, State state, String failure) { - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, - failure, userMetadata, version, source, clones); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, repositoryStateId, shards, + failure, userMetadata, version, source, clones, startTime); } /** @@ -300,8 +311,8 @@ public Entry fail(ImmutableOpenMap shards, State s */ public Entry withShardStates(ImmutableOpenMap shards) { if (completed(shards.values())) { - return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, startTime, repositoryStateId, - shards, failure, userMetadata, version); + return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, featureStates, + repositoryStateId, shards, failure, userMetadata, version, startTime); } return withStartedShards(shards); } @@ -312,7 +323,7 @@ public Entry withShardStates(ImmutableOpenMap shar */ public Entry withStartedShards(ImmutableOpenMap shards) { final SnapshotsInProgress.Entry updated = new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, - startTime, repositoryStateId, shards, failure, userMetadata, version); + featureStates, repositoryStateId, shards, failure, userMetadata, version, startTime); assert updated.state().completed() == false && completed(updated.shards().values()) == false : "Only running snapshots allowed but saw [" + updated + "]"; return updated; @@ -359,6 +370,10 @@ public List dataStreams() { return dataStreams; } + public List featureStates() { + return featureStates; + } + @Override public long repositoryStateId() { return repositoryStateId; @@ -406,6 +421,8 @@ public boolean equals(Object o) { if (version.equals(entry.version) == false) return false; if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; + if (!featureStates.equals(entry.featureStates)) return false; + // GWB-> data streams is missing return true; } @@ -423,6 +440,7 @@ public int hashCode() { result = 31 * result + version.hashCode(); result = 31 * result + (source == null ? 0 : source.hashCode()); result = 31 * result + clones.hashCode(); + result = 31 * result + featureStates.hashCode(); return result; } @@ -465,6 +483,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); + builder.startArray(FEATURE_STATES); + { + for (SnapshotFeatureInfo featureState : featureStates) { + featureState.toXContent(builder, params); + } + } + builder.endArray(); builder.array(DATA_STREAMS, dataStreams.toArray(new String[0])); builder.endObject(); return builder; @@ -488,6 +513,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(source); out.writeMap(clones); } + if (out.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { + out.writeList(featureStates); + } } @Override @@ -767,6 +795,7 @@ public void writeTo(StreamOutput out) throws IOException { private static final String INDEX = "index"; private static final String SHARD = "shard"; private static final String NODE = "node"; + private static final String FEATURE_STATES = "feature_states"; @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 13063d5486b1d..2d3185aa0963b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -135,6 +135,11 @@ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, I return concreteIndexNames(context, request.indices()); } + public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesOptions options, String... indexExpressions) { + Context context = new Context(state, options, true); + return concreteIndexNames(context, indexExpressions); + } + public List dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) { // Allow system index access - they'll be filtered out below as there's no such thing (yet) as system data streams Context context = new Context(state, options, false, false, true, true, true); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index 1daebc7a334cd..965c2cf0673e9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -49,12 +49,12 @@ public class SnapshotFeatureInfo implements Writeable, ToXContentObject { SNAPSHOT_FEATURE_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); } - SnapshotFeatureInfo(String pluginName, List indices) { + public SnapshotFeatureInfo(String pluginName, List indices) { this.pluginName = pluginName; this.indices = indices; } - SnapshotFeatureInfo(final StreamInput in) throws IOException { + public SnapshotFeatureInfo(final StreamInput in) throws IOException { this.pluginName = in.readString(); this.indices = in.readStringList(); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 89beddb402b61..eb5a2102a9a18 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -287,7 +287,7 @@ public SnapshotInfo(SnapshotsInProgress.Entry entry) { entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), Collections.emptyList(), null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), SnapshotState.IN_PROGRESS - ); // TODO: Add featureStates to SnapshotInProgress.Entry + ); // GWB-> Add featureStates to SnapshotInProgress.Entry } public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 009551f8c5aee..21995d7d313c0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -117,6 +117,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; /** @@ -257,6 +258,24 @@ public ClusterState execute(ClusterState currentState) { // Store newSnapshot here to be processed in clusterStateProcessed List indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request)); + List featureStates = Collections.emptyList(); + if (request.includeGlobalState() || request.featureStates().length > 0) { + Set featureStatesSet = new HashSet<>(Arrays.asList(request.featureStates())); + featureStates = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature) + || (featureStatesSet.isEmpty() && request.includeGlobalState())) + .map(feature -> new SnapshotFeatureInfo(feature, systemIndexDescriptorMap.get(feature).stream() + .map(descriptor -> descriptor.getIndexPattern()) + .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, LENIENT_EXPAND_OPEN_CLOSED, pattern))) + .collect(Collectors.toList()))) + .collect(Collectors.toList()); + + indices = Stream.concat(featureStates.stream().flatMap(state -> state.getIndices().stream()), indices.stream()) + .distinct() + .collect(Collectors.toList()); + + } // GWB-> Refactor this awful mess + final List dataStreams = indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices()); @@ -281,7 +300,8 @@ public ClusterState execute(ClusterState currentState) { } newEntry = SnapshotsInProgress.startedEntry( new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), - indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, userMeta, version); + indexIds, dataStreams, featureStates, repositoryData.getGenId(), shards, userMeta, version, + threadPool.absoluteTimeInMillis()); final List newEntries = new ArrayList<>(runningSnapshots); newEntries.add(newEntry); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, @@ -1155,7 +1175,7 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), - Collections.emptyList(), failure, threadPool.absoluteTimeInMillis(), + Collections.emptyList(), failure, threadPool.absoluteTimeInMillis(), //GWB-> Once Entry has featureStates, include here entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index a5985ed1216bd..ceb6ae3eb0d1a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -84,8 +84,8 @@ public void testDeleteSnapshotting() { Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); SnapshotsInProgress snaps = SnapshotsInProgress.of(List.of(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), - Collections.emptyList(), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, - SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())))); + Collections.emptyList(), Collections.emptyList(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, + SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), System.currentTimeMillis()))); ClusterState state = ClusterState.builder(clusterState(index)) .putCustom(SnapshotsInProgress.TYPE, snaps) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 16623ec51c469..84305411c2fad 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -397,8 +397,8 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5))); final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, - Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), randomNonNegativeLong(), randomLong(), - shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); + Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), Collections.emptyList(), randomLong(), + shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), randomNonNegativeLong()); return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(List.of(entry))).build(); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index 74ac7793b59f3..057aeca7ed26e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -79,9 +80,10 @@ private Entry randomSnapshot() { shardState.failed() ? randomAlphaOfLength(10) : null, "1")); } } + //GWB-> actually generate featureStates here ImmutableOpenMap shards = builder.build(); - return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, - startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); + return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, Collections.emptyList(), + repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), startTime); } @Override diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 8e15d6d5a1e48..527375b0f0bcc 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -391,7 +391,7 @@ private static ClusterState applyUpdates(ClusterState state, SnapshotsService.Sh private static SnapshotsInProgress.Entry snapshotEntry(Snapshot snapshot, List indexIds, ImmutableOpenMap shards) { return SnapshotsInProgress.startedEntry(snapshot, randomBoolean(), randomBoolean(), indexIds, Collections.emptyList(), - 1L, randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT); + Collections.emptyList(), randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT, 1L); } private static SnapshotsInProgress.Entry cloneEntry( diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index d51a8137f33fa..877e2dc21dfbc 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -115,13 +115,13 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo SnapshotsInProgress.State.SUCCESS, Collections.emptyList(), List.of(dataStreamName), - 0, + Collections.emptyList(), 1, ImmutableOpenMap.of(), null, null, - null - ); + null, + 0); } public void testDeleteNonexistentDataStream() { From e4a4580eeb5ef16c4a72ae2ae7199181e78e95ab Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 21 Oct 2020 16:07:14 -0600 Subject: [PATCH 021/107] Address various TODOs --- .../cluster/snapshots/get/TransportGetSnapshotsAction.java | 1 - .../main/java/org/elasticsearch/snapshots/SnapshotInfo.java | 4 ++-- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 2 +- .../snapshots/SnapshotsInProgressSerializationTests.java | 5 ++--- .../xpack/core/slm/SnapshotRetentionConfigurationTests.java | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 86efe4147fee3..c80fb6a199f5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -289,7 +289,6 @@ private static List buildSimpleSnapshotInfos(final Set for (SnapshotId snapshotId : toResolve) { final List indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList()); CollectionUtil.timSort(indices); - // GWB-> is emptyList the correct featureStates? snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), Collections.emptyList(), repositoryData.getSnapshotState(snapshotId))); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index eb5a2102a9a18..6844eda1422c0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -284,10 +284,10 @@ public SnapshotInfo(SnapshotId snapshotId, List indices, List da public SnapshotInfo(SnapshotsInProgress.Entry entry) { this(entry.snapshot().getSnapshotId(), - entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), Collections.emptyList(), + entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), entry.featureStates(), null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), SnapshotState.IN_PROGRESS - ); // GWB-> Add featureStates to SnapshotInProgress.Entry + ); } public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 21995d7d313c0..d8fe9a63ae01d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1175,7 +1175,7 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), - Collections.emptyList(), failure, threadPool.absoluteTimeInMillis(), //GWB-> Once Entry has featureStates, include here + entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime() ); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index 057aeca7ed26e..6859983d8edc0 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -37,7 +37,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -80,9 +79,9 @@ private Entry randomSnapshot() { shardState.failed() ? randomAlphaOfLength(10) : null, "1")); } } - //GWB-> actually generate featureStates here + List featureStates = randomList(5, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo); ImmutableOpenMap shards = builder.build(); - return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, Collections.emptyList(), + return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, featureStates, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), startTime); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java index 80cda1d3ce54f..905b2393fbc4e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java @@ -286,7 +286,7 @@ private SnapshotInfo makeFailureInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - Collections.emptyList(), // GWB-> Actually generate some feature states + Collections.emptyList(), "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime ); assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED)); From b7ef98ea81542f8c95b9f3d1474899c321fc5775 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 21 Oct 2020 16:19:48 -0600 Subject: [PATCH 022/107] Refactor giant pile of streams --- .../snapshots/SnapshotsService.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d8fe9a63ae01d..94292dadc5fff 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -261,20 +261,18 @@ public ClusterState execute(ClusterState currentState) { List featureStates = Collections.emptyList(); if (request.includeGlobalState() || request.featureStates().length > 0) { Set featureStatesSet = new HashSet<>(Arrays.asList(request.featureStates())); + boolean includeAllFeatureStates = request.includeGlobalState() && featureStatesSet.isEmpty(); + featureStates = systemIndexDescriptorMap.keySet().stream() - .filter(feature -> featureStatesSet.contains(feature) - || (featureStatesSet.isEmpty() && request.includeGlobalState())) - .map(feature -> new SnapshotFeatureInfo(feature, systemIndexDescriptorMap.get(feature).stream() - .map(descriptor -> descriptor.getIndexPattern()) - .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, LENIENT_EXPAND_OPEN_CLOSED, pattern))) - .collect(Collectors.toList()))) + .filter(feature -> includeAllFeatureStates || featureStatesSet.contains(feature)) + .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) .collect(Collectors.toList()); - indices = Stream.concat(featureStates.stream().flatMap(state -> state.getIndices().stream()), indices.stream()) + // Add all resolved indices from the feature states to the list of indices + indices = Stream.concat(indices.stream(), featureStates.stream().flatMap(state -> state.getIndices().stream())) .distinct() .collect(Collectors.toList()); - - } // GWB-> Refactor this awful mess + } final List dataStreams = indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices()); @@ -333,6 +331,18 @@ public TimeValue timeout() { }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } + private List resolveFeatureIndexNames(ClusterState currentState, String feature) { + if (systemIndexDescriptorMap.containsKey(feature) == false) { + throw new IllegalArgumentException("requested snapshot of feature state for unknown feature [" + feature + "]"); + } + + return systemIndexDescriptorMap.get(feature).stream() + .map(SystemIndexDescriptor::getIndexPattern) + .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, + LENIENT_EXPAND_OPEN_CLOSED, pattern))) + .collect(Collectors.toList()); + } + private static void ensureSnapshotNameNotRunning(List runningSnapshots, String repositoryName, String snapshotName) { if (runningSnapshots.stream().anyMatch(s -> { From 9197bb8f50d6b3f95ccff5052985e0adedece161 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 21 Oct 2020 16:25:49 -0600 Subject: [PATCH 023/107] Spotless --- .../action/DeleteDataStreamTransportActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index 877e2dc21dfbc..07bbb2f5119f5 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -121,7 +121,8 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo null, null, null, - 0); + 0 + ); } public void testDeleteNonexistentDataStream() { From 8d6cbcb43f7ef57651374bd306f62ee8d03a07d4 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 21 Oct 2020 17:11:47 -0600 Subject: [PATCH 024/107] Line length --- .../cluster/metadata/MetadataIndexStateServiceTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 84305411c2fad..0c1c98cef3a88 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -398,7 +398,8 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), Collections.emptyList(), randomLong(), - shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), randomNonNegativeLong()); + shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), + randomNonNegativeLong()); return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(List.of(entry))).build(); } From a344a6599a2820e40aae298ae3eece05d2f7d4b4 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 28 Oct 2020 10:34:51 -0600 Subject: [PATCH 025/107] Fix accidental change to startTime and add featureStates to Entry serialization tests --- .../cluster/ClusterStateDiffIT.java | 4 +- .../cluster/SnapshotsInProgress.java | 35 ++++++++------- .../MetadataDeleteIndexServiceTests.java | 4 +- .../MetadataIndexStateServiceTests.java | 7 +-- ...SnapshotsInProgressSerializationTests.java | 43 ++++++++++++------- .../DeleteDataStreamTransportActionTests.java | 4 +- 6 files changed, 54 insertions(+), 43 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index cde9928687b77..05e33d4decd9d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -721,11 +721,11 @@ public ClusterState.Custom randomCreate(String name) { Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), - randomIntBetween(0, 1000), + Math.abs(randomLong()), randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), - randomVersion(random()), Math.abs(randomLong())))); + randomVersion(random())))); case 1: return new RestoreInProgress.Builder().add( new RestoreInProgress.Entry( diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 22fb6f925d8c5..00006a41961e8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -100,7 +100,7 @@ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, Version version, long startTime) { return new SnapshotsInProgress.Entry(snapshot, includeGlobalState, partial, completed(shards.values()) ? State.SUCCESS : State.STARTED, - indices, dataStreams, featureStates, repositoryStateId, shards, null, userMetadata, version, startTime); + indices, dataStreams, featureStates, startTime, repositoryStateId, shards, null, userMetadata, version); } /** @@ -118,8 +118,8 @@ public static Entry startClone(Snapshot snapshot, SnapshotId source, List Is featureStates right here? return new SnapshotsInProgress.Entry(snapshot, true, false, State.STARTED, indices, Collections.emptyList(), - Collections.emptyList(), repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, - ImmutableOpenMap.of(), startTime); + Collections.emptyList(), startTime, repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, + ImmutableOpenMap.of()); } public static class Entry implements Writeable, ToXContent, RepositoryOperation { @@ -156,18 +156,18 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation // visible for testing, use #startedEntry and copy constructors in production code public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, List featureStates, long repositoryStateId, + List dataStreams, List featureStates, long startTime, long repositoryStateId, ImmutableOpenMap shards, String failure, Map userMetadata, - Version version, long startTime) { - this(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, repositoryStateId, shards, failure, - userMetadata, version, null, ImmutableOpenMap.of(), startTime); + Version version) { + this(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, repositoryStateId, shards, + failure, userMetadata, version, null, ImmutableOpenMap.of()); } private Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, List featureStates, long repositoryStateId, + List dataStreams, List featureStates, long startTime, long repositoryStateId, ImmutableOpenMap shards, String failure, Map userMetadata, Version version, @Nullable SnapshotId source, - @Nullable ImmutableOpenMap clones, long startTime) { + @Nullable ImmutableOpenMap clones) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; @@ -250,8 +250,8 @@ assert hasFailures(clones) == false || state == State.FAILED public Entry withRepoGen(long newRepoGen) { assert newRepoGen > repositoryStateId : "Updated repository generation [" + newRepoGen + "] must be higher than current generation [" + repositoryStateId + "]"; - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, newRepoGen, shards, failure, - userMetadata, version, source, clones, startTime); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, newRepoGen, + shards, failure, userMetadata, version, source, clones); } public Entry withClones(ImmutableOpenMap updatedClones) { @@ -260,8 +260,8 @@ public Entry withClones(ImmutableOpenMap } return new Entry(snapshot, includeGlobalState, partial, completed(updatedClones.values()) ? (hasFailures(updatedClones) ? State.FAILED : State.SUCCESS) : - state, indices, dataStreams, featureStates, repositoryStateId, shards, failure, userMetadata, version, source, - updatedClones, startTime); + state, indices, dataStreams, featureStates, startTime, repositoryStateId, shards, failure, userMetadata, + version, source, updatedClones); } /** @@ -297,8 +297,8 @@ public Entry abort() { } public Entry fail(ImmutableOpenMap shards, State state, String failure) { - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, repositoryStateId, shards, - failure, userMetadata, version, source, clones, startTime); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, + repositoryStateId, shards, failure, userMetadata, version, source, clones); } /** @@ -312,7 +312,7 @@ public Entry fail(ImmutableOpenMap shards, State s public Entry withShardStates(ImmutableOpenMap shards) { if (completed(shards.values())) { return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, featureStates, - repositoryStateId, shards, failure, userMetadata, version, startTime); + startTime, repositoryStateId, shards, failure, userMetadata, version); } return withStartedShards(shards); } @@ -323,7 +323,7 @@ public Entry withShardStates(ImmutableOpenMap shar */ public Entry withStartedShards(ImmutableOpenMap shards) { final SnapshotsInProgress.Entry updated = new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, - featureStates, repositoryStateId, shards, failure, userMetadata, version, startTime); + featureStates, startTime, repositoryStateId, shards, failure, userMetadata, version); assert updated.state().completed() == false && completed(updated.shards().values()) == false : "Only running snapshots allowed but saw [" + updated + "]"; return updated; @@ -425,7 +425,6 @@ public boolean equals(Object o) { if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; if (!featureStates.equals(entry.featureStates)) return false; - // GWB-> data streams is missing return true; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index ceb6ae3eb0d1a..bff90522930a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -84,8 +84,8 @@ public void testDeleteSnapshotting() { Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); SnapshotsInProgress snaps = SnapshotsInProgress.of(List.of(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), - Collections.emptyList(), Collections.emptyList(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, - SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), System.currentTimeMillis()))); + Collections.emptyList(), Collections.emptyList(), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), + ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())))); ClusterState state = ClusterState.builder(clusterState(index)) .putCustom(SnapshotsInProgress.TYPE, snaps) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 0c1c98cef3a88..2a42b6d110d2f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -397,9 +397,10 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5))); final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, - Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), Collections.emptyList(), randomLong(), - shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), - randomNonNegativeLong()); + Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), Collections.emptyList(), + randomNonNegativeLong(), randomLong(), shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), + VersionUtils.randomVersion(random()) + ); return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(List.of(entry))).build(); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index a4271603096ab..c143bfe2ddd2a 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -86,7 +86,7 @@ private Entry randomSnapshot() { List featureStates = randomList(5, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo); ImmutableOpenMap shards = builder.build(); return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, featureStates, - repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()), startTime); + startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); } @Override @@ -153,38 +153,40 @@ protected Custom mutateInstance(Custom instance) { } private Entry mutateEntry(Entry entry) { - switch (randomInt(7)) { + switch (randomInt(8)) { case 0: boolean includeGlobalState = !entry.includeGlobalState(); return new Entry(entry.snapshot(), includeGlobalState, entry.partial(), entry.state(), entry.indices(), entry.dataStreams(), - entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), entry.version()); + entry.featureStates(), entry.repositoryStateId(), entry.startTime(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 1: boolean partial = !entry.partial(); return new Entry(entry.snapshot(), entry.includeGlobalState(), partial, entry.state(), entry.indices(), entry.dataStreams(), - entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), entry.version()); + entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 2: List dataStreams = Stream.concat( entry.dataStreams().stream(), Stream.of(randomAlphaOfLength(10))) .collect(Collectors.toList()); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - dataStreams, entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + dataStreams, entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 3: long startTime = randomValueOtherThan(entry.startTime(), ESTestCase::randomLong); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), startTime, entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), startTime, entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 4: long repositoryStateId = randomValueOtherThan(entry.startTime(), ESTestCase::randomLong); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), repositoryStateId, entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), repositoryStateId, entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 5: String failure = randomValueOtherThan(entry.failure(), () -> randomAlphaOfLengthBetween(2, 10)); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), entry.shards(), failure, entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), failure, + entry.userMetadata(), entry.version()); case 6: List indices = entry.indices(); ImmutableOpenMap shards = entry.shards(); @@ -204,8 +206,8 @@ private Entry mutateEntry(Entry entry) { } shards = builder.build(); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), randomState(shards), indices, - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), shards, entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), shards, entry.failure(), + entry.userMetadata(), entry.version()); case 7: Map userMetadata = entry.userMetadata() != null ? new HashMap<>(entry.userMetadata()) : new HashMap<>(); String key = randomAlphaOfLengthBetween(2, 10); @@ -215,8 +217,17 @@ private Entry mutateEntry(Entry entry) { userMetadata.put(key, randomAlphaOfLengthBetween(2, 10)); } return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), userMetadata, - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), + entry.failure(), userMetadata, entry.version()); + case 8: + logger.error("randomizing feature states"); + List featureStates = randomList(1, 5, + () -> randomValueOtherThanMany(entry.featureStates()::contains, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo)); + final Entry newEntry = new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), + entry.indices(), + entry.dataStreams(), featureStates, entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); + return newEntry; default: throw new IllegalArgumentException("invalid randomization case"); } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index 07bbb2f5119f5..dad1726cd5a1c 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -116,12 +116,12 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo Collections.emptyList(), List.of(dataStreamName), Collections.emptyList(), + 0, 1, ImmutableOpenMap.of(), null, null, - null, - 0 + null ); } From 7ecabdc928e69d6df7368b9231761b2f11b20c55 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 30 Oct 2020 16:29:34 -0600 Subject: [PATCH 026/107] Fix NPE in SnapshotResiliencyTests --- .../org/elasticsearch/snapshots/SnapshotResiliencyTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index b25f8be94a51c..8ee1e2a2cc4ea 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1488,7 +1488,7 @@ public void onFailure(final Exception e) { ); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService(settings, clusterService, indexNameExpressionResolver, repositoriesService, - transportService, actionFilters, null); + transportService, actionFilters, Collections.emptyMap()); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); From f069cf163e8593defc1624e9229f5f717e5a9a4f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 30 Oct 2020 16:33:29 -0600 Subject: [PATCH 027/107] Remove null guards for SnapshotInfo featureStates as featureStates cannot be null --- .../elasticsearch/snapshots/SnapshotInfo.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 6844eda1422c0..93fbc9ba6acaa 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -581,16 +581,13 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SUCCESSFUL, successfulShards); builder.endObject(); } - if (verbose || featureStates != null) { - if (featureStates != null) { - builder.startArray(FEATURE_STATES); - for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { - builder.value(snapshotFeatureInfo); - } - builder.endArray(); - } else { - builder.field(FEATURE_STATES, (Object) null); + if (verbose || featureStates.isEmpty() == false) { + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); } + builder.endArray(); + } builder.endObject(); return builder; @@ -629,13 +626,11 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final shardFailure.toXContent(builder, params); } builder.endArray(); - if (featureStates != null) { - builder.startArray(FEATURE_STATES); - for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { - builder.value(snapshotFeatureInfo); - } - builder.endArray(); + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); } + builder.endArray(); builder.endObject(); return builder; From 93c96df3d9a97b246e1e978f797c0b4bdc679ea6 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 30 Oct 2020 16:38:59 -0600 Subject: [PATCH 028/107] Omit empty featureStates when creating the snapshots --- .../main/java/org/elasticsearch/snapshots/SnapshotsService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 7b6f738a925eb..0932546b911f3 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -265,6 +265,7 @@ public ClusterState execute(ClusterState currentState) { featureStates = systemIndexDescriptorMap.keySet().stream() .filter(feature -> includeAllFeatureStates || featureStatesSet.contains(feature)) .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) + .filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates .collect(Collectors.toList()); // Add all resolved indices from the feature states to the list of indices From 9a1ee0ff247cf17a11f6719842f81867e6d3b968 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 3 Nov 2020 14:17:59 -0700 Subject: [PATCH 029/107] Ensure built-in features (i.e. tasks) are propagated to SnapshotsService --- .../elasticsearch/indices/SystemIndices.java | 34 ++++++++++++------- .../java/org/elasticsearch/node/Node.java | 2 +- .../tasks/TaskResultsService.java | 2 ++ .../indices/SystemIndicesTests.java | 4 +-- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 78ea911e8ca34..fd7873a9045fd 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.Index; -import org.elasticsearch.tasks.TaskResultsService; import java.util.Collection; import java.util.Comparator; @@ -39,6 +38,7 @@ import java.util.stream.Collectors; import static java.util.stream.Collectors.toUnmodifiableList; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; /** @@ -48,20 +48,16 @@ */ public class SystemIndices { private static final Map> SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( - TaskResultsService.class.getSimpleName(), List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index")) + TASKS_FEATURE_NAME, List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index")) ); private final CharacterRunAutomaton runAutomaton; - private final Collection systemIndexDescriptors; + private final Map> featureSystemIndexDescriptors; public SystemIndices(Map> pluginAndModulesDescriptors) { - final Map> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); - checkForOverlappingPatterns(descriptorsMap); - this.systemIndexDescriptors = descriptorsMap.values() - .stream() - .flatMap(Collection::stream) - .collect(Collectors.toUnmodifiableList()); - this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors); + featureSystemIndexDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); + checkForOverlappingPatterns(featureSystemIndexDescriptors); + this.runAutomaton = buildCharacterRunAutomaton(featureSystemIndexDescriptors); } /** @@ -89,7 +85,8 @@ public boolean isSystemIndex(String indexName) { * @throws IllegalStateException if multiple descriptors match the name */ public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) { - final List matchingDescriptors = systemIndexDescriptors.stream() + final List matchingDescriptors = featureSystemIndexDescriptors.values().stream() + .flatMap(Collection::stream) .filter(descriptor -> descriptor.matchesIndexPattern(name)) .collect(toUnmodifiableList()); @@ -112,8 +109,19 @@ public boolean isSystemIndex(String indexName) { } } - private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection descriptors) { - Optional automaton = descriptors.stream() + /** + * Gets all system index descriptors, collected by feature. + * @return A Map of feature name to system index descriptors + */ + public Map> getSystemIndexDescriptorsByFeature() { + // GWB-> I don't like exposing this as public, but we also need to be able to get the merged list of plugin/module features + // plus internal/built-in ones (i.e. tasks) + return featureSystemIndexDescriptors; + } + + private static CharacterRunAutomaton buildCharacterRunAutomaton(Map> descriptors) { + Optional automaton = descriptors.values().stream() + .flatMap(Collection::stream) .map(descriptor -> Regex.simpleMatchToAutomaton(descriptor.getIndexPattern())) .reduce(Operations::union); return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE)); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 7463a0a4a501e..791d4f7bdab61 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -581,7 +581,7 @@ protected Node(final Environment initialEnvironment, repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService(settings, clusterService, clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters(), - systemIndexDescriptorMap); + systemIndices.getSystemIndexDescriptorsByFeature()); SnapshotShardsService snapshotShardsService = new SnapshotShardsService(settings, clusterService, repositoryService, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index bd2f052844614..fa08eee8aac5c 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -65,6 +65,8 @@ public class TaskResultsService { private static final Logger logger = LogManager.getLogger(TaskResultsService.class); + public static final String TASKS_FEATURE_NAME = "tasks"; + public static final String TASK_INDEX = ".tasks"; public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-index-mapping.json"; diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java index 83420fc703e2c..1c5d1568bdf37 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices; -import org.elasticsearch.tasks.TaskResultsService; import org.elasticsearch.test.ESTestCase; import java.util.Collection; @@ -27,6 +26,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -95,7 +95,7 @@ public void testBuiltInSystemIndices() { public void testPluginCannotOverrideBuiltInSystemIndex() { Map> pluginMap = Map.of( - TaskResultsService.class.getSimpleName(), List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) + TASKS_FEATURE_NAME, List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SystemIndices(pluginMap)); assertThat(e.getMessage(), containsString("plugin or module attempted to define the same source")); From b86dcf663503c34df4fe786fac508180c7a83d30 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 5 Nov 2020 13:03:31 -0700 Subject: [PATCH 030/107] Don't include global state in test snapshot (for now) to prevent conflicts --- .../src/test/java/org/elasticsearch/client/SnapshotIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index f8941077b2e80..2f2ca9c65fce4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -263,6 +263,7 @@ public void testRestoreSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot); createSnapshotRequest.indices(testIndex); createSnapshotRequest.waitForCompletion(true); + createSnapshotRequest.includeGlobalState(false); // GWB-> Remove this once index replacement is in place if (randomBoolean()) { createSnapshotRequest.userMetadata(randomUserMetadata()); } From c8ed77a604c1d41255a2db8496f7f2556621789a Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 12 Nov 2020 14:31:33 -0500 Subject: [PATCH 031/107] Add tests for snapshot and restore of system indices --- .../snapshots/SystemIndicesSnapshotIT.java | 246 ++++++++++++++++++ .../elasticsearch/cluster/ClusterModule.java | 8 +- .../java/org/elasticsearch/node/Node.java | 3 +- .../snapshots/RestoreService.java | 20 +- .../snapshots/SnapshotResiliencyTests.java | 1 + 5 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java new file mode 100644 index 0000000000000..21f1b2215cad9 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -0,0 +1,246 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.in; + +public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(SystemIndexTestPlugin.class); + plugins.add(AnotherSystemIndexTestPlugin.class); + return plugins; + } + + public void testRestoreSystemIndicesAsGlobalState() { + createRepository("test-repo", "fs"); + + // create index and add a document + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices as global state without closing the index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + public void testSnapshotWithoutGlobalState() { + createRepository("test-repo", "fs"); + + // create index and add a document + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + assertAcked(prepareCreate("not-a-system-index", 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc"); + indexDoc("not-a-system-index", "1", "purpose", "non system index doc"); + + // run a snapshot without global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(false) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + clusterAdmin().prepareGetRepositories("test-repo").get(); + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots("test-repo").get() + .getSnapshots("test-repo").stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + assertThat("not-a-system-index", in(snapshottedIndices)); + // TODO: without global state the system index shouldn't be snapshotted + // assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices))); + } + + public void testSnapshotByFeature() { + createRepository("test-repo", "fs"); + + // create indices + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + assertAcked(prepareCreate(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + // put a document in each one + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot by feature + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + // setFeatureList() + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add some other documents + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices as global state without closing the index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + public void testRestoreByFeature() { + createRepository("test-repo", "fs"); + + // create indices + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + assertAcked(prepareCreate(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + // put a document in each one + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add some other documents + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices by feature + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + // restore by feature + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + } + + private long getDocCount(String indexName) { + return client().admin().indices().prepareStats(indexName).get().getPrimaries().getDocs().getCount(); + } + + public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); + } + + @Override + public String getPluginName() { + return SystemIndexTestPlugin.class.getSimpleName(); + } + } + + public static class AnotherSystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".another-test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); + } + + @Override + public String getPluginName() { + return AnotherSystemIndexTestPlugin.class.getSimpleName(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index d828823eebf32..8c7a981a53219 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -106,6 +106,7 @@ public class ClusterModule extends AbstractModule { private final AllocationDeciders allocationDeciders; private final AllocationService allocationService; private final List clusterPlugins; + private final MetadataDeleteIndexService metadataDeleteIndexService; // pkg private for tests final Collection deciderList; final ShardsAllocator shardsAllocator; @@ -119,6 +120,7 @@ public ClusterModule(Settings settings, ClusterService clusterService, List getNamedWriteables() { @@ -259,13 +261,17 @@ public AllocationService getAllocationService() { return allocationService; } + public MetadataDeleteIndexService getMetadataDeleteIndexService() { + return metadataDeleteIndexService; + } + @Override protected void configure() { bind(GatewayAllocator.class).asEagerSingleton(); bind(AllocationService.class).toInstance(allocationService); bind(ClusterService.class).toInstance(clusterService); bind(NodeConnectionsService.class).asEagerSingleton(); - bind(MetadataDeleteIndexService.class).asEagerSingleton(); + bind(MetadataDeleteIndexService.class).toInstance(metadataDeleteIndexService); bind(MetadataIndexStateService.class).asEagerSingleton(); bind(MetadataMappingService.class).asEagerSingleton(); bind(MetadataIndexAliasesService.class).asEagerSingleton(); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 791d4f7bdab61..279cc46a72b02 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -585,7 +585,8 @@ protected Node(final Environment initialEnvironment, SnapshotShardsService snapshotShardsService = new SnapshotShardsService(settings, clusterService, repositoryService, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), - metadataCreateIndexService, metadataIndexUpgradeService, clusterService.getClusterSettings(), shardLimitValidator); + metadataCreateIndexService, clusterModule.getMetadataDeleteIndexService(), metadataIndexUpgradeService, + clusterService.getClusterSettings(), shardLimitValidator); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state, clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, rerouteService); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 74dfca8fedeed..378e846eacc60 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -48,6 +48,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.MetadataIndexUpgradeService; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; @@ -156,6 +157,8 @@ public class RestoreService implements ClusterStateApplier { private final MetadataIndexUpgradeService metadataIndexUpgradeService; + private final MetadataDeleteIndexService metadataDeleteIndexService; + private final ShardLimitValidator shardLimitValidator; private final ClusterSettings clusterSettings; @@ -164,13 +167,14 @@ public class RestoreService implements ClusterStateApplier { public RestoreService(ClusterService clusterService, RepositoriesService repositoriesService, AllocationService allocationService, MetadataCreateIndexService createIndexService, - MetadataIndexUpgradeService metadataIndexUpgradeService, ClusterSettings clusterSettings, - ShardLimitValidator shardLimitValidator) { + MetadataDeleteIndexService metadataDeleteIndexService, MetadataIndexUpgradeService metadataIndexUpgradeService, + ClusterSettings clusterSettings, ShardLimitValidator shardLimitValidator) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; this.allocationService = allocationService; this.createIndexService = createIndexService; this.metadataIndexUpgradeService = metadataIndexUpgradeService; + this.metadataDeleteIndexService = metadataDeleteIndexService; if (DiscoveryNode.isMasterNode(clusterService.getSettings())) { clusterService.addStateApplier(this); } @@ -264,15 +268,23 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, metadataBuilder = Metadata.builder(); } + List systemIndices = new ArrayList<>(); + // is this filtered by what's in the request? final List indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot); for (IndexId indexId : indexIdsInSnapshot) { - metadataBuilder.put(repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId), false); + IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); + if (snapshotIndexMetaData.isSystem()) { + // add to system indices list + systemIndices.add(snapshotIndexMetaData.getIndex()); + } + metadataBuilder.put(snapshotIndexMetaData, false); } final Metadata metadata = metadataBuilder.build(); // Apply renaming on index names, returning a map of names where // the key is the renamed index and the value is the original name + // TODO: don't apply to system indices final Map indices = renamedIndices(request, indicesInSnapshot, dataStreamIndices); // Now we can start the actual restore process by adding shards to be recovered in the cluster state @@ -293,6 +305,8 @@ public ClusterState execute(ClusterState currentState) { deletionsInProgress.getEntries().get(0) + "]"); } + currentState = metadataDeleteIndexService.deleteIndices(currentState, new HashSet<>(systemIndices)); + // Updating cluster state ClusterState.Builder builder = ClusterState.builder(currentState); Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 8ee1e2a2cc4ea..75899b78d5ef9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1596,6 +1596,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi final RestoreService restoreService = new RestoreService( clusterService, repositoriesService, allocationService, metadataCreateIndexService, + new MetadataDeleteIndexService(settings, clusterService, allocationService), new MetadataIndexUpgradeService( settings, namedXContentRegistry, mapperRegistry, From e9b25ac83f10b79c58ff690833aebba99d2ffd89 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 12 Nov 2020 17:05:46 -0700 Subject: [PATCH 032/107] Add feature description to system index plugins --- .../java/org/elasticsearch/kibana/KibanaPlugin.java | 7 ++++++- .../java/org/elasticsearch/http/SystemIndexRestIT.java | 7 ++++++- .../elasticsearch/cluster/ClusterInfoServiceIT.java | 7 ++++++- server/src/main/java/org/elasticsearch/node/Node.java | 2 +- .../org/elasticsearch/plugins/SystemIndexPlugin.java | 10 +++++++++- .../xpack/async/AsyncResultsIndexPlugin.java | 7 ++++++- .../org/elasticsearch/xpack/enrich/EnrichPlugin.java | 7 ++++++- .../org/elasticsearch/xpack/logstash/Logstash.java | 7 ++++++- .../org/elasticsearch/xpack/ml/MachineLearning.java | 7 ++++++- .../LocalStateSearchableSnapshots.java | 9 +++++++-- .../SearchableSnapshotsSystemIndicesIntegTests.java | 7 ++++++- .../xpack/searchablesnapshots/SearchableSnapshots.java | 7 ++++++- .../org/elasticsearch/xpack/security/Security.java | 7 ++++++- .../org/elasticsearch/xpack/transform/Transform.java | 7 ++++++- .../java/org/elasticsearch/xpack/watcher/Watcher.java | 7 ++++++- 15 files changed, 89 insertions(+), 16 deletions(-) diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 86c73e1296fa6..5d688c5b70c2c 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -76,10 +76,15 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "kibana"; } + @Override + public String getFeatureDescription() { + return "Manages Kibana configuration and reports"; + } + @Override public List getRestHandlers( Settings settings, diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java index 5088614abc700..fca99e665c9ec 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -144,10 +144,15 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return SystemIndexRestIT.class.getSimpleName(); } + @Override + public String getFeatureDescription() { + return "test plugin"; + } + public static class AddDocRestHandler extends BaseRestHandler { @Override public boolean allowSystemIndexAccessByDefault() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index c9f99704b6473..9666f9c97b297 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -94,9 +94,14 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return ClusterInfoServiceIT.class.getSimpleName(); } + + @Override + public String getFeatureDescription() { + return "test plugin"; + } } public static class BlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 791d4f7bdab61..f72d6f963f4f9 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -492,7 +492,7 @@ protected Node(final Environment initialEnvironment, .filterPlugins(SystemIndexPlugin.class) .stream() .collect(Collectors.toUnmodifiableMap( - plugin -> plugin.getPluginName(), + plugin -> plugin.getFeatureName(), plugin -> plugin.getSystemIndexDescriptors(settings))); final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap); diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 5007da185c3b3..4054c9904ae38 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -41,5 +41,13 @@ default Collection getSystemIndexDescriptors(Settings set return Collections.emptyList(); } - String getPluginName(); + /** + * @return The name of the feature, as used for specifying feature states in snapshot creation and restoration. + */ + String getFeatureName(); + + /** + * @return A description of the feature, as used for the Get Snapshottable Features API. + */ + String getFeatureDescription(); } diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java index 18f2db149769c..8185739f84ca3 100644 --- a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -49,10 +49,15 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "async_search"; } + @Override + public String getFeatureDescription() { + return "Manages results of async searches"; + } + @Override public Collection createComponents( Client client, diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 1f85735016290..f873ad3374519 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -238,7 +238,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "enrich"; } + + @Override + public String getFeatureDescription() { + return "Manages data related to Enrich policies"; + } } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 5351206089d1c..e36967ff5c040 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -102,7 +102,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "logstash_management"; } + + @Override + public String getFeatureDescription() { + return "Enables Logstash Central Management pipeline storage"; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index cfd1e7d6fd1a1..86ee10155d689 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1114,10 +1114,15 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "machine_learning"; } + @Override + public String getFeatureDescription() { + return "Provides anomaly detection and forecasting functionality"; + } + @Override public BreakerSettings getCircuitBreaker(Settings settings) { return BreakerSettings.updateFromSettings( diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java index 89df64ca42223..513df283e15d0 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java @@ -38,7 +38,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { - return plugin.getPluginName(); + public String getFeatureName() { + return plugin.getFeatureName(); + } + + @Override + public String getFeatureDescription() { + return plugin.getFeatureDescription(); } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java index a84cd78809e10..81ad79320e84d 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java @@ -93,8 +93,13 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return SearchableSnapshotsSystemIndicesIntegTests.class.getSimpleName(); } + + @Override + public String getFeatureDescription() { + return "test plugin"; + } } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 7067fcfe1b3e7..92449b439d7c9 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -229,10 +229,15 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "searchable_snapshots"; } + @Override + public String getFeatureDescription() { + return "Manages caches and configuration for searchable snapshots"; + } + @Override public Map getDirectoryFactories() { return Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index c691234207a0c..e37119f6db57d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1156,7 +1156,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "security"; } + + @Override + public String getFeatureDescription() { + return "Manages configuration for Security features, such as users and roles"; + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 2ae982b376457..385bd8a47ff10 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -376,7 +376,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "transform"; } + + @Override + public String getFeatureDescription() { + return "Manages configuration and state for transforms"; + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 84f9ba02b028f..07dc27902078e 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -707,7 +707,12 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return "watcher"; } + + @Override + public String getFeatureDescription() { + return "Manages Watch definitions and state"; + } } From a8398fa8a75ace1c97b066c83dd9db4a0cebc86c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 12 Nov 2020 17:11:18 -0700 Subject: [PATCH 033/107] Fix typo in test name --- ...onseTest.java => GetSnapshottableFeaturesResponseTests.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/{GetSnapshottableFeaturesResponseTest.java => GetSnapshottableFeaturesResponseTests.java} (94%) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTest.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java similarity index 94% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTest.java rename to server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java index 2da34eb554cc6..fcb39be29d022 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTest.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java @@ -26,7 +26,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class GetSnapshottableFeaturesResponseTest extends AbstractWireSerializingTestCase { +public class GetSnapshottableFeaturesResponseTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { From 410b8bb13c754b24e0190c4a590dd162f8b6ade8 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 17 Nov 2020 11:09:19 -0500 Subject: [PATCH 034/107] Delete indices by UUID rather than name --- .../elasticsearch/indices/IndicesService.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 0c422da7e9ae5..5198729f7b026 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -857,23 +857,25 @@ public void afterIndexShardClosed(ShardId shardId, IndexShard indexShard, Settin * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override - public void deleteUnassignedIndex(String reason, IndexMetadata metadata, ClusterState clusterState) { + public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { - String indexName = metadata.getIndex().getName(); + Index index = oldIndexMetadata.getIndex(); try { - if (clusterState.metadata().hasIndex(indexName)) { - final IndexMetadata index = clusterState.metadata().index(indexName); - throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + - "the cluster state [" + index.getIndexUUID() + "] [" + metadata.getIndexUUID() + "]"); + if (clusterState.metadata().hasIndex(index)) { + final IndexMetadata currentMetadata = clusterState.metadata().index(index); + throw new IllegalStateException("Can't delete unassigned index store for [" + index.getName() + "] - it's still part " + + "of the cluster state [" + currentMetadata.getIndexUUID() + "] [" + + oldIndexMetadata.getIndexUUID() + "]"); } - deleteIndexStore(reason, metadata); + deleteIndexStore(reason, oldIndexMetadata); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", - metadata.getIndex(), reason), e); + oldIndexMetadata.getIndex(), reason), e); } } } + /** * Deletes the index store trying to acquire all shards locks for this index. * This method will delete the metadata for the index even if the actual shards can't be locked. From 952ffb077faae6d35d450d0cb52179915f0c9b45 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 17 Nov 2020 16:49:38 -0700 Subject: [PATCH 035/107] Plumbing through descriptions --- .../GetSnapshottableFeaturesResponse.java | 12 +++- .../TransportSnapshottableFeaturesAction.java | 6 +- .../elasticsearch/indices/SystemIndices.java | 63 ++++++++++++------- .../java/org/elasticsearch/node/Node.java | 5 +- ...GetSnapshottableFeaturesResponseTests.java | 8 ++- .../get/TransportGetAliasesActionTests.java | 6 +- .../action/bulk/TransportBulkActionTests.java | 3 +- .../action/support/AutoCreateIndexTests.java | 3 +- .../MetadataCreateIndexServiceTests.java | 2 +- .../MetadataIndexUpgradeServiceTests.java | 3 +- .../indices/SystemIndicesTests.java | 18 +++--- 11 files changed, 82 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index a852b34c2b8e8..67f7b56a64f13 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -79,27 +79,37 @@ public int hashCode() { public static class SnapshottableFeature implements Writeable, ToXContentObject { private final String featureName; - public SnapshottableFeature(String featureName) { + private final String description; + + public SnapshottableFeature(String featureName, String description) { this.featureName = featureName; + this.description = description; } public SnapshottableFeature(StreamInput in) throws IOException { featureName = in.readString(); + description = in.readString(); } public String getFeatureName() { return featureName; } + public String getDescription() { + return description; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(featureName); + out.writeString(description); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("name", featureName); + builder.field("description", description); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java index 8aa60baff8629..ef284b0bef81c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java @@ -53,8 +53,10 @@ public TransportSnapshottableFeaturesAction(TransportService transportService, C @Override protected void masterOperation(Task task, GetSnapshottableFeaturesRequest request, ClusterState state, ActionListener listener) throws Exception { - listener.onResponse(new GetSnapshottableFeaturesResponse(systemIndices.getFeatures().stream() - .map(GetSnapshottableFeaturesResponse.SnapshottableFeature::new) + listener.onResponse(new GetSnapshottableFeaturesResponse(systemIndices.getFeatures().entrySet().stream() + .map(featureEntry -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( + featureEntry.getKey(), + featureEntry.getValue().getDescription())) .collect(Collectors.toList()))); } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 90a1128693f18..b98ab7b551a3e 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -47,17 +47,17 @@ * to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s. */ public class SystemIndices { - private static final Map> SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( - TASKS_FEATURE_NAME, List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index")) + private static final Map SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( + TASKS_FEATURE_NAME, new Feature("Manages task results", List.of(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"))) ); private final CharacterRunAutomaton runAutomaton; - private final Map> featureSystemIndexDescriptors; + private final Map systemIndexDescriptors; - public SystemIndices(Map> pluginAndModulesDescriptors) { - featureSystemIndexDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); - checkForOverlappingPatterns(featureSystemIndexDescriptors); - this.runAutomaton = buildCharacterRunAutomaton(featureSystemIndexDescriptors); + public SystemIndices(Map pluginAndModulesDescriptors) { + systemIndexDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); + checkForOverlappingPatterns(systemIndexDescriptors); + this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors); } /** @@ -85,8 +85,8 @@ public boolean isSystemIndex(String indexName) { * @throws IllegalStateException if multiple descriptors match the name */ public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) { - final List matchingDescriptors = featureSystemIndexDescriptors.values().stream() - .flatMap(Collection::stream) + final List matchingDescriptors = systemIndexDescriptors.values().stream() + .flatMap(feature -> feature.getIndexDescriptors().stream()) .filter(descriptor -> descriptor.matchesIndexPattern(name)) .collect(toUnmodifiableList()); @@ -116,16 +116,17 @@ public boolean isSystemIndex(String indexName) { public Map> getSystemIndexDescriptorsByFeature() { // GWB-> I don't like exposing this as public, but we also need to be able to get the merged list of plugin/module features // plus internal/built-in ones (i.e. tasks) - return featureSystemIndexDescriptors; + return systemIndexDescriptors.entrySet().stream() + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, feature -> feature.getValue().getIndexDescriptors())); } - public Collection getFeatures() { - return featureSystemIndexDescriptors.keySet(); + public Map getFeatures() { + return systemIndexDescriptors; } - private static CharacterRunAutomaton buildCharacterRunAutomaton(Map> descriptors) { + private static CharacterRunAutomaton buildCharacterRunAutomaton(Map descriptors) { Optional automaton = descriptors.values().stream() - .flatMap(Collection::stream) + .flatMap(feature -> feature.getIndexDescriptors().stream()) .map(descriptor -> Regex.simpleMatchToAutomaton(descriptor.getIndexPattern())) .reduce(Operations::union); return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE)); @@ -138,9 +139,9 @@ private static CharacterRunAutomaton buildCharacterRunAutomaton(Map> sourceToDescriptors) { + static void checkForOverlappingPatterns(Map sourceToDescriptors) { List> sourceDescriptorPair = sourceToDescriptors.entrySet().stream() - .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) + .flatMap(entry -> entry.getValue().getIndexDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message .collect(Collectors.toUnmodifiableList()); @@ -169,18 +170,34 @@ private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; } - private static Map> buildSystemIndexDescriptorMap( - Map> pluginAndModulesMap) { - final Map> map = - new HashMap<>(pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()); - map.putAll(pluginAndModulesMap); + private static Map buildSystemIndexDescriptorMap(Map featuresMap) { + final Map map = new HashMap<>(featuresMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()); + map.putAll(featuresMap); // put the server items last since we expect less of them - SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> { - if (map.putIfAbsent(source, descriptors) != null) { + SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, feature) -> { + if (map.putIfAbsent(source, feature) != null) { throw new IllegalArgumentException("plugin or module attempted to define the same source [" + source + "] as a built-in system index"); } }); return Map.copyOf(map); } + + public static class Feature { + private final String description; + private final Collection indexDescriptors; + + public Feature(String description, Collection indexDescriptors) { + this.description = description; + this.indexDescriptors = indexDescriptors; + } + + public String getDescription() { + return description; + } + + public Collection getIndexDescriptors() { + return indexDescriptors; + } + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index d8c3af5b238a9..f01346ce9111b 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -109,7 +109,6 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.ShardLimitValidator; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.breaker.BreakerSettings; @@ -489,12 +488,12 @@ protected Node(final Environment initialEnvironment, .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map> systemIndexDescriptorMap = pluginsService + final Map systemIndexDescriptorMap = pluginsService .filterPlugins(SystemIndexPlugin.class) .stream() .collect(Collectors.toUnmodifiableMap( plugin -> plugin.getFeatureName(), - plugin -> plugin.getSystemIndexDescriptors(settings))); + plugin -> new SystemIndices.Feature(plugin.getFeatureDescription(), plugin.getSystemIndexDescriptors(settings)))); final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap); final RerouteService rerouteService diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java index fcb39be29d022..3c073cfd959b8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java @@ -36,7 +36,9 @@ protected Writeable.Reader instanceReader() { @Override protected GetSnapshottableFeaturesResponse createTestInstance() { return new GetSnapshottableFeaturesResponse(randomList(10, - () -> new GetSnapshottableFeaturesResponse.SnapshottableFeature(randomAlphaOfLengthBetween(4, 10)))); + () -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5,10)))); } @Override @@ -48,9 +50,9 @@ protected GetSnapshottableFeaturesResponse mutateInstance(GetSnapshottableFeatur Set existingFeatureNames = instance.getSnapshottableFeatures().stream() .map(feature -> feature.getFeatureName()) .collect(Collectors.toSet()); - return new GetSnapshottableFeaturesResponse(randomList(minSize, 10, () -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( - randomValueOtherThanMany(existingFeatureNames::contains, () -> randomAlphaOfLengthBetween(4, 10))))); + randomValueOtherThanMany(existingFeatureNames::contains, () -> randomAlphaOfLengthBetween(4, 10)), + randomAlphaOfLengthBetween(5, 10)))); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 5af88f6fc768c..9b9c4695ce1ce 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -162,8 +162,10 @@ public void testDeprecationWarningNotEmittedWhenOnlyNonsystemIndexRequested() { public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemPattern() { ClusterState state = systemIndexTestClusterState(); - SystemIndices systemIndices = new SystemIndices(Collections.singletonMap(this.getTestName(), - Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist")))); + SystemIndices systemIndices = new SystemIndices(Collections.singletonMap( + this.getTestName(), + new SystemIndices.Feature("test feature", + Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist"))))); GetAliasesRequest request = new GetAliasesRequest(".y"); ImmutableOpenMap> aliases = ImmutableOpenMap.>builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index ff83ff4e4e392..7f8384f94443e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -254,7 +254,8 @@ public void testOnlySystem() { new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); indicesLookup.put(".bar", new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(".test", "")))); + SystemIndices systemIndices = new SystemIndices( + Map.of("plugin", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".test", ""))))); List onlySystem = List.of(".foo", ".bar"); assertTrue(bulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); diff --git a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java index 678a3632cf443..55ec3a3ece046 100644 --- a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java @@ -329,7 +329,8 @@ private static ClusterState buildClusterState(String... indices) { } private AutoCreateIndex newAutoCreateIndex(Settings settings) { - SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "")))); + SystemIndices systemIndices = new SystemIndices(Map.of( + "plugin", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, ""))))); return new AutoCreateIndex(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), systemIndices); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 24c92ad4aceae..46d9dcb827fe8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -531,7 +531,7 @@ public void testValidateDotIndex() { null, threadPool, null, - new SystemIndices(Collections.singletonMap("foo", systemIndexDescriptors)), + new SystemIndices(Collections.singletonMap("foo", new SystemIndices.Feature("test feature", systemIndexDescriptors))), false ); // Check deprecations diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index 590285fa0ca88..c36e74e48ecd0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -161,7 +161,8 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { xContentRegistry(), new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - new SystemIndices(Map.of("system-plugin", List.of(new SystemIndexDescriptor(".system", "a system index")))), + new SystemIndices(Map.of("system-plugin", + new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".system", "a system index"))))), null ); } diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java index 1c5d1568bdf37..424a53ad1e42a 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.test.ESTestCase; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,9 +44,10 @@ public void testBasicOverlappingPatterns() { // across tests String broadPatternSource = "AAA" + randomAlphaOfLength(5); String otherSource = "ZZZ" + randomAlphaOfLength(6); - Map> descriptors = new HashMap<>(); - descriptors.put(broadPatternSource, List.of(broadPattern)); - descriptors.put(otherSource, List.of(notOverlapping, overlapping1, overlapping2, overlapping3)); + Map descriptors = new HashMap<>(); + descriptors.put(broadPatternSource, new SystemIndices.Feature("test feature", List.of(broadPattern))); + descriptors.put(otherSource, + new SystemIndices.Feature("test 2", List.of(notOverlapping, overlapping1, overlapping2, overlapping3))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> SystemIndices.checkForOverlappingPatterns(descriptors)); @@ -72,9 +72,9 @@ public void testComplexOverlappingPatterns() { // across tests String source1 = "AAA" + randomAlphaOfLength(5); String source2 = "ZZZ" + randomAlphaOfLength(6); - Map> descriptors = new HashMap<>(); - descriptors.put(source1, List.of(pattern1)); - descriptors.put(source2, List.of(pattern2)); + Map descriptors = new HashMap<>(); + descriptors.put(source1, new SystemIndices.Feature("test", List.of(pattern1))); + descriptors.put(source2, new SystemIndices.Feature("test", List.of(pattern2))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> SystemIndices.checkForOverlappingPatterns(descriptors)); @@ -94,8 +94,8 @@ public void testBuiltInSystemIndices() { } public void testPluginCannotOverrideBuiltInSystemIndex() { - Map> pluginMap = Map.of( - TASKS_FEATURE_NAME, List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) + Map pluginMap = Map.of( + TASKS_FEATURE_NAME, new SystemIndices.Feature("test", List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index"))) ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SystemIndices(pluginMap)); assertThat(e.getMessage(), containsString("plugin or module attempted to define the same source")); From 7ba77bd367c68b93616c9f3ff2b638e5fb949240 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 18 Nov 2020 15:33:47 -0500 Subject: [PATCH 036/107] Allow restoring by feature state --- .../snapshots/SystemIndicesSnapshotIT.java | 22 +++++++++--- .../create/CreateSnapshotRequestBuilder.java | 11 ++++++ .../RestoreSnapshotRequestBuilder.java | 8 +++++ .../snapshots/RestoreService.java | 35 +++++++++++++++---- .../elasticsearch/snapshots/SnapshotInfo.java | 4 +++ 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 21f1b2215cad9..b39e775893008 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -138,7 +138,7 @@ public void testSnapshotByFeature() { // snapshot by feature CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) - // setFeatureList() + .setFeatureStates("TestSystemIndex") .get(); assertSnapshotSuccess(createSnapshotResponse); @@ -195,13 +195,15 @@ public void testRestoreByFeature() { // restore indices by feature RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) - // restore by feature + .setFeatureStates("SystemIndexTestPlugin") .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); // verify only the original document is restored assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); - assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + + // but the non-requested feature should still have its new document + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { @@ -224,9 +226,14 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return SystemIndexTestPlugin.class.getSimpleName(); } + + @Override + public String getFeatureDescription() { + return "A simple test plugin"; + } } public static class AnotherSystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { @@ -239,8 +246,13 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public String getPluginName() { + public String getFeatureName() { return AnotherSystemIndexTestPlugin.class.getSimpleName(); } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin"; + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index b65912da5dba0..106bfbad199e8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -122,4 +122,15 @@ public CreateSnapshotRequestBuilder setIncludeGlobalState(boolean includeGlobalS request.includeGlobalState(includeGlobalState); return this; } + + /** + * Provide a list of features whose state indices should be included in the snapshot + * + * @param featureStates A list of feature names + * @return this builder + */ + public CreateSnapshotRequestBuilder setFeatureStates(String... featureStates) { + request.featureStates(featureStates); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 08d509ce3ac74..423b2f9b3b999 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -234,4 +234,12 @@ public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(List ignoreI request.ignoreIndexSettings(ignoreIndexSettings); return this; } + + /** + * Sets the list of features whose states should be restored as part of this snapshot + */ + public RestoreSnapshotRequestBuilder setFeatureStates(String... featureStates) { + request.featureStates(featureStates); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 378e846eacc60..a72de20063100 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -68,6 +68,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; @@ -80,6 +81,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -91,6 +93,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; @@ -268,14 +271,32 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, metadataBuilder = Metadata.builder(); } - List systemIndices = new ArrayList<>(); - // is this filtered by what's in the request? - final List indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot); + // set up feature state indices + Set requestedFeatureStateIndexes = new HashSet<>(); + Set nonRequestedFeatureStateIndexes = new HashSet<>(); + + if (request.includeGlobalState() == false) { + for (SnapshotFeatureInfo snapshotFeatureInfo : snapshotInfo.featureStates()) { + if (Set.of(request.featureStates()).contains(snapshotFeatureInfo.getPluginName())) { + requestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); + } else { + nonRequestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); + } + } + } + + List requestedIndicesIncludingSystem = Stream.concat( + indicesInSnapshot.stream(), requestedFeatureStateIndexes.stream()) + .distinct() + .filter(index -> nonRequestedFeatureStateIndexes.contains(index) == false) + .collect(Collectors.toList()); + + List systemIndicesToDelete = new ArrayList<>(); + final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); if (snapshotIndexMetaData.isSystem()) { - // add to system indices list - systemIndices.add(snapshotIndexMetaData.getIndex()); + systemIndicesToDelete.add(snapshotIndexMetaData.getIndex()); } metadataBuilder.put(snapshotIndexMetaData, false); } @@ -285,7 +306,7 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, // Apply renaming on index names, returning a map of names where // the key is the renamed index and the value is the original name // TODO: don't apply to system indices - final Map indices = renamedIndices(request, indicesInSnapshot, dataStreamIndices); + final Map indices = renamedIndices(request, requestedIndicesIncludingSystem, dataStreamIndices); // Now we can start the actual restore process by adding shards to be recovered in the cluster state // and updating cluster metadata (global and index) as needed @@ -305,7 +326,7 @@ public ClusterState execute(ClusterState currentState) { deletionsInProgress.getEntries().get(0) + "]"); } - currentState = metadataDeleteIndexService.deleteIndices(currentState, new HashSet<>(systemIndices)); + currentState = metadataDeleteIndexService.deleteIndices(currentState, new HashSet<>(systemIndicesToDelete)); // Updating cluster state ClusterState.Builder builder = ClusterState.builder(currentState); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 93fbc9ba6acaa..2c92dedd91b79 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -479,6 +479,10 @@ public Map userMetadata() { return userMetadata; } + public List featureStates() { + return List.copyOf(featureStates); + } + /** * Compares two snapshots by their start time; if the start times are the same, then * compares the two snapshots by their snapshot ids. From d544f1ee4edbb08282318fe1cae831195d8c66c1 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 19 Nov 2020 11:55:41 -0500 Subject: [PATCH 037/107] Code review fixes --- .../java/org/elasticsearch/snapshots/RestoreService.java | 9 ++++----- .../java/org/elasticsearch/snapshots/SnapshotInfo.java | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index a72de20063100..bb1c9a2406b3e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -68,7 +68,6 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; @@ -81,7 +80,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -275,9 +273,10 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, Set requestedFeatureStateIndexes = new HashSet<>(); Set nonRequestedFeatureStateIndexes = new HashSet<>(); + Set requestedFeatureStates = Set.of(request.featureStates()); if (request.includeGlobalState() == false) { for (SnapshotFeatureInfo snapshotFeatureInfo : snapshotInfo.featureStates()) { - if (Set.of(request.featureStates()).contains(snapshotFeatureInfo.getPluginName())) { + if (requestedFeatureStates.contains(snapshotFeatureInfo.getPluginName())) { requestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); } else { nonRequestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); @@ -291,7 +290,7 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, .filter(index -> nonRequestedFeatureStateIndexes.contains(index) == false) .collect(Collectors.toList()); - List systemIndicesToDelete = new ArrayList<>(); + Set systemIndicesToDelete = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); @@ -326,7 +325,7 @@ public ClusterState execute(ClusterState currentState) { deletionsInProgress.getEntries().get(0) + "]"); } - currentState = metadataDeleteIndexService.deleteIndices(currentState, new HashSet<>(systemIndicesToDelete)); + currentState = metadataDeleteIndexService.deleteIndices(currentState, systemIndicesToDelete); // Updating cluster state ClusterState.Builder builder = ClusterState.builder(currentState); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 2c92dedd91b79..f516577bad44c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -480,7 +480,7 @@ public Map userMetadata() { } public List featureStates() { - return List.copyOf(featureStates); + return featureStates; } /** From ab4106a427d1976a722369e7db574a0fb32ab706 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 1 Dec 2020 17:34:15 -0700 Subject: [PATCH 038/107] Treat a `null` and empty list of feature states differently A `null` means "default to `include_global_state`" - i.e. restore all system indices if it's `true`, or none otherwise. An empty list means restore *no* system indices, even if `include_global_state` is `true`. --- .../snapshots/SystemIndicesSnapshotIT.java | 9 +++++- .../restore/RestoreSnapshotRequest.java | 10 +++---- .../snapshots/RestoreService.java | 29 ++++++++++--------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index b39e775893008..c6b7e4e3c84e6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -169,6 +169,9 @@ public void testRestoreByFeature() { .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); assertAcked(prepareCreate(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + // And one regular index + final String regularIndex = "test-idx"; + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); ensureGreen(); @@ -192,9 +195,13 @@ public void testRestoreByFeature() { assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); - // restore indices by feature + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore indices by feature, with only the regular index named explicitly RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) + .setIndices(regularIndex) .setFeatureStates("SystemIndexTestPlugin") .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index d15dfff0c696c..cde7a8835ec5e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -54,7 +54,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest requestedFeatureStateIndexes = new HashSet<>(); - Set nonRequestedFeatureStateIndexes = new HashSet<>(); - - Set requestedFeatureStates = Set.of(request.featureStates()); - if (request.includeGlobalState() == false) { - for (SnapshotFeatureInfo snapshotFeatureInfo : snapshotInfo.featureStates()) { - if (requestedFeatureStates.contains(snapshotFeatureInfo.getPluginName())) { - requestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); - } else { - nonRequestedFeatureStateIndexes.addAll(snapshotFeatureInfo.getIndices()); - } + Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() + .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); + + Map> featureStatesToRestore = new HashMap<>(snapshotFeatureStates); + if (request.featureStates() != null) { + final Set requestedStates = Set.of(request.featureStates()); + if (snapshotFeatureStates.keySet().containsAll(requestedStates) == false) { + Set nonExistingRequestedStates = new HashSet<>(requestedStates); + nonExistingRequestedStates.removeAll(snapshotFeatureStates.keySet()); + throw new SnapshotRestoreException(snapshot, "requested feature states [" + nonExistingRequestedStates + + "] are not present in snapshot"); } + featureStatesToRestore.keySet().retainAll(requestedStates); + } else if (request.includeGlobalState() == false) { + featureStatesToRestore = new HashMap<>(); } List requestedIndicesIncludingSystem = Stream.concat( - indicesInSnapshot.stream(), requestedFeatureStateIndexes.stream()) + indicesInSnapshot.stream(), featureStatesToRestore.values().stream().flatMap(Collection::stream)) .distinct() - .filter(index -> nonRequestedFeatureStateIndexes.contains(index) == false) .collect(Collectors.toList()); Set systemIndicesToDelete = new HashSet<>(); From 031e86e3c0ba023afdd09c9c9a28d5c6ac1e9c4b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 2 Dec 2020 16:35:48 -0700 Subject: [PATCH 039/107] Don't rename system indices --- .../snapshots/RestoreService.java | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f279ef64a6909..c4c368f9f30ab 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -271,30 +271,17 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, metadataBuilder = Metadata.builder(); } - // set up feature state indices - Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() - .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); - - Map> featureStatesToRestore = new HashMap<>(snapshotFeatureStates); - if (request.featureStates() != null) { - final Set requestedStates = Set.of(request.featureStates()); - if (snapshotFeatureStates.keySet().containsAll(requestedStates) == false) { - Set nonExistingRequestedStates = new HashSet<>(requestedStates); - nonExistingRequestedStates.removeAll(snapshotFeatureStates.keySet()); - throw new SnapshotRestoreException(snapshot, "requested feature states [" + nonExistingRequestedStates + - "] are not present in snapshot"); - } - featureStatesToRestore.keySet().retainAll(requestedStates); - } else if (request.includeGlobalState() == false) { - featureStatesToRestore = new HashMap<>(); - } + final Map> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot); - List requestedIndicesIncludingSystem = Stream.concat( - indicesInSnapshot.stream(), featureStatesToRestore.values().stream().flatMap(Collection::stream)) + final Set featureStateIndices = featureStatesToRestore.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + final List requestedIndicesIncludingSystem = Stream.concat(indicesInSnapshot.stream(), featureStateIndices.stream()) .distinct() .collect(Collectors.toList()); - Set systemIndicesToDelete = new HashSet<>(); + final Set systemIndicesToDelete = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); @@ -308,8 +295,8 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, // Apply renaming on index names, returning a map of names where // the key is the renamed index and the value is the original name - // TODO: don't apply to system indices - final Map indices = renamedIndices(request, requestedIndicesIncludingSystem, dataStreamIndices); + final Map indices = renamedIndices(request, requestedIndicesIncludingSystem, dataStreamIndices, + featureStateIndices); // Now we can start the actual restore process by adding shards to be recovered in the cluster state // and updating cluster metadata (global and index) as needed @@ -650,6 +637,27 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } } + private Map> getFeatureStatesToRestore(RestoreSnapshotRequest request, SnapshotInfo snapshotInfo, + Snapshot snapshot) { + final Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() + .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); + + final Map> featureStatesToRestore = new HashMap<>(snapshotFeatureStates); + if (request.featureStates() != null) { + final Set requestedStates = Set.of(request.featureStates()); + if (snapshotFeatureStates.keySet().containsAll(requestedStates) == false) { + Set nonExistingRequestedStates = new HashSet<>(requestedStates); + nonExistingRequestedStates.removeAll(snapshotFeatureStates.keySet()); + throw new SnapshotRestoreException(snapshot, "requested feature states [" + nonExistingRequestedStates + + "] are not present in snapshot"); + } + featureStatesToRestore.keySet().retainAll(requestedStates); + } else if (request.includeGlobalState() == false) { + featureStatesToRestore.clear(); + } + return featureStatesToRestore; + } + //visible for testing static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) { String dataStreamName = dataStream.getName(); @@ -923,10 +931,16 @@ public static int failedShards(ImmutableOpenMap renamedIndices(RestoreSnapshotRequest request, List filteredIndices, - Set dataStreamIndices) { + Set dataStreamIndices, Set featureIndices) { Map renamedIndices = new HashMap<>(); for (String index : filteredIndices) { - String renamedIndex = renameIndex(index, request, dataStreamIndices.contains(index)); + String renamedIndex; + if (featureIndices.contains(index)) { + // Don't rename system indices + renamedIndex = index; + } else { + renamedIndex = renameIndex(index, request, dataStreamIndices.contains(index)); + } String previousIndex = renamedIndices.put(renamedIndex, index); if (previousIndex != null) { throw new SnapshotRestoreException(request.repository(), request.snapshot(), From 9b6d1ff5d88c21d427ae37797a0c32044c8114fe Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 2 Dec 2020 17:09:42 -0700 Subject: [PATCH 040/107] Refactoring/cleanup. No functionality changes --- .../snapshots/RestoreService.java | 87 +++++++++++-------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c4c368f9f30ab..8f00e00f9b27f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -232,54 +232,47 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, // Make sure that we can restore from this snapshot validateSnapshotRestorable(repositoryName, snapshotInfo); + // Get the global state if necessary Metadata globalMetadata = null; - // Resolve the indices from the snapshot that need to be restored - Map dataStreams; - List requestIndices = new ArrayList<>(Arrays.asList(request.indices())); - - List requestedDataStreams = filterIndices(snapshotInfo.dataStreams(), requestIndices.toArray(String[]::new), - IndicesOptions.fromOptions(true, true, true, true)); - if (requestedDataStreams.isEmpty()) { - dataStreams = new HashMap<>(); - } else { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); - final Map dataStreamsInSnapshot = globalMetadata.dataStreams(); - dataStreams = new HashMap<>(requestedDataStreams.size()); - for (String requestedDataStream : requestedDataStreams) { - final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream); - assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot"; - dataStreams.put(requestedDataStream, dataStreamInSnapshot); - } - } - requestIndices.removeAll(dataStreams.keySet()); - Set dataStreamIndices = dataStreams.values().stream() - .flatMap(ds -> ds.getIndices().stream()) - .map(Index::getName) - .collect(Collectors.toSet()); - requestIndices.addAll(dataStreamIndices); - - final List indicesInSnapshot = filterIndices(snapshotInfo.indices(), requestIndices.toArray(String[]::new), - request.indicesOptions()); - final Metadata.Builder metadataBuilder; if (request.includeGlobalState()) { - if (globalMetadata == null) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); - } + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); metadataBuilder = Metadata.builder(globalMetadata); } else { metadataBuilder = Metadata.builder(); } - final Map> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot); + List requestIndices = new ArrayList<>(Arrays.asList(request.indices())); + // Get data stream metadata for requested data streams + Map dataStreamsToRestore = getDataStreamsToRestore(repository, snapshotId, snapshotInfo, globalMetadata, + requestIndices); + + // Remove the data streams from the list of requested indices + requestIndices.removeAll(dataStreamsToRestore.keySet()); + + // And add the backing indices + Set dataStreamIndices = dataStreamsToRestore.values().stream() + .flatMap(ds -> ds.getIndices().stream()) + .map(Index::getName) + .collect(Collectors.toSet()); + requestIndices.addAll(dataStreamIndices); + + // Determine system indices to restore from requested feature states + final Map> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot); final Set featureStateIndices = featureStatesToRestore.values().stream() .flatMap(Collection::stream) .collect(Collectors.toSet()); - final List requestedIndicesIncludingSystem = Stream.concat(indicesInSnapshot.stream(), featureStateIndices.stream()) - .distinct() - .collect(Collectors.toList()); + // Resolve the indices that were directly requested + final List requestedIndicesInSnapshot = filterIndices(snapshotInfo.indices(), requestIndices.toArray(String[]::new), + request.indicesOptions()); + + // Combine into the final list of indices to be restored + final List requestedIndicesIncludingSystem = Stream.concat( + requestedIndicesInSnapshot.stream(), + featureStateIndices.stream() + ).distinct().collect(Collectors.toList()); final Set systemIndicesToDelete = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); @@ -457,7 +450,7 @@ restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), checkAliasNameConflicts(indices, aliases); Map updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams()); - updatedDataStreams.putAll(dataStreams.values().stream() + updatedDataStreams.putAll(dataStreamsToRestore.values().stream() .map(ds -> updateDataStream(ds, mdBuilder, request)) .collect(Collectors.toMap(DataStream::getName, Function.identity()))); mdBuilder.dataStreams(updatedDataStreams); @@ -637,6 +630,28 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } } + private Map getDataStreamsToRestore(Repository repository, SnapshotId snapshotId, SnapshotInfo snapshotInfo, + Metadata globalMetadata, List requestIndices) { + Map dataStreams; + List requestedDataStreams = filterIndices(snapshotInfo.dataStreams(), requestIndices.toArray(String[]::new), + IndicesOptions.fromOptions(true, true, true, true)); + if (requestedDataStreams.isEmpty()) { + dataStreams = new HashMap<>(); + } else { + if (globalMetadata == null) { + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); + } + final Map dataStreamsInSnapshot = globalMetadata.dataStreams(); + dataStreams = new HashMap<>(requestedDataStreams.size()); + for (String requestedDataStream : requestedDataStreams) { + final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream); + assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot"; + dataStreams.put(requestedDataStream, dataStreamInSnapshot); + } + } + return dataStreams; + } + private Map> getFeatureStatesToRestore(RestoreSnapshotRequest request, SnapshotInfo snapshotInfo, Snapshot snapshot) { final Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() From c8ca4a16998163dc073a2855149fe3432371241f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 3 Dec 2020 17:30:11 -0700 Subject: [PATCH 041/107] Name and description for Fleet --- .../main/java/org/elasticsearch/xpack/fleet/Fleet.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 1aee05afe4471..1b70da63ee7fe 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -30,4 +30,14 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(".fleet-actions*", "Fleet actions") ); } + + @Override + public String getFeatureName() { + return "fleet"; + } + + @Override + public String getFeatureDescription() { + return "Manages configuration for Fleet"; + } } From c16d60f78cd14af7874cfbeb38d6909cccca1d40 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 4 Dec 2020 14:07:51 -0500 Subject: [PATCH 042/107] Add deprecation warning for directly restoring system indices --- .../snapshots/SystemIndicesSnapshotIT.java | 48 +++++++++++++++++++ .../snapshots/RestoreService.java | 13 +++++ 2 files changed, 61 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index c6b7e4e3c84e6..11c7cc7e67982 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -19,12 +19,16 @@ package org.elasticsearch.snapshots; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.test.MockLogAppender; import java.util.ArrayList; import java.util.Collection; @@ -213,6 +217,50 @@ public void testRestoreByFeature() { assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessException { + createRepository("test-repo", "fs"); + + // create system index + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + // put a document in system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + MockLogAppender mockLogAppender = new MockLogAppender(); + Loggers.addAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); + mockLogAppender.start(); + mockLogAppender.addExpectation(new MockLogAppender.SeenEventExpectation( + "restore-system-index-from-snapshot", + "org.elasticsearch.deprecation.snapshots.RestoreService", + DeprecationLogger.DEPRECATION, + "Restoring system indices by name is deprecated. Use feature states instead. System indices: [.test-system-idx]")); + + // restore system index by name, rather than feature state + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIndices(SystemIndexTestPlugin.SYSTEM_INDEX_NAME) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + mockLogAppender.assertAllExpectationsMatched(); + mockLogAppender.stop(); + Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 8f00e00f9b27f..fde238ca47048 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -64,6 +64,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; @@ -130,6 +131,7 @@ public class RestoreService implements ClusterStateApplier { private static final Logger logger = LogManager.getLogger(RestoreService.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestoreService.class); private static final Set UNMODIFIABLE_SETTINGS = unmodifiableSet(newHashSet( SETTING_NUMBER_OF_SHARDS, @@ -275,15 +277,26 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, ).distinct().collect(Collectors.toList()); final Set systemIndicesToDelete = new HashSet<>(); + Set explicitlyRequestedSystemIndices = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); if (snapshotIndexMetaData.isSystem()) { systemIndicesToDelete.add(snapshotIndexMetaData.getIndex()); + if (requestedIndicesInSnapshot.contains(indexId.getName())) { + explicitlyRequestedSystemIndices.add(indexId.getName()); + } } metadataBuilder.put(snapshotIndexMetaData, false); } + // log a deprecation warning if the any of the indexes to delete were included in the request + if (explicitlyRequestedSystemIndices.isEmpty() == false) { + deprecationLogger.deprecate("restore-system-index-from-snapshot", + "Restoring system indices by name is deprecated. Use feature states instead. System indices: " + + explicitlyRequestedSystemIndices); + } + final Metadata metadata = metadataBuilder.build(); // Apply renaming on index names, returning a map of names where From 3b0fb7f7d7c6d10133dddcac5840c06a5793a5b7 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 4 Dec 2020 14:08:21 -0500 Subject: [PATCH 043/107] Add test for bad feature state request --- .../snapshots/SystemIndicesSnapshotIT.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 11c7cc7e67982..59d6877434008 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -40,6 +40,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.in; @@ -217,6 +218,38 @@ public void testRestoreByFeature() { assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + public void testRestoreFeatureNotInSnapshot() { + createRepository("test-repo", "fs"); + + // create indices + assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + // put a document in each one + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + final String fakeFeatureStateName = "NonExistentTestPlugin"; + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setFeatureStates("SystemIndexTestPlugin", fakeFeatureStateName) + .get()); + + assertThat(exception.getMessage(), + containsString("requested feature states [[" + fakeFeatureStateName + "]] are not present in snapshot")); + } + public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessException { createRepository("test-repo", "fs"); From cfbfbcba0f7d87e73224fa344c882e27872bcf8c Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 4 Dec 2020 14:45:49 -0500 Subject: [PATCH 044/107] Make HashSet variable final --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index fde238ca47048..5460bed7624d9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -277,7 +277,7 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, ).distinct().collect(Collectors.toList()); final Set systemIndicesToDelete = new HashSet<>(); - Set explicitlyRequestedSystemIndices = new HashSet<>(); + final Set explicitlyRequestedSystemIndices = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); From 091ca4cda3016735f38c218a6424dbca05790f96 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 11:30:02 -0700 Subject: [PATCH 045/107] Only warn when explicitly requesting system indices if the snapshot is from a version with feature states --- .../java/org/elasticsearch/snapshots/RestoreService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 5460bed7624d9..a7c7e0ef0ef91 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -97,6 +97,7 @@ import java.util.stream.Stream; import static java.util.Collections.unmodifiableSet; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; @@ -290,8 +291,9 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, metadataBuilder.put(snapshotIndexMetaData, false); } - // log a deprecation warning if the any of the indexes to delete were included in the request - if (explicitlyRequestedSystemIndices.isEmpty() == false) { + // log a deprecation warning if the any of the indexes to delete were included in the request and the snapshot + // is from a version that should have feature states + if (snapshotInfo.version().onOrAfter(FEATURE_STATES_VERSION) && explicitlyRequestedSystemIndices.isEmpty() == false) { deprecationLogger.deprecate("restore-system-index-from-snapshot", "Restoring system indices by name is deprecated. Use feature states instead. System indices: " + explicitlyRequestedSystemIndices); From 05fd6d71351992ed6d2def1d43c7345350c37ffd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 11:31:38 -0700 Subject: [PATCH 046/107] Some missed s/plugin/feature/ --- .../java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index 965c2cf0673e9..7d0497a0428cc 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -38,7 +38,7 @@ public class SnapshotFeatureInfo implements Writeable, ToXContentObject { final List indices; static final ConstructingObjectParser SNAPSHOT_FEATURE_INFO_PARSER = - new ConstructingObjectParser<>("plugin_info", true, (a, name) -> { + new ConstructingObjectParser<>("feature_info", true, (a, name) -> { String pluginName = (String) a[0]; List indices = (List) a[1]; return new SnapshotFeatureInfo(pluginName, indices); @@ -81,7 +81,7 @@ public List getIndices() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field("plugin_name", pluginName); + builder.field("feature_name", pluginName); builder.startArray("indices"); for (String index : indices) { builder.value(index); From 4af0a2417d2a0d941f232e096ecf6f38f01efcab Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 11:45:49 -0700 Subject: [PATCH 047/107] Address a couple minor TODO comments --- .../src/test/java/org/elasticsearch/client/SnapshotIT.java | 1 - .../main/java/org/elasticsearch/cluster/SnapshotsInProgress.java | 1 - 2 files changed, 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 2f2ca9c65fce4..f8941077b2e80 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -263,7 +263,6 @@ public void testRestoreSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot); createSnapshotRequest.indices(testIndex); createSnapshotRequest.waitForCompletion(true); - createSnapshotRequest.includeGlobalState(false); // GWB-> Remove this once index replacement is in place if (randomBoolean()) { createSnapshotRequest.userMetadata(randomUserMetadata()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 2c97f19502516..4a3cedea5632b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -117,7 +117,6 @@ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, */ public static Entry startClone(Snapshot snapshot, SnapshotId source, List indices, long startTime, long repositoryStateId, Version version) { - // GWB-> Is featureStates right here? return new SnapshotsInProgress.Entry(snapshot, true, false, State.STARTED, indices, Collections.emptyList(), Collections.emptyList(), startTime, repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, ImmutableOpenMap.of()); From 439679df5388774561eb6d1db8bca001045271b7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 11:49:51 -0700 Subject: [PATCH 048/107] Address TODO by generating random feature infos in test --- .../cluster/snapshots/get/GetSnapshotsResponseTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index ad6b270600913..2f405fdd41275 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotInfoTests; @@ -44,6 +45,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; +import static org.elasticsearch.snapshots.SnapshotFeatureInfoTests.randomSnapshotFeatureInfo; import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; import static org.hamcrest.CoreMatchers.containsString; @@ -81,9 +83,9 @@ private List createSnapshotInfos() { String reason = randomBoolean() ? null : "reason"; ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); - // GWB-> generate actual feature states here + List featureInfos = randomList(0, () -> randomSnapshotFeatureInfo()); snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), - Collections.emptyList(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), + featureInfos, reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), SnapshotInfoTests.randomUserMetadata(), System.currentTimeMillis() )); From 3c3b1b86d10eb9bc2f432820c011e8f7749a6bcd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 16:01:23 -0700 Subject: [PATCH 049/107] One missed s/plugin/feature/ --- .../java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index 7d0497a0428cc..d553922394afb 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -45,7 +45,7 @@ public class SnapshotFeatureInfo implements Writeable, ToXContentObject { }); static { - SNAPSHOT_FEATURE_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("plugin_name")); + SNAPSHOT_FEATURE_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("feature_name")); SNAPSHOT_FEATURE_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); } From d36f6c844f003d4a9b2b1a7c37c3519e4d0550a7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 8 Dec 2020 16:20:10 -0700 Subject: [PATCH 050/107] Basic associated indices implementation --- .../elasticsearch/indices/SystemIndices.java | 13 +++++++++++- .../java/org/elasticsearch/node/Node.java | 13 ++++++++---- .../plugins/SystemIndexPlugin.java | 10 ++++++++++ .../snapshots/SnapshotsService.java | 20 ++++++++++--------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 849db7c20a51b..3470f4d007957 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.Index; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -214,10 +215,16 @@ Collection getSystemIndexDescriptors() { public static class Feature { private final String description; private final Collection indexDescriptors; + private final Collection associatedIndexPatterns; - public Feature(String description, Collection indexDescriptors) { + public Feature(String description, Collection indexDescriptors, Collection associatedIndexPatterns) { this.description = description; this.indexDescriptors = indexDescriptors; + this.associatedIndexPatterns = associatedIndexPatterns; + } + + public Feature(String description, Collection indexDescriptors) { + this(description, indexDescriptors, Collections.emptyList()); } public String getDescription() { @@ -227,5 +234,9 @@ public String getDescription() { public Collection getIndexDescriptors() { return indexDescriptors; } + + public Collection getAssociatedIndexPatterns() { + return associatedIndexPatterns; + } } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index d4920fc0661db..02e9d06c5726f 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -490,13 +490,18 @@ protected Node(final Environment initialEnvironment, .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map systemIndexDescriptorMap = pluginsService + final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) .stream() .collect(Collectors.toUnmodifiableMap( plugin -> plugin.getFeatureName(), - plugin -> new SystemIndices.Feature(plugin.getFeatureDescription(), plugin.getSystemIndexDescriptors(settings)))); - final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap); + plugin -> new SystemIndices.Feature( + plugin.getFeatureDescription(), + plugin.getSystemIndexDescriptors(settings), + plugin.getAssociatedIndexPatterns() + )) + ); + final SystemIndices systemIndices = new SystemIndices(featuresMap); final SystemIndexManager systemIndexManager = new SystemIndexManager(systemIndices, client); clusterService.addListener(systemIndexManager); @@ -587,7 +592,7 @@ protected Node(final Environment initialEnvironment, repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService(settings, clusterService, clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters(), - systemIndices.getSystemIndexDescriptorsByFeature()); + systemIndices.getFeatures()); SnapshotShardsService snapshotShardsService = new SnapshotShardsService(settings, clusterService, repositoryService, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 4054c9904ae38..71f121c4b4318 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -50,4 +50,14 @@ default Collection getSystemIndexDescriptors(Settings set * @return A description of the feature, as used for the Get Snapshottable Features API. */ String getFeatureDescription(); + + /** + * Returns a list of index patterns for "associated indices": indices which depend on this plugin's system indices, but are not + * themselves system indices. + * + * @return A list of index patterns which depend on the contents of this plugin's system indices, but are not themselves system indices + */ + default Collection getAssociatedIndexPatterns() { + return Collections.emptyList(); + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d207466d6a44d..c793c424bd4e3 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -48,8 +48,6 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; import org.elasticsearch.cluster.SnapshotsInProgress.ShardState; import org.elasticsearch.cluster.SnapshotsInProgress.State; @@ -77,9 +75,11 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -172,7 +172,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final OngoingRepositoryOperations repositoryOperations = new OngoingRepositoryOperations(); - private final Map> systemIndexDescriptorMap; + private final Map systemIndexDescriptorMap; /** * Setting that specifies the maximum number of allowed concurrent snapshot create and delete operations in the @@ -186,7 +186,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public SnapshotsService(Settings settings, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters, - Map> systemIndexDescriptorMap) { + Map systemIndexDescriptorMap) { this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.repositoriesService = repositoriesService; @@ -326,13 +326,15 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } - private List resolveFeatureIndexNames(ClusterState currentState, String feature) { - if (systemIndexDescriptorMap.containsKey(feature) == false) { - throw new IllegalArgumentException("requested snapshot of feature state for unknown feature [" + feature + "]"); + private List resolveFeatureIndexNames(ClusterState currentState, String featureName) { + if (systemIndexDescriptorMap.containsKey(featureName) == false) { + throw new IllegalArgumentException("requested snapshot of feature state for unknown feature [" + featureName + "]"); } - return systemIndexDescriptorMap.get(feature).stream() - .map(SystemIndexDescriptor::getIndexPattern) + final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); + final Stream systemIndexPatterns = feature.getIndexDescriptors().stream().map(SystemIndexDescriptor::getIndexPattern); + final Stream associatedIndexPatterns = feature.getAssociatedIndexPatterns().stream(); + return Stream.concat(systemIndexPatterns, associatedIndexPatterns) .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, LENIENT_EXPAND_OPEN_CLOSED, pattern))) .collect(Collectors.toList()); From ef88902fd57ec7a5506ce533253e9c99dd23c27f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 9 Dec 2020 15:23:52 -0700 Subject: [PATCH 051/107] Remove unused method --- .../java/org/elasticsearch/indices/SystemIndices.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 3470f4d007957..2c3c68b8a8e75 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -132,17 +132,6 @@ public boolean isSystemIndex(String indexName) { } } - /** - * Gets all system index descriptors, collected by feature. - * @return A Map of feature name to system index descriptors - */ - public Map> getSystemIndexDescriptorsByFeature() { - // GWB-> I don't like exposing this as public, but we also need to be able to get the merged list of plugin/module features - // plus internal/built-in ones (i.e. tasks) - return featureDescriptors.entrySet().stream() - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, feature -> feature.getValue().getIndexDescriptors())); - } - public Map getFeatures() { return featureDescriptors; } From 10f28020d6aa16e223f8340169fced8ade079d8f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 9 Dec 2020 16:02:57 -0700 Subject: [PATCH 052/107] Don't include associated indices in the feature state (but snapshot them) We don't control the namespace for hidden indices as much as we do for system indices, so we shouldn't just stomp on any existing indices that conflict with associated indices. But we can at least ensure they're included in the snapshot. --- .../snapshots/SystemIndicesSnapshotIT.java | 94 ++++++++++++++++++- .../snapshots/SnapshotsService.java | 36 +++++-- 2 files changed, 121 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 59d6877434008..85a6cd92225b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -43,6 +43,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.in; public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { @@ -52,6 +53,7 @@ protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(SystemIndexTestPlugin.class); plugins.add(AnotherSystemIndexTestPlugin.class); + plugins.add(AssociatedIndicesTestPlugin.class); return plugins; } @@ -120,7 +122,7 @@ public void testSnapshotWithoutGlobalState() { .collect(Collectors.toSet()); assertThat("not-a-system-index", in(snapshottedIndices)); - // TODO: without global state the system index shouldn't be snapshotted + // TODO: without global state the system index shouldn't be snapshotted (8.0 & later only) // assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices))); } @@ -143,7 +145,7 @@ public void testSnapshotByFeature() { // snapshot by feature CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) - .setFeatureStates("TestSystemIndex") + .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName()) .get(); assertSnapshotSuccess(createSnapshotResponse); @@ -218,6 +220,68 @@ public void testRestoreByFeature() { assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + public void testSnapshotAndRestoreAssociatedIndices() { + createRepository("test-repo", "fs"); + + final String regularIndex = "regular-idx"; + + // create indices + assertAcked(prepareCreate(regularIndex, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + assertAcked(prepareCreate(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + assertAcked(prepareCreate(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, 2, Settings.builder() + .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); + + ensureGreen(); + + // put a document in each one + indexDoc(regularIndex, "1", "purpose", "pre-snaphost doc"); + indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, + AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME); + + // snapshot + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setIndices(regularIndex) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .setWaitForCompletion(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots("test-repo").get() + .getSnapshots("test-repo").stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME)); + assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME)); + + // add some other documents + indexDoc(regularIndex, "2", "purpose", "post-snapshot doc"); + indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(regularIndex), equalTo(2L)); + assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // And delete the associated index so we can restore it + assertAcked(client().admin().indices().prepareDelete(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME).get()); + + // restore the feature state and its associated index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME) + .setWaitForCompletion(true) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L)); + } + public void testRestoreFeatureNotInSnapshot() { createRepository("test-repo", "fs"); @@ -343,4 +407,30 @@ public String getFeatureDescription() { return "Another simple test plugin"; } } + + public static class AssociatedIndicesTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".third-test-system-idx"; + public static final String ASSOCIATED_INDEX_NAME = ".associated-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System & associated indices for tests")); + } + + @Override + public Collection getAssociatedIndexPatterns() { + return Collections.singletonList(ASSOCIATED_INDEX_NAME); + } + + @Override + public String getFeatureName() { + return AssociatedIndicesTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin"; + } + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index c793c424bd4e3..d80a1f1b53452 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -120,6 +120,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; /** @@ -261,17 +262,27 @@ public ClusterState execute(ClusterState currentState) { List featureStates = Collections.emptyList(); if (request.includeGlobalState() || request.featureStates().length > 0) { - Set featureStatesSet = new HashSet<>(Arrays.asList(request.featureStates())); - boolean includeAllFeatureStates = request.includeGlobalState() && featureStatesSet.isEmpty(); + final Set featureStatesSet = new HashSet<>(); + if (request.includeGlobalState() && request.featureStates().length == 0) { + featureStatesSet.addAll(systemIndexDescriptorMap.keySet()); + } else { + featureStatesSet.addAll(Arrays.asList(request.featureStates())); + } featureStates = systemIndexDescriptorMap.keySet().stream() - .filter(feature -> includeAllFeatureStates || featureStatesSet.contains(feature)) + .filter(feature -> featureStatesSet.contains(feature)) .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) .filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates .collect(Collectors.toList()); + final Stream featureStateIndices = featureStates.stream().flatMap(feature -> feature.getIndices().stream()); + + final Stream associatedIndices = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature)) + .flatMap(feature -> resolveAssociatedIndices(currentState, feature).stream()); // Add all resolved indices from the feature states to the list of indices - indices = Stream.concat(indices.stream(), featureStates.stream().flatMap(state -> state.getIndices().stream())) + indices = Stream.of(indices.stream(), featureStateIndices, associatedIndices) + .flatMap(s -> s) .distinct() .collect(Collectors.toList()); } @@ -332,14 +343,25 @@ private List resolveFeatureIndexNames(ClusterState currentState, String } final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); - final Stream systemIndexPatterns = feature.getIndexDescriptors().stream().map(SystemIndexDescriptor::getIndexPattern); - final Stream associatedIndexPatterns = feature.getAssociatedIndexPatterns().stream(); - return Stream.concat(systemIndexPatterns, associatedIndexPatterns) + return feature.getIndexDescriptors().stream() + .map(SystemIndexDescriptor::getIndexPattern) .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, LENIENT_EXPAND_OPEN_CLOSED, pattern))) .collect(Collectors.toList()); } + private List resolveAssociatedIndices(ClusterState currentState, String featureName) { + if (systemIndexDescriptorMap.containsKey(featureName) == false) { + throw new IllegalArgumentException("requested associated indices for feature state for unknown feature [" + featureName + "]"); + } + + final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); + return feature.getAssociatedIndexPatterns().stream() + .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, + LENIENT_EXPAND_OPEN_CLOSED_HIDDEN, pattern))) + .collect(Collectors.toList()); + } + private static void ensureSnapshotNameNotRunning(List runningSnapshots, String repositoryName, String snapshotName) { if (runningSnapshots.stream().anyMatch(s -> { From e9b0f80992b75390a8ff1417d90de417d47b1f16 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 9 Dec 2020 17:03:15 -0700 Subject: [PATCH 053/107] Delete all system indices for features that are being restored --- .../snapshots/SystemIndicesSnapshotIT.java | 3 +- .../java/org/elasticsearch/node/Node.java | 2 +- .../snapshots/RestoreService.java | 39 +++++++++++++++++-- .../snapshots/SnapshotResiliencyTests.java | 4 +- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 85a6cd92225b1..1349a1e499088 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -144,8 +144,9 @@ public void testSnapshotByFeature() { // snapshot by feature CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + .setIncludeGlobalState(false) .setWaitForCompletion(true) - .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName()) + .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName(), AnotherSystemIndexTestPlugin.class.getSimpleName()) .get(); assertSnapshotSuccess(createSnapshotResponse); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 02e9d06c5726f..efb40944576ec 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -597,7 +597,7 @@ protected Node(final Environment initialEnvironment, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), metadataCreateIndexService, clusterModule.getMetadataDeleteIndexService(), metadataIndexUpgradeService, - clusterService.getClusterSettings(), shardLimitValidator); + clusterService.getClusterSettings(), shardLimitValidator, systemIndices, clusterModule.getIndexNameExpressionResolver()); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state, clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, rerouteService); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 082f29af6f52f..9b0f90e52b305 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -45,6 +45,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; @@ -75,6 +76,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.ShardLimitValidator; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -88,6 +90,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; @@ -98,6 +101,7 @@ import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; @@ -169,12 +173,17 @@ public class RestoreService implements ClusterStateApplier { private final ClusterSettings clusterSettings; + private final SystemIndices systemIndices; + + private final IndexNameExpressionResolver indexNameExpressionResolver; + private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(); public RestoreService(ClusterService clusterService, RepositoriesService repositoriesService, AllocationService allocationService, MetadataCreateIndexService createIndexService, MetadataDeleteIndexService metadataDeleteIndexService, MetadataIndexUpgradeService metadataIndexUpgradeService, - ClusterSettings clusterSettings, ShardLimitValidator shardLimitValidator) { + ClusterSettings clusterSettings, ShardLimitValidator shardLimitValidator, SystemIndices systemIndices, + IndexNameExpressionResolver indexNameExpressionResolver) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; this.allocationService = allocationService; @@ -186,6 +195,8 @@ public RestoreService(ClusterService clusterService, RepositoriesService reposit } this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; + this.systemIndices = systemIndices; + this.indexNameExpressionResolver = indexNameExpressionResolver; } /** @@ -277,13 +288,11 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, featureStateIndices.stream() ).distinct().collect(Collectors.toList()); - final Set systemIndicesToDelete = new HashSet<>(); final Set explicitlyRequestedSystemIndices = new HashSet<>(); final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); for (IndexId indexId : indexIdsInSnapshot) { IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); if (snapshotIndexMetaData.isSystem()) { - systemIndicesToDelete.add(snapshotIndexMetaData.getIndex()); if (requestedIndicesInSnapshot.contains(indexId.getName())) { explicitlyRequestedSystemIndices.add(indexId.getName()); } @@ -324,6 +333,8 @@ public ClusterState execute(ClusterState currentState) { deletionsInProgress.getEntries().get(0) + "]"); } + // Clear out all existing indices which fall within a system index pattern being restored + final Set systemIndicesToDelete = resolveIndicesToDelete(currentState, featureStatesToRestore); currentState = metadataDeleteIndexService.deleteIndices(currentState, systemIndicesToDelete); // Updating cluster state @@ -685,9 +696,31 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque } else if (request.includeGlobalState() == false) { featureStatesToRestore.clear(); } + + final List featuresNotOnThisNode = featureStatesToRestore.keySet().stream() + .filter(featureName -> systemIndices.getFeatures().containsKey(featureName)) + .collect(Collectors.toList()); + if (featuresNotOnThisNode.isEmpty() == false) { + logger.warn("while restoring feature states from snapshot [{}], some requested feature states are not present on this node: {}", + snapshot, featuresNotOnThisNode); + } return featureStatesToRestore; } + private Set resolveIndicesToDelete(ClusterState currentState, Map> featureStatesToRestore) { + final String[] indexPatternsToDelete = featureStatesToRestore.keySet().stream() + .map(featureName -> systemIndices.getFeatures().get(featureName)) + .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` + .flatMap(feature -> feature.getIndexDescriptors().stream()) + .map(descriptor -> descriptor.getIndexPattern()) + .toArray(String[]::new); + final String[] indexNamesToDelete = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, + LENIENT_EXPAND_OPEN_CLOSED, indexPatternsToDelete); + return Stream.of(indexNamesToDelete) + .map(idxName -> currentState.metadata().index(idxName).getIndex()) + .collect(Collectors.toSet()); + } + //visible for testing static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) { String dataStreamName = dataStream.getName(); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index d791d696fe32a..4d9057d65e5a5 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1580,7 +1580,9 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi systemIndices, null), clusterSettings, - shardLimitValidator + shardLimitValidator, + systemIndices, + indexNameExpressionResolver ); actions.put(PutMappingAction.INSTANCE, new TransportPutMappingAction(transportService, clusterService, threadPool, metadataMappingService, From 2e54c9470f9345df39eb6593a9013f7bd9df86a6 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 9 Dec 2020 17:09:50 -0700 Subject: [PATCH 054/107] Always restore aliases for system indices --- .../java/org/elasticsearch/snapshots/RestoreService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 9b0f90e52b305..9f61d0371d59d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -386,7 +386,8 @@ public ClusterState execute(ClusterState currentState) { .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())) .timestampMillisRange(IndexLongFieldRange.NO_SHARDS); shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState); - if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty()) { + if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty() + && isSystemIndex(snapshotIndexMetadata) == false) { // Remove all aliases - they shouldn't be restored indexMdBuilder.removeAllAliases(); } else { @@ -424,7 +425,7 @@ public ClusterState execute(ClusterState currentState) { Math.max(snapshotIndexMetadata.primaryTerm(shard), currentIndexMetadata.primaryTerm(shard))); } - if (!request.includeAliases()) { + if (!request.includeAliases() && isSystemIndex(snapshotIndexMetadata) == false) { // Remove all snapshot aliases if (!snapshotIndexMetadata.getAliases().isEmpty()) { indexMdBuilder.removeAllAliases(); @@ -656,6 +657,10 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } } + private boolean isSystemIndex(IndexMetadata indexMetadata) { + return indexMetadata.isSystem() || systemIndices.isSystemIndex(indexMetadata.getIndex()); + } + private Map getDataStreamsToRestore(Repository repository, SnapshotId snapshotId, SnapshotInfo snapshotInfo, Metadata globalMetadata, List requestIndices) { Map dataStreams; From cdc3bd1433a127b5124cff9f6ab1f65fdb73572d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Dec 2020 16:42:19 -0700 Subject: [PATCH 055/107] Don't delete all indices if there aren't any feature states to restore --- .../java/org/elasticsearch/snapshots/RestoreService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 9f61d0371d59d..0c2aaf0a1d38e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -719,6 +719,12 @@ private Set resolveIndicesToDelete(ClusterState currentState, Map feature.getIndexDescriptors().stream()) .map(descriptor -> descriptor.getIndexPattern()) .toArray(String[]::new); + + if (indexPatternsToDelete.length == 0) { + // If this is empty, it'll resolve to all indices, so explicitly return an empty set here. + return Collections.emptySet(); + } + final String[] indexNamesToDelete = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, LENIENT_EXPAND_OPEN_CLOSED, indexPatternsToDelete); return Stream.of(indexNamesToDelete) From 9520873444802dde081d85dc36d37a8dc1865d18 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Dec 2020 16:42:31 -0700 Subject: [PATCH 056/107] Null guards for FeatureStates --- .../java/org/elasticsearch/snapshots/RestoreService.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 0c2aaf0a1d38e..aeab7ca8a3f25 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -685,6 +685,9 @@ private Map getDataStreamsToRestore(Repository repository, S private Map> getFeatureStatesToRestore(RestoreSnapshotRequest request, SnapshotInfo snapshotInfo, Snapshot snapshot) { + if (snapshotInfo.featureStates() == null) { + return Collections.emptyMap(); + } final Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); @@ -713,6 +716,10 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque } private Set resolveIndicesToDelete(ClusterState currentState, Map> featureStatesToRestore) { + if (featureStatesToRestore == null) { + return Collections.emptySet(); + } + final String[] indexPatternsToDelete = featureStatesToRestore.keySet().stream() .map(featureName -> systemIndices.getFeatures().get(featureName)) .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` From a0baac2828ee5e297493c7e92ead623a45f5b6f7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Dec 2020 17:48:49 -0700 Subject: [PATCH 057/107] Make Get Snapshottable Features non-operator only --- .../org/elasticsearch/xpack/security/operator/Constants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 7a1f63529b1a1..330c95617bd87 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -80,6 +80,7 @@ public class Constants { "cluster:admin/snapshot/restore", "cluster:admin/snapshot/status", "cluster:admin/snapshot/status[nodes]", + "cluster:admin/snapshot/features/get", "cluster:admin/tasks/cancel", "cluster:admin/transform/delete", "cluster:admin/transform/preview", From 262760f1dad23aec28740dee496bc15cc8093fdd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 10 Dec 2020 18:16:02 -0700 Subject: [PATCH 058/107] Fix test now that system indices aren't always deleted --- .../org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 1349a1e499088..ba800274f489c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -335,6 +335,9 @@ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessExc .get(); assertSnapshotSuccess(createSnapshotResponse); + // Delete the index so we can restore it without requesting the feature state + assertAcked(client().admin().indices().prepareDelete(SystemIndexTestPlugin.SYSTEM_INDEX_NAME).get()); + MockLogAppender mockLogAppender = new MockLogAppender(); Loggers.addAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); mockLogAppender.start(); From 2dde730a2db2ad65bd1e93f887ce264ff905cca4 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 17 Dec 2020 18:52:18 -0500 Subject: [PATCH 059/107] Improve system index snapshot IT I did some refactoring and cleanup of the test class, and removed some unnecessary operations from the tests so it's easier to see what they're doing. I also added comments. We have four new tests here as well: 1.`testSystemIndicesCannotBeRenamed` - verify that even if a system index matches a rename pattern, it isn't renamed in a restore operation 2. `testRestoreSystemIndicesAsGlobalStateWithNullFeatureStateList` - with `restoreGlobalState` set to true, explicitly pass a `null` for the list of feature states and verify that all feature states are restored. 3. `testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates` - again restoring global state, pass an empty list for the list of feature states and verify that no feature states are restored. 4. `testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoIndicesSpecified` - test for BWC edge case behavior. If no indices are specified and an empty list of feature states is passed, verify that we try to restore the system index because it was included in the expansion of "all indices." This behavior will change in the future. --- .../snapshots/SystemIndicesSnapshotIT.java | 306 ++++++++++++------ 1 file changed, 205 insertions(+), 101 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index ba800274f489c..ad11858bd8c05 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -29,16 +29,15 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.test.MockLogAppender; +import org.junit.Before; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -48,6 +47,8 @@ public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { + public static final String REPO_NAME = "test-repo"; + @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); @@ -57,20 +58,22 @@ protected Collection> nodePlugins() { return plugins; } - public void testRestoreSystemIndicesAsGlobalState() { - createRepository("test-repo", "fs"); - - // create index and add a document - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); + @Before + public void setup() { + createRepository(REPO_NAME, "fs"); + } + /** + * Test that if a snapshot includes system indices and we restore global state, + * with no reference to feature state, the system indices are restored too. + */ + public void testRestoreSystemIndicesAsGlobalState() { + // put a document in a system index indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); // run a snapshot including global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIncludeGlobalState(true) .get(); @@ -82,8 +85,8 @@ public void testRestoreSystemIndicesAsGlobalState() { assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); - // restore indices as global state without closing the index - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + // restore snapshot with global state, without closing the system index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setRestoreGlobalState(true) .get(); @@ -93,30 +96,24 @@ public void testRestoreSystemIndicesAsGlobalState() { assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); } + /** + * If we take a snapshot with includeGlobalState set to false, are system indices included? + */ public void testSnapshotWithoutGlobalState() { - createRepository("test-repo", "fs"); - - // create index and add a document - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - assertAcked(prepareCreate("not-a-system-index", 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); - indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc"); indexDoc("not-a-system-index", "1", "purpose", "non system index doc"); // run a snapshot without global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIncludeGlobalState(false) .get(); assertSnapshotSuccess(createSnapshotResponse); - clusterAdmin().prepareGetRepositories("test-repo").get(); - Set snapshottedIndices = clusterAdmin().prepareGetSnapshots("test-repo").get() - .getSnapshots("test-repo").stream() + // check snapshot info for for which + clusterAdmin().prepareGetRepositories(REPO_NAME).get(); + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() .map(SnapshotInfo::indices) .flatMap(Collection::stream) .collect(Collectors.toSet()); @@ -126,24 +123,16 @@ public void testSnapshotWithoutGlobalState() { // assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices))); } + /** + * Test that we can snapshot feature states by name. + */ public void testSnapshotByFeature() { - createRepository("test-repo", "fs"); - - // create indices - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - assertAcked(prepareCreate(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); - - // put a document in each one - indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); // snapshot by feature - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setIncludeGlobalState(false) .setWaitForCompletion(true) .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName(), AnotherSystemIndexTestPlugin.class.getSimpleName()) @@ -159,7 +148,7 @@ public void testSnapshotByFeature() { assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); // restore indices as global state without closing the index - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setRestoreGlobalState(true) .get(); @@ -167,29 +156,22 @@ public void testSnapshotByFeature() { // verify only the original document is restored assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); } + /** + * Take a snapshot with global state but restore features by state. + */ public void testRestoreByFeature() { - createRepository("test-repo", "fs"); - - // create indices - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - assertAcked(prepareCreate(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - // And one regular index final String regularIndex = "test-idx"; - indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); - ensureGreen(); - - // put a document in each one - indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); // snapshot including global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIncludeGlobalState(true) .get(); @@ -207,52 +189,44 @@ public void testRestoreByFeature() { assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); // restore indices by feature, with only the regular index named explicitly - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIndices(regularIndex) .setFeatureStates("SystemIndexTestPlugin") .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - // verify only the original document is restored + // verify that the restored system index has only one document assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); // but the non-requested feature should still have its new document assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + /** + * Test that if a feature state has associated indices, they are included in the snapshot + * when that feature state is selected. + */ public void testSnapshotAndRestoreAssociatedIndices() { - createRepository("test-repo", "fs"); - final String regularIndex = "regular-idx"; - // create indices - assertAcked(prepareCreate(regularIndex, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - assertAcked(prepareCreate(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - assertAcked(prepareCreate(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); - - // put a document in each one - indexDoc(regularIndex, "1", "purpose", "pre-snaphost doc"); + // put documents into a regular index as well as the system index and associated index of a feature + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); - refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, - AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME); // snapshot - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setIndices(regularIndex) .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) .setWaitForCompletion(true) .get(); assertSnapshotSuccess(createSnapshotResponse); - Set snapshottedIndices = clusterAdmin().prepareGetSnapshots("test-repo").get() - .getSnapshots("test-repo").stream() + // verify the correctness of the snapshot + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() .map(SnapshotInfo::indices) .flatMap(Collection::stream) .collect(Collectors.toSet()); @@ -271,7 +245,7 @@ public void testSnapshotAndRestoreAssociatedIndices() { assertAcked(client().admin().indices().prepareDelete(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME).get()); // restore the feature state and its associated index - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME) .setWaitForCompletion(true) .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) @@ -283,21 +257,15 @@ public void testSnapshotAndRestoreAssociatedIndices() { assertThat(getDocCount(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L)); } + /** + * Check that if we request a feature not in the snapshot, we get an error. + */ public void testRestoreFeatureNotInSnapshot() { - createRepository("test-repo", "fs"); - - // create indices - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); - - // put a document in each one - indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); // snapshot including global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIncludeGlobalState(true) .get(); @@ -306,7 +274,7 @@ public void testRestoreFeatureNotInSnapshot() { final String fakeFeatureStateName = "NonExistentTestPlugin"; SnapshotRestoreException exception = expectThrows( SnapshotRestoreException.class, - () -> clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setFeatureStates("SystemIndexTestPlugin", fakeFeatureStateName) .get()); @@ -315,21 +283,17 @@ public void testRestoreFeatureNotInSnapshot() { containsString("requested feature states [[" + fakeFeatureStateName + "]] are not present in snapshot")); } + /** + * Check that directly requesting a system index in a restore request logs a deprecation warning. + * @throws IllegalAccessException if something goes wrong with the mock log appender + */ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessException { - createRepository("test-repo", "fs"); - - // create system index - assertAcked(prepareCreate(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, 2, Settings.builder() - .put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)).put("refresh_interval", 10, TimeUnit.SECONDS))); - - ensureGreen(); - // put a document in system index - indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snaphost doc"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); // snapshot including global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIncludeGlobalState(true) .get(); @@ -338,6 +302,7 @@ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessExc // Delete the index so we can restore it without requesting the feature state assertAcked(client().admin().indices().prepareDelete(SystemIndexTestPlugin.SYSTEM_INDEX_NAME).get()); + // Set up a mock log appender to watch for the log message we expect MockLogAppender mockLogAppender = new MockLogAppender(); Loggers.addAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); mockLogAppender.start(); @@ -348,12 +313,13 @@ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessExc "Restoring system indices by name is deprecated. Use feature states instead. System indices: [.test-system-idx]")); // restore system index by name, rather than feature state - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap") + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setIndices(SystemIndexTestPlugin.SYSTEM_INDEX_NAME) .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + // Check that the message was logged and remove log appender mockLogAppender.assertAllExpectationsMatched(); mockLogAppender.stop(); Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); @@ -362,6 +328,144 @@ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessExc assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); } + /** + * Check that if a system index matches a rename pattern in a restore request, it's not renamed + */ + public void testSystemIndicesCannotBeRenamed() { + final String nonSystemIndex = ".test-non-system-index"; + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(nonSystemIndex, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + assertAcked(client().admin().indices().prepareDelete(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, nonSystemIndex).get()); + + // Restore using a rename pattern that matches both the regular and the system index + clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setRenamePattern(".test-(.+)") + .setRenameReplacement(".test-restored-$1") + .get(); + + // The original system index and the renamed normal index should exist + assertTrue("System index not renamed", indexExists(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)); + assertTrue("Non-system index was renamed", indexExists(".test-restored-non-system-index")); + + // The original normal index should still be deleted, and there shouldn't be a renamed version of the system index + assertFalse("Renamed system index doesn't exist", indexExists(".test-restored-system-index")); + assertFalse("Original non-system index doesn't exist", indexExists(nonSystemIndex)); + } + + /** + * If the list of feature states to restore is null and we are restoring global state, + * all feature states should be restored. + */ + public void testRestoreSystemIndicesAsGlobalStateWithNullFeatureStateList() { + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices as global state a null list of feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setFeatureStates((String[]) null) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify that the system index is destroyed + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + /** + * If the list of feature states to restore is an empty list and we are restoring global state, + * no feature states should be restored. + * + * In this test, we explicitly request a regular index to avoid any confusion over the meaning of + * "all indices." + */ + public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() { + String regularIndex = "my-index"; + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, regularIndex); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertAcked(client().admin().indices().prepareDelete(regularIndex).get()); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore regular index, with global state and an empty list of feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setFeatureStates(new String[]{}) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify that the system index still has the updated document, i.e. has not been restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + + /** + * If the list of feature states to restore is an empty list and we are restoring global state, + * no feature states should be restored. However, for backwards compatibility, if no index is + * specified, system indices are included in "all indices." In this edge case, we get an error + * saying that the system index must be closed, because here it is included in "all indices." + */ + public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoIndicesSpecified() { + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // restore indices as global state without closing the index + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setFeatureStates(new String[]{}) + .get()); + + assertThat(exception.getMessage(), containsString("cannot restore index [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + + "] because an open index with same name already exists in the cluster.")); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), From 510ab18e81d24019c5459ad5e352739454de92af Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 15 Dec 2020 17:01:24 -0700 Subject: [PATCH 060/107] Name/description for test plugins --- .../indices/SystemIndexManagerIT.java | 10 +++++++++ .../core/async/AsyncTaskServiceTests.java | 22 ++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java index b46e0a2fc0ed7..dbd886f7f1659 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexManagerIT.java @@ -237,5 +237,15 @@ public static class TestPlugin extends Plugin implements SystemIndexPlugin { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new TestSystemIndexDescriptor()); } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java index 66fa5da060072..dea6fc00804f7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java @@ -5,6 +5,12 @@ */ package org.elasticsearch.xpack.core.async; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.delete.DeleteResponse; @@ -26,12 +32,6 @@ import org.elasticsearch.xpack.core.security.user.User; import org.junit.Before; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - // TODO: test CRUD operations public class AsyncTaskServiceTests extends ESSingleNodeTestCase { private AsyncTaskIndexService indexService; @@ -62,6 +62,16 @@ public static class TestPlugin extends Plugin implements SystemIndexPlugin { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(AsyncTaskIndexService.getSystemIndexDescriptor()); } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } } public void testEnsuredAuthenticatedUserIsSame() throws IOException { From 20c6b98b76205259c3fc7e209b47e7fdfb0101d6 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 17 Dec 2020 17:36:08 -0700 Subject: [PATCH 061/107] Add a test to ensure system indices in a feature state are removed when restoring that state --- .../snapshots/SystemIndicesSnapshotIT.java | 84 +++++++++++++++---- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index ad11858bd8c05..3b1d3be2ecf37 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -19,6 +19,20 @@ package org.elasticsearch.snapshots; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.in; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -31,20 +45,6 @@ import org.elasticsearch.test.MockLogAppender; import org.junit.Before; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.in; - public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { public static final String REPO_NAME = "test-repo"; @@ -466,6 +466,60 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoI + "] because an open index with same name already exists in the cluster.")); } + /** + * When a feature state is restored, all indices that are part of that feature state should be deleted, then the indices in + * the snapshot should be restored. + * + * However, other feature states should be unaffected. + */ + public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { + // Create a system index we'll snapshot and restore + final String systemIndexInSnapshot = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; + indexDoc(systemIndexInSnapshot, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "*"); + + // And one we'll snapshot but not restore + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Now index another doc and create another index in the same pattern as the first index + final String systemIndexNotInSnapshot = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-2"; + indexDoc(systemIndexInSnapshot, "2", "purpose", "post-snapshot doc"); + indexDoc(systemIndexNotInSnapshot, "1", "purpose", "post-snapshot doc"); + + // Add another doc to the second system index, so we can be sure it hasn't been touched + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore the snapshot + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setFeatureStates("SystemIndexTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The index we created after the snapshot should be gone + assertFalse(indexExists(systemIndexNotInSnapshot)); + // And the first index should have a single doc + assertThat(getDocCount(systemIndexInSnapshot), equalTo(1L)); + // And the system index whose state we didn't restore shouldn't have been touched and still have 2 docs + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), @@ -482,7 +536,7 @@ public static class SystemIndexTestPlugin extends Plugin implements SystemIndexP @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "System indices for tests")); } @Override From 15b4903da98c3917d0c7ced9901cc6b54aa24767 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 17 Dec 2020 17:48:56 -0700 Subject: [PATCH 062/107] Add test to ensure system index aliases are always restored --- .../snapshots/SystemIndicesSnapshotIT.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 3b1d3be2ecf37..8a533e0f3b025 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -520,6 +520,53 @@ public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + public void testSystemIndexAliasesAreAlwaysRestored() { + // Create a system index + final String systemIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; + indexDoc(systemIndexName, "1", "purpose", "pre-snapshot doc"); + + // And a regular index + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + final String regularAlias = "regular-alias"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // And make sure they both have aliases + final String systemIndexAlias = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-alias"; + assertAcked(client().admin().indices().prepareAliases() + .addAlias(systemIndexName, systemIndexAlias) + .addAlias(regularIndex, regularAlias).get()); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // And delete both the indices + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex, systemIndexName)); + + // Now restore the snapshot with no aliases + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setFeatureStates("SystemIndexTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setIncludeAliases(false) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The regular index should exist + assertTrue(indexExists(regularIndex)); + assertFalse(indexExists(regularAlias)); + // And the system index, queried by alias, should have a doc + assertTrue(indexExists(systemIndexName)); + assertTrue(indexExists(systemIndexAlias)); + assertThat(getDocCount(systemIndexAlias), equalTo(1L)); + + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), From 792d735cad4503fb25eaea7544eae1b88536a54e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 17 Dec 2020 17:49:09 -0700 Subject: [PATCH 063/107] Missed a necessary refresh --- .../org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 8a533e0f3b025..26edea3f74d53 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -499,6 +499,7 @@ public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { // Add another doc to the second system index, so we can be sure it hasn't been touched indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(systemIndexInSnapshot, systemIndexNotInSnapshot, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); // Delete the regular index so we can restore it assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); From 85b90e8e72ebd0c06e38048be3f5d229c500f2f9 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 5 Jan 2021 12:08:29 -0500 Subject: [PATCH 064/107] Fix rest test and add test coverage elsewhere --- .../org/elasticsearch/client/SnapshotIT.java | 1 + .../snapshots/SystemIndicesSnapshotIT.java | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index f8941077b2e80..18eaef2ec65fd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -273,6 +273,7 @@ public void testRestoreSnapshot() throws IOException { assertFalse("index [" + testIndex + "] should have been deleted", indexExists(testIndex)); RestoreSnapshotRequest request = new RestoreSnapshotRequest(testRepository, testSnapshot); + request.indices(testIndex); request.waitForCompletion(true); request.renamePattern(testIndex); request.renameReplacement(restoredIndex); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 26edea3f74d53..a9d173fe89f65 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -159,6 +159,43 @@ public void testSnapshotByFeature() { assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); } + /** + * Take a snapshot with global state but don't restore system indexes. By + * default, snapshot restorations ignore global state. This means that, + * for now, the system index is treated as part of the snapshot and must be + * handled explicitly. Otherwise, as in this test, there will be an + * exception. + */ + public void testDefaultRestoreOnlyRegularIndices() { + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore indices by feature, with only the regular index named explicitly + SnapshotRestoreException exception = expectThrows(SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .get()); + + assertThat(exception.getMessage(), containsString( + "cannot restore index [" + + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + + "] because an open index with same name already exists")); + } + /** * Take a snapshot with global state but restore features by state. */ From 3909cfc8cc010bdaf7c199160b1d425f7e919e4d Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 5 Jan 2021 15:02:46 -0500 Subject: [PATCH 065/107] Ignore global state in searchable snapshot test --- .../AbstractSearchableSnapshotsRestTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index c35d45e49dd65..5eacd658f680c 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -235,6 +235,7 @@ public void testSnapshotOfSearchableSnapshot() throws Exception { try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); builder.field("indices", restoredIndexName); + builder.field("include_global_state", "false"); builder.endObject(); snapshotRequest.setEntity(new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON)); } From 86ed638debfc7e9a71ee6eedeb0612223cf76aac Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 6 Jan 2021 12:07:12 -0700 Subject: [PATCH 066/107] Nest features array under an object key in Get Features API to better match API standards --- .../GetSnapshottableFeaturesResponse.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index 67f7b56a64f13..76c7a2f2d7078 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -19,6 +19,11 @@ package org.elasticsearch.action.admin.cluster.snapshots.features; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -26,11 +31,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - public class GetSnapshottableFeaturesResponse extends ActionResponse implements ToXContentObject { private final List snapshottableFeatures; @@ -55,11 +55,15 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(); - for (SnapshottableFeature feature : snapshottableFeatures) { - builder.value(feature); + builder.startObject(); + { + builder.startArray("features"); + for (SnapshottableFeature feature : snapshottableFeatures) { + builder.value(feature); + } + builder.endArray(); } - builder.endArray(); + builder.endObject(); return builder; } From 0a70ae1750da52003d919d71ecfa8e208f5d85f4 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 6 Jan 2021 12:07:45 -0700 Subject: [PATCH 067/107] Get Snapshottable Features API spec & yaml spec/smoke test --- .../rest-api-spec/api/snapshot.features.json | 29 +++++++++++++++++++ .../test/snapshot.features/10_basic.yml | 6 ++++ 2 files changed, 35 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json new file mode 100644 index 0000000000000..e5fe40e2e972a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json @@ -0,0 +1,29 @@ +{ + "snapshot.features":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", + "description":"Returns a list of features which can be snapshotted in this cluster." + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_snapshottable_features", + "methods":[ + "GET" + ] + } + ] + }, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + } + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml new file mode 100644 index 0000000000000..18cb2151432ba --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml @@ -0,0 +1,6 @@ +--- +"Get Features": + - skip: + features: contains + - do: { snapshot.features: {}} + - contains: {'features': {'name': 'tasks'}} From 9f16920c4c915ee5e131979c4abc5e2e59fc85be Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 6 Jan 2021 12:20:49 -0700 Subject: [PATCH 068/107] Update docs for feature states --- .../apis/create-snapshot-api.asciidoc | 15 +++++ .../apis/get-snapshot-api.asciidoc | 9 +++ .../get-snapshottable-features-api.asciidoc | 56 +++++++++++++++++++ .../apis/restore-snapshot-api.asciidoc | 12 ++++ .../apis/snapshot-restore-apis.asciidoc | 1 + .../restore-snapshot.asciidoc | 5 +- .../snapshot-restore/take-snapshot.asciidoc | 8 ++- 7 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index 063080999d17b..66c87cf0e4b7c 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -102,10 +102,25 @@ The cluster state includes: * Legacy index templates * Ingest pipelines * {ilm-init} lifecycle policies +* Data stored in system indices, such as Watches and task records (configurable via `feature_states`) -- + IMPORTANT: By default, the entire snapshot will fail if one or more indices included in the snapshot do not have all primary shards available. You can change this behavior by setting <> to `true`. +[[create-snapshot-api-feature-states]] +`feature_states`:: +(Optional, array of strings) +A list of feature states to be included in this snapshot. A list of features +available for inclusion in the snapshot and their descriptions be can be +retrieved using the <>. +Each feature state includes one or more system indices containing data necessary +for the function of that feature. Providing an empty array will include no feature +states in the snapshot, regardless of the value of `include_global_state`. ++ +By default, all available feature states will be included in the snapshot if +`include_global_state` is `true`, or no feature states if `include_global_state` +is `false`. + include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `metadata`:: diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 59f7fe5029bf0..252cfb713df5b 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -115,6 +115,15 @@ List of <> included in the snapshot. (Boolean) Indicates whether the current cluster state is included in the snapshot. +[[get-snapshot-api-feature-states]] +`feature_states`:: +(array) +List of feature states which were included when the snapshot was taken, +including the list of system indices included as part of the feature state. The +`feature_name` field of each can be used in the `feature_states` parameter when +restoring the snapshot to restore a subset of feature states. Only present if +the snapshot includes one or more feature states. + `start_time`:: (string) Date timestamp of when the snapshot creation process started. diff --git a/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc new file mode 100644 index 0000000000000..6515a03936586 --- /dev/null +++ b/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc @@ -0,0 +1,56 @@ +[[get-snapshottable-features-api]] +=== Get Snapshottable Features API +++++ +Get snapshottable features +++++ + +Gets a list of features which can be included in snapshots using the +<> when creating a +snapshot. + +[source,console] +----------------------------------- +GET /_snapshottable_features +----------------------------------- + +[[get-snapshottable-features-api-request]] +==== {api-request-title} + +`GET /_snapshottable_features` + + +[[get-snapshottable-features-api-desc]] +==== {api-description-title} + +You can use the get snapshottable features API to determine which feature states +to include when <>. By default, all +feature states are included in a snapshot if that snapshot includes the global +state, or none if it does not. + +A feature state includes one or more system indices necessary for a given +feature to function. In order to ensure data integrity, all system indices that +comprise a feature state are snapshotted and restored together. + +The features listed by this API are a combination of built-in features and +features defined by plugins. In order for a feature's state to be listed in this +API and recognized as a valid feature state by the create snapshot API, the +plugin which defines that feature must be installed on the master node. + +==== {api-examples-title} + +[source,console-result] +---- +{ + "features": [ + { + "name": "tasks", + "description": "Manages task results" + }, + { + "name": "kibana", + "description": "Manages Kibana configuration and reports" + } + ] +} +---- +// TESTRESPONSE[skip:response differs between default distro and OSS] diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index 51ce892c9d9b1..b911467aafdbf 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -134,10 +134,22 @@ The cluster state includes: * Legacy index templates * Ingest pipelines * {ilm-init} lifecycle policies +* For snapshots taken after 7.12.0, data stored in system indices, such as Watches and task records, replacing any existing configuration (configurable via `feature_states`) -- + IMPORTANT: By default, the entire restore operation will fail if one or more indices included in the snapshot do not have all primary shards available. You can change this behavior by setting <> to `true`. +[[restore-snapshot-api-feature-states]] +`feature_states`:: +(Optional, array of strings) +A comma-separated list of feature states you wish to restore. Each feature state contains one or more system indices. The list of feature states +available in a given snapshot are returned by the <>. Note that feature +states restored this way will completely replace any existing configuration, rather than returning an error if the system index already exists. +Providing an empty array will restore no feature states, regardless of the value of `include_global_state`. ++ +By default, all available feature states will be restored if `include_global_state` is `true`, and no feature states will be restored if +`include_global_state` is `false`. + [[restore-snapshot-api-index-settings]] `index_settings`:: (Optional, string) diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index cf70d3bcb2eab..2691f56fc786d 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -36,3 +36,4 @@ include::get-snapshot-api.asciidoc[] include::get-snapshot-status-api.asciidoc[] include::restore-snapshot-api.asciidoc[] include::delete-snapshot-api.asciidoc[] +include::get-snapshottable-features-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 907856f53050a..b3187008fc657 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -32,6 +32,9 @@ By default, all data streams and indices in the snapshot are restored, but the c supports <>. To include the global cluster state, set `include_global_state` to `true` in the restore request body. +Because all indices in the snapshot are restored by default, all system indices will be restored +by default as well, + [WARNING] ==== Each data stream requires a matching @@ -88,7 +91,7 @@ POST /_snapshot/my_backup/snapshot_1/_restore // TEST[continued] <1> By default, `include_global_state` is `false`, meaning the snapshot's -cluster state is not restored. +cluster state and any feature states are not restored. + If `true`, the snapshot's persistent settings, index templates, ingest pipelines, and {ilm-init} policies are restored into the current cluster. This diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index ddc2812dbe280..0245646db5391 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -77,8 +77,10 @@ The snapshot process starts immediately for the primary shards that have been st relocation or initialization of shards to complete before snapshotting them. Besides creating a copy of each data stream and index, the snapshot process can also store global cluster metadata, which includes persistent -cluster settings and templates. The transient settings and registered snapshot repositories are not stored as part of -the snapshot. +cluster settings and templates as well as data stored in system indices, such as Watches and task records, regardless of whether those system +indices are named in the `indices` section of the request. The <> can be used to +select a subset of system indices to be included in the snapshot. The transient settings and registered snapshot repositories are not stored +as part of the snapshot. While a snapshot of a particular shard is being created, this shard cannot be moved to another node, which can interfere with rebalancing and allocation @@ -125,4 +127,4 @@ PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E ----------------------------------- // TEST[continued] -NOTE: You can also create snapshots that are copies of part of an existing snapshot using the <>. \ No newline at end of file +NOTE: You can also create snapshots that are copies of part of an existing snapshot using the <>. From afdf54c6d21babe818d8f2375784f5e83a6bd7d1 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 6 Jan 2021 15:44:59 -0700 Subject: [PATCH 069/107] Add version-based skip to Get Features YAML test --- .../resources/rest-api-spec/test/snapshot.features/10_basic.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml index 18cb2151432ba..46c6bda1d70ad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml @@ -2,5 +2,7 @@ "Get Features": - skip: features: contains + version: " - 7.99.99" # Adjust this after backport + reason: "This API was added in 7.12.0" - do: { snapshot.features: {}} - contains: {'features': {'name': 'tasks'}} From 8c324fd6ff4fa684d84cb036061d771f7579b8ac Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 7 Jan 2021 14:54:07 -0700 Subject: [PATCH 070/107] Add HLRC support for Get Features API --- .../elasticsearch/client/SnapshotClient.java | 49 +++++++- .../client/SnapshotRequestConverters.java | 14 ++- .../GetSnapshottableFeaturesRequest.java | 28 +++++ .../GetSnapshottableFeaturesResponse.java | 119 ++++++++++++++++++ .../org/elasticsearch/client/SnapshotIT.java | 35 ++++-- ...GetSnapshottableFeaturesResponseTests.java | 67 ++++++++++ ...atures.json => snapshot.get_features.json} | 2 +- .../test/snapshot.features/10_basic.yml | 2 +- .../GetSnapshottableFeaturesResponse.java | 2 +- .../RestSnapshottableFeaturesAction.java | 14 ++- 10 files changed, 308 insertions(+), 24 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java rename rest-api-spec/src/main/resources/rest-api-spec/api/{snapshot.features.json => snapshot.get_features.json} (95%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index 51f3940774860..f4dc08eca2e4d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -19,6 +19,10 @@ package org.elasticsearch.client; +import static java.util.Collections.emptySet; + +import java.io.IOException; + import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -39,10 +43,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; - -import java.io.IOException; - -import static java.util.Collections.emptySet; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; /** * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Snapshot API. @@ -389,4 +391,43 @@ public Cancellable deleteAsync(DeleteSnapshotRequest deleteSnapshotRequest, Requ SnapshotRequestConverters::deleteSnapshot, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } + + /** + * Get a list of features which can be included in a snapshot as feature states. + * See [link here] + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetSnapshottableFeaturesResponse getFeatures(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + getFeaturesRequest, + SnapshotRequestConverters::getSnapshottableFeatures, + options, + GetSnapshottableFeaturesResponse::parse, + emptySet() + ); + } + + /** + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable getFeaturesAsync(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity( + getFeaturesRequest, + SnapshotRequestConverters::getSnapshottableFeatures, + options, + GetSnapshottableFeaturesResponse::parse, + listener, + emptySet() + ); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 57fc15a212bce..83af683423f7f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -19,6 +19,8 @@ package org.elasticsearch.client; +import java.io.IOException; + import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -34,10 +36,9 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; import org.elasticsearch.common.Strings; -import java.io.IOException; - final class SnapshotRequestConverters { private SnapshotRequestConverters() {} @@ -201,4 +202,13 @@ static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) { request.addParameters(parameters.asMap()); return request; } + + static Request getSnapshottableFeatures(GetSnapshottableFeaturesRequest getSnapshottableFeaturesRequest) { + String endpoint = "/_snapshottable_features"; + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(); + parameters.withMasterTimeout(getSnapshottableFeaturesRequest.masterNodeTimeout()); + request.addParameters(parameters.asMap()); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java new file mode 100644 index 0000000000000..b50abf31b6611 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.client.TimedRequest; + +/** + * A {@link TimedRequest} to get the list of features available to be included in snapshots in the cluster. + */ +public class GetSnapshottableFeaturesRequest extends TimedRequest { +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java new file mode 100644 index 0000000000000..ed33c3d1492c6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.snapshots; + +import java.util.List; +import java.util.Objects; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +public class GetSnapshottableFeaturesResponse { + + private final List features; + + private static final ParseField FEATURES = new ParseField("features"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "snapshottable_features_response", true, (a, ctx) -> new GetSnapshottableFeaturesResponse((List) a[0]) + ); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), SnapshottableFeature::parse, FEATURES); + } + + public GetSnapshottableFeaturesResponse(List features) { + this.features = features; + } + + public List getFeatures() { + return features; + } + + public static GetSnapshottableFeaturesResponse parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GetSnapshottableFeaturesResponse)) return false; + GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; + return getFeatures().equals(that.getFeatures()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatures()); + } + + public static class SnapshottableFeature { + + private final String featureName; + private final String description; + + private static final ParseField FEATURE_NAME = new ParseField("name"); + private static final ParseField DESCRIPTION = new ParseField("description"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "feature", true, (a, ctx) -> new SnapshottableFeature((String) a[0], (String) a[1]) + ); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), FEATURE_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), DESCRIPTION, ObjectParser.ValueType.STRING); + } + + public SnapshottableFeature(String featureName, String description) { + this.featureName = featureName; + this.description = description; + } + + public static SnapshottableFeature parse(XContentParser parser, Void ctx) { + return PARSER.apply(parser, ctx); + } + + public String getFeatureName() { + return featureName; + } + + public String getDescription() { + return description; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SnapshottableFeature)) return false; + SnapshottableFeature feature = (SnapshottableFeature) o; + return Objects.equals(getFeatureName(), feature.getFeatureName()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatureName()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 18eaef2ec65fd..c6b76e1ee1571 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -19,6 +19,17 @@ package org.elasticsearch.client; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -39,6 +50,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -47,16 +60,6 @@ import org.elasticsearch.snapshots.RestoreInfo; import org.mockito.internal.util.collections.Sets; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; - public class SnapshotIT extends ESRestHighLevelClientTestCase { private AcknowledgedResponse createTestRepository(String repository, String type, String settings) throws IOException { @@ -376,6 +379,18 @@ public void testCloneSnapshot() throws IOException { assertTrue(response.isAcknowledged()); } + public void testGetFeatures() throws IOException { + GetSnapshottableFeaturesRequest request = new GetSnapshottableFeaturesRequest(); + + GetSnapshottableFeaturesResponse response = execute(request, + highLevelClient().snapshot()::getFeatures, highLevelClient().snapshot()::getFeaturesAsync); + + assertThat(response, notNullValue()); + assertThat(response.getFeatures(), notNullValue()); + assertThat(response.getFeatures().size(), greaterThan(1)); + assertTrue(response.getFeatures().stream().anyMatch(feature -> "tasks".equals(feature.getFeatureName()))); + } + private static Map randomUserMetadata() { if (randomBoolean()) { return null; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java new file mode 100644 index 0000000000000..dacffe33b67f5 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.snapshots; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.is; + +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +public class GetSnapshottableFeaturesResponseTests + extends AbstractResponseTestCase { + + @Override + protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance(XContentType xContentType) { + return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse(randomList(10, + () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5, 10)))); + + } + + @Override + protected GetSnapshottableFeaturesResponse doParseToClientInstance(XContentParser parser) throws IOException { + return GetSnapshottableFeaturesResponse.parse(parser); + } + + @Override + protected void assertInstances(org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, GetSnapshottableFeaturesResponse clientInstance) { + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + + assertThat(clientInstance.getFeatures(), hasSize(serverTestInstance.getSnapshottableFeatures().size())); + + Map clientFeatures = clientInstance.getFeatures().stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + Map serverFeatures = serverTestInstance.getSnapshottableFeatures().stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + + assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json similarity index 95% rename from rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json rename to rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json index e5fe40e2e972a..76b340d329dd8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.features.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json @@ -1,5 +1,5 @@ { - "snapshot.features":{ + "snapshot.get_features":{ "documentation":{ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", "description":"Returns a list of features which can be snapshotted in this cluster." diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml index 46c6bda1d70ad..6d0567a72e312 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml @@ -4,5 +4,5 @@ features: contains version: " - 7.99.99" # Adjust this after backport reason: "This API was added in 7.12.0" - - do: { snapshot.features: {}} + - do: { snapshot.get_features: {}} - contains: {'features': {'name': 'tasks'}} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index 76c7a2f2d7078..af21d0b88b062 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -81,8 +81,8 @@ public int hashCode() { } public static class SnapshottableFeature implements Writeable, ToXContentObject { - private final String featureName; + private final String featureName; private final String description; public SnapshottableFeature(String featureName, String description) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java index 8390f29ee94a4..76190221ade30 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java @@ -19,6 +19,9 @@ package org.elasticsearch.rest.action.admin.cluster; +import java.io.IOException; +import java.util.List; + import org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesRequest; import org.elasticsearch.action.admin.cluster.snapshots.features.SnapshottableFeaturesAction; import org.elasticsearch.client.node.NodeClient; @@ -26,9 +29,6 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import java.io.IOException; -import java.util.List; - public class RestSnapshottableFeaturesAction extends BaseRestHandler { @Override public List routes() { @@ -42,7 +42,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - return restChannel -> client.execute(SnapshottableFeaturesAction.INSTANCE, new GetSnapshottableFeaturesRequest(), - new RestToXContentListener<>(restChannel)); + final GetSnapshottableFeaturesRequest req = new GetSnapshottableFeaturesRequest(); + req.masterNodeTimeout(request.paramAsTime("master_timeout", req.masterNodeTimeout())); + + return restChannel -> { + client.execute(SnapshottableFeaturesAction.INSTANCE, req, new RestToXContentListener<>(restChannel)); + }; } } From 1b999412b5c7efaaa3222bed30b7a2c7c6bf90ab Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 7 Jan 2021 14:59:13 -0700 Subject: [PATCH 071/107] Checkstyle --- ...GetSnapshottableFeaturesResponseTests.java | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java index dacffe33b67f5..6b9612d975c29 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -32,16 +32,23 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -public class GetSnapshottableFeaturesResponseTests - extends AbstractResponseTestCase { @Override - protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance(XContentType xContentType) { - return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse(randomList(10, - () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( - randomAlphaOfLengthBetween(4, 10), - randomAlphaOfLengthBetween(5, 10)))); + protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance( + XContentType xContentType + ) { + return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse( + randomList( + 10, + () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5, 10) + ) + ) + ); } @@ -51,15 +58,20 @@ protected GetSnapshottableFeaturesResponse doParseToClientInstance(XContentParse } @Override - protected void assertInstances(org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, GetSnapshottableFeaturesResponse clientInstance) { + protected void assertInstances( + org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, + GetSnapshottableFeaturesResponse clientInstance + ) { assertNotNull(serverTestInstance.getSnapshottableFeatures()); assertNotNull(serverTestInstance.getSnapshottableFeatures()); assertThat(clientInstance.getFeatures(), hasSize(serverTestInstance.getSnapshottableFeatures().size())); - Map clientFeatures = clientInstance.getFeatures().stream() + Map clientFeatures = clientInstance.getFeatures() + .stream() .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); - Map serverFeatures = serverTestInstance.getSnapshottableFeatures().stream() + Map serverFeatures = serverTestInstance.getSnapshottableFeatures() + .stream() .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); From 290b203351fed4daa088117a447623d7f4d02e2f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:21:43 -0700 Subject: [PATCH 072/107] Cluster state -> global state in docs where it makes sense --- .../snapshot-restore/apis/create-snapshot-api.asciidoc | 4 ++-- .../snapshot-restore/apis/restore-snapshot-api.asciidoc | 6 +++--- docs/reference/snapshot-restore/take-snapshot.asciidoc | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index 66c87cf0e4b7c..e371b74310462 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -92,10 +92,10 @@ argument is provided, the snapshot only includes the specified data streams and + -- (Optional, Boolean) -If `true`, the current cluster state is included in the snapshot. +If `true`, the current global state is included in the snapshot. Defaults to `true`. -The cluster state includes: +The global state includes: * Persistent cluster settings * Index templates diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index b911467aafdbf..23f4736e994e1 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -123,11 +123,11 @@ indices. + -- (Optional, Boolean) -If `false`, the cluster state is not restored. Defaults to `false`. +If `false`, the global state is not restored. Defaults to `false`. -If `true`, the current cluster state is included in the restore operation. +If `true`, the current global state is included in the restore operation. -The cluster state includes: +The global state includes: * Persistent cluster settings * Index templates diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index 0245646db5391..d5da7eec16d72 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -103,7 +103,7 @@ the snapshot. IMPORTANT: The global cluster state includes the cluster's index templates, such as those <>. If your snapshot includes data streams, we recommend storing the -cluster state as part of the snapshot. This lets you later restored any +global state as part of the snapshot. This lets you later restored any templates required for a data stream. By default, the entire snapshot will fail if one or more indices participating in the snapshot do not have From f7a0dc2158c51225964319a80f2b65e72973e049 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:25:20 -0700 Subject: [PATCH 073/107] Fix Client javadoc --- .../main/java/org/elasticsearch/client/SnapshotClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index f4dc08eca2e4d..87d0287bcf8da 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -394,7 +394,8 @@ public Cancellable deleteAsync(DeleteSnapshotRequest deleteSnapshotRequest, Requ /** * Get a list of features which can be included in a snapshot as feature states. - * See [link here] + * See Get Snapshottable + * Features API on elastic.co * * @param getFeaturesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized @@ -413,6 +414,9 @@ public GetSnapshottableFeaturesResponse getFeatures(GetSnapshottableFeaturesRequ } /** + * Asynchronously get a list of features which can be included in a snapshot as feature states. + * See Get Snapshottable + * Features API on elastic.co * * @param getFeaturesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized From 774f11c3e0f398f5e8d85b3e9fdd7539404cc7ad Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:27:29 -0700 Subject: [PATCH 074/107] Typo/wording fixes per review --- docs/reference/snapshot-restore/restore-snapshot.asciidoc | 4 ++-- docs/reference/snapshot-restore/take-snapshot.asciidoc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index b3187008fc657..f889fe5053f8b 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -33,7 +33,7 @@ supports <>. To include the global cluster stat `include_global_state` to `true` in the restore request body. Because all indices in the snapshot are restored by default, all system indices will be restored -by default as well, +by default as well. [WARNING] ==== @@ -91,7 +91,7 @@ POST /_snapshot/my_backup/snapshot_1/_restore // TEST[continued] <1> By default, `include_global_state` is `false`, meaning the snapshot's -cluster state and any feature states are not restored. +cluster state and feature states are not restored. + If `true`, the snapshot's persistent settings, index templates, ingest pipelines, and {ilm-init} policies are restored into the current cluster. This diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index d5da7eec16d72..5723ffde7ec9f 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -77,7 +77,7 @@ The snapshot process starts immediately for the primary shards that have been st relocation or initialization of shards to complete before snapshotting them. Besides creating a copy of each data stream and index, the snapshot process can also store global cluster metadata, which includes persistent -cluster settings and templates as well as data stored in system indices, such as Watches and task records, regardless of whether those system +cluster settings, templates, and data stored in system indices, such as Watches and task records, regardless of whether those system indices are named in the `indices` section of the request. The <> can be used to select a subset of system indices to be included in the snapshot. The transient settings and registered snapshot repositories are not stored as part of the snapshot. From 82cfdaca4d294da32478428b6795e6ff083b7a19 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:29:03 -0700 Subject: [PATCH 075/107] Remove excess line --- .../elasticsearch/indices/IndicesService.java | 75 +++++++++---------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 3ed1721b26e6b..227b66d18df66 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -19,6 +19,43 @@ package org.elasticsearch.indices; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; +import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX; +import static org.elasticsearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; +import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.function.Predicate; +import java.util.stream.Collectors; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -139,43 +176,6 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.threadpool.ThreadPool; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.LongSupplier; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; -import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; -import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX; -import static org.elasticsearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; -import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; -import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; - public class IndicesService extends AbstractLifecycleComponent implements IndicesClusterStateService.AllocatedIndices, IndexService.ShardStoreDeleter { private static final Logger logger = LogManager.getLogger(IndicesService.class); @@ -895,7 +895,6 @@ public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, } } - /** * Deletes the index store trying to acquire all shards locks for this index. * This method will delete the metadata for the index even if the actual shards can't be locked. From 46f5eed68118b5cf570de08742f7a6427dd8939c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:33:57 -0700 Subject: [PATCH 076/107] new HashMap<>() -> Collections.emptyMap() --- .../snapshots/RestoreService.java | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 8eb8ff2c0c802..0cc17e04f3a6e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -18,10 +18,37 @@ */ package org.elasticsearch.snapshots; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import static java.util.Collections.unmodifiableSet; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; +import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -82,36 +109,10 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.unmodifiableSet; -import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; -import static org.elasticsearch.common.util.set.Sets.newHashSet; -import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; +import com.carrotsearch.hppc.IntHashSet; +import com.carrotsearch.hppc.IntSet; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; /** * Service responsible for restoring snapshots @@ -668,7 +669,7 @@ private Map getDataStreamsToRestore(Repository repository, S List requestedDataStreams = filterIndices(snapshotInfo.dataStreams(), requestIndices.toArray(String[]::new), IndicesOptions.fromOptions(true, true, true, true)); if (requestedDataStreams.isEmpty()) { - dataStreams = new HashMap<>(); + dataStreams = Collections.emptyMap(); } else { if (globalMetadata == null) { globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); From 8be4e746c75e891c1b8bc10480237c82acd68c52 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 14:42:59 -0700 Subject: [PATCH 077/107] Throw an error when attempting to restore feature states that go with features which are not installed --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 0cc17e04f3a6e..459ed68d2be4e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -711,8 +711,8 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque .filter(featureName -> systemIndices.getFeatures().containsKey(featureName)) .collect(Collectors.toList()); if (featuresNotOnThisNode.isEmpty() == false) { - logger.warn("while restoring feature states from snapshot [{}], some requested feature states are not present on this node: {}", - snapshot, featuresNotOnThisNode); + throw new SnapshotRestoreException(snapshot, "requested feature states [" + featuresNotOnThisNode + "] are present in" + + "snapshot but those features are not installed on the current master node"); } return featureStatesToRestore; } From 59ab24067e267fdcadc21ae5b8f2d2fd26d71933 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 15:21:59 -0700 Subject: [PATCH 078/107] Clean up error message --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 459ed68d2be4e..e2d531a8b706d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -711,7 +711,7 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque .filter(featureName -> systemIndices.getFeatures().containsKey(featureName)) .collect(Collectors.toList()); if (featuresNotOnThisNode.isEmpty() == false) { - throw new SnapshotRestoreException(snapshot, "requested feature states [" + featuresNotOnThisNode + "] are present in" + + throw new SnapshotRestoreException(snapshot, "requested feature states " + featuresNotOnThisNode + " are present in " + "snapshot but those features are not installed on the current master node"); } return featureStatesToRestore; From b9383238c2fd70210e3480806b2dd024abcaa58c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 15 Jan 2021 16:13:48 -0700 Subject: [PATCH 079/107] Flip incorrect boolean logic (d'oh) --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index e2d531a8b706d..835b7dacce28c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -708,7 +708,7 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque } final List featuresNotOnThisNode = featureStatesToRestore.keySet().stream() - .filter(featureName -> systemIndices.getFeatures().containsKey(featureName)) + .filter(featureName -> systemIndices.getFeatures().containsKey(featureName) == false) .collect(Collectors.toList()); if (featuresNotOnThisNode.isEmpty() == false) { throw new SnapshotRestoreException(snapshot, "requested feature states " + featuresNotOnThisNode + " are present in " + From 7ceecaea0e031fce27645fe5de4375fd3a30cb1c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 19 Jan 2021 11:40:18 -0700 Subject: [PATCH 080/107] Add category for deprecation warning --- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 835b7dacce28c..bcdadde78c26d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -92,6 +92,7 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.regex.Regex; @@ -304,7 +305,7 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, // log a deprecation warning if the any of the indexes to delete were included in the request and the snapshot // is from a version that should have feature states if (snapshotInfo.version().onOrAfter(FEATURE_STATES_VERSION) && explicitlyRequestedSystemIndices.isEmpty() == false) { - deprecationLogger.deprecate("restore-system-index-from-snapshot", + deprecationLogger.deprecate(DeprecationCategory.API, "restore-system-index-from-snapshot", "Restoring system indices by name is deprecated. Use feature states instead. System indices: " + explicitlyRequestedSystemIndices); } From 6c4669f714c93fbc390ea8aef4f373c9d0a77c8e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 19 Jan 2021 14:59:05 -0700 Subject: [PATCH 081/107] Handle null & empty feature states the same in Create/Restore requests --- .../create/CreateSnapshotRequest.java | 26 +++++------ .../restore/RestoreSnapshotRequest.java | 45 ++++++++++--------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 0ceebdef32c24..d11f56997f27e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -19,6 +19,18 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.Strings.EMPTY_ARRAY; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -34,18 +46,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.common.Strings.EMPTY_ARRAY; -import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; -import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; - /** * Create snapshot request *

@@ -431,7 +431,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(index); } builder.endArray(); - if (featureStates != null && featureStates.length != 0) { + if (featureStates != null) { builder.startArray("feature_states"); for (String plugin : featureStates) { builder.value(plugin); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index cde7a8835ec5e..8ab87164cf6ac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -19,6 +19,20 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -31,20 +45,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; -import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; -import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; - /** * Restore snapshot request */ @@ -54,7 +54,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest featureStates) { return featureStates(featureStates.toArray(new String[featureStates.size()])); @@ -561,7 +564,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } - if (featureStates != null && featureStates.length != 0) { + if (featureStates != null) { builder.startArray("feature_states"); for (String plugin : featureStates) { builder.value(plugin); From 596c9ac3bc5d3d96649455e1f8ddb85c4b7e4019 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 19 Jan 2021 17:06:49 -0700 Subject: [PATCH 082/107] Add "none" special case for including no feature states Also removes `[]` as a way to specify no feature states --- .../snapshots/SystemIndicesSnapshotIT.java | 128 +++++++++++++++++- .../elasticsearch/indices/SystemIndices.java | 30 ++-- .../java/org/elasticsearch/node/Node.java | 58 ++++---- .../snapshots/RestoreService.java | 27 +++- .../snapshots/SnapshotsService.java | 88 +++++++----- 5 files changed, 243 insertions(+), 88 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index a9d173fe89f65..86edfe35dc891 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -20,11 +20,13 @@ package org.elasticsearch.snapshots; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.not; import java.util.ArrayList; import java.util.Collection; @@ -401,10 +403,10 @@ public void testSystemIndicesCannotBeRenamed() { } /** - * If the list of feature states to restore is null and we are restoring global state, + * If the list of feature states to restore is left unspecified and we are restoring global state, * all feature states should be restored. */ - public void testRestoreSystemIndicesAsGlobalStateWithNullFeatureStateList() { + public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -425,7 +427,6 @@ public void testRestoreSystemIndicesAsGlobalStateWithNullFeatureStateList() { RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setRestoreGlobalState(true) - .setFeatureStates((String[]) null) .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); @@ -434,7 +435,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithNullFeatureStateList() { } /** - * If the list of feature states to restore is an empty list and we are restoring global state, + * If the list of feature states to restore contains only "none" and we are restoring global state, * no feature states should be restored. * * In this test, we explicitly request a regular index to avoid any confusion over the meaning of @@ -465,7 +466,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() .setIndices(regularIndex) .setWaitForCompletion(true) .setRestoreGlobalState(true) - .setFeatureStates(new String[]{}) + .setFeatureStates(new String[]{ randomFrom("none", "NONE") }) .get(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); @@ -474,7 +475,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() } /** - * If the list of feature states to restore is an empty list and we are restoring global state, + * If the list of feature states to restore contains only "none" and we are restoring global state, * no feature states should be restored. However, for backwards compatibility, if no index is * specified, system indices are included in "all indices." In this edge case, we get an error * saying that the system index must be closed, because here it is included in "all indices." @@ -496,7 +497,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoI () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") .setWaitForCompletion(true) .setRestoreGlobalState(true) - .setFeatureStates(new String[]{}) + .setFeatureStates(new String[]{ randomFrom("none", "NONE") }) .get()); assertThat(exception.getMessage(), containsString("cannot restore index [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME @@ -605,6 +606,119 @@ public void testSystemIndexAliasesAreAlwaysRestored() { } + /** + * Tests that the special "none" feature state name cannot be combined with other + * feature state names, and an error occurs if it's tried. + */ + public void testNoneFeatureStateMustBeAlone() { + // put a document in a system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + IllegalArgumentException createEx = expectThrows( + IllegalArgumentException.class, + () -> clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(randomBoolean()) + .setFeatureStates("SystemIndexTestPlugin", "none", "AnotherSystemIndexTestPlugin") + .get() + ); + assertThat(createEx.getMessage(), equalTo("the feature_states value [none] indicates that no feature states should be " + + "snapshotted, but other feature states were requested: [SystemIndexTestPlugin, none, AnotherSystemIndexTestPlugin]")); + + // create a successful snapshot with global state/all features + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + SnapshotRestoreException restoreEx = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(randomBoolean()) + .setFeatureStates("SystemIndexTestPlugin", "none") + .get() + ); + assertThat( + restoreEx.getMessage(), + allOf( // the order of the requested feature states is non-deterministic so just check that it includes most of the right stuff + containsString( + "the feature_states value [none] indicates that no feature states should be restored, but other feature states were " + + "requested:" + ), + containsString("SystemIndexTestPlugin") + ) + ); + } + + /** + * Tests that using the special "none" feature state value creates a snapshot with no feature states included + */ + public void testNoneFeatureStateOnCreation() { + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .setFeatureStates(randomFrom("none", "NONE")) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Verify that the system index was not included + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)))); + } + + public void testNoneFeatureStateOnRestore() { + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // Create a snapshot + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Index another doc into the system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + // And delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // Restore the snapshot specifying the regular index and "none" for feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setRestoreGlobalState(randomBoolean()) + .setFeatureStates(randomFrom("none", "NONE")) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The regular index should only have one doc + assertThat(getDocCount(regularIndex), equalTo(1L)); + // But the system index shouldn't have been touched + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 2c3c68b8a8e75..9a46ffef07d43 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -19,14 +19,9 @@ package org.elasticsearch.indices; -import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.MinimizationOperations; -import org.apache.lucene.util.automaton.Operations; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.index.Index; +import static java.util.stream.Collectors.toUnmodifiableList; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; +import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; import java.util.Collection; import java.util.Collections; @@ -37,9 +32,15 @@ import java.util.Optional; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toUnmodifiableList; -import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; -import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.Index; +import org.elasticsearch.snapshots.SnapshotsService; /** * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the @@ -201,6 +202,13 @@ Collection getSystemIndexDescriptors() { .collect(Collectors.toList()); } + public static void validateFeatureName(String name, String plugin) { + if (SnapshotsService.NO_FEATURE_STATES_VALUE.equalsIgnoreCase(name)) { + throw new IllegalArgumentException("feature name cannot be reserved name [\"" + SnapshotsService.NO_FEATURE_STATES_VALUE + + "\"], but was for plugin [" + plugin + "]"); + } + } + public static class Feature { private final String description; private final Collection indexDescriptors; diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 23baa25ec8f84..80c285ea5ec3d 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -19,6 +19,35 @@ package org.elasticsearch.node; +import static java.util.stream.Collectors.toList; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.net.ssl.SNIHostName; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; @@ -177,34 +206,6 @@ import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; -import javax.net.ssl.SNIHostName; -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; - /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used * in order to use a {@link Client} to perform actions/operations against the cluster. @@ -509,6 +510,7 @@ protected Node(final Environment initialEnvironment, final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) .stream() + .peek(plugin -> SystemIndices.validateFeatureName(plugin.getFeatureName(), plugin.getClass().getCanonicalName())) .collect(Collectors.toUnmodifiableMap( plugin -> plugin.getFeatureName(), plugin -> new SystemIndices.Feature( diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index bcdadde78c26d..44d9d45fe8b78 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -31,6 +31,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; import java.util.ArrayList; import java.util.Arrays; @@ -694,18 +695,34 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque final Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); - final Map> featureStatesToRestore = new HashMap<>(snapshotFeatureStates); - if (request.featureStates() != null) { - final Set requestedStates = Set.of(request.featureStates()); + final Map> featureStatesToRestore; + final String[] requestedFeatureStates = request.featureStates(); + + if (requestedFeatureStates == null || requestedFeatureStates.length == 0) { + // Handle the default cases - defer to the global state value + if (request.includeGlobalState()) { + featureStatesToRestore = new HashMap<>(snapshotFeatureStates); + } else { + featureStatesToRestore = Collections.emptyMap(); + } + } else if (requestedFeatureStates.length == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedFeatureStates[0])) { + // If there's exactly one value and it's "none", include no states + featureStatesToRestore = Collections.emptyMap(); + } else { + // Otherwise, handle the list of requested feature states + final Set requestedStates = Set.of(requestedFeatureStates); + if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { + throw new SnapshotRestoreException(snapshot, "the feature_states value [" + NO_FEATURE_STATES_VALUE + + "] indicates that no feature states should be restored, but other feature states were requested: " + requestedStates); + } if (snapshotFeatureStates.keySet().containsAll(requestedStates) == false) { Set nonExistingRequestedStates = new HashSet<>(requestedStates); nonExistingRequestedStates.removeAll(snapshotFeatureStates.keySet()); throw new SnapshotRestoreException(snapshot, "requested feature states [" + nonExistingRequestedStates + "] are not present in snapshot"); } + featureStatesToRestore = new HashMap<>(snapshotFeatureStates); featureStatesToRestore.keySet().retainAll(requestedStates); - } else if (request.includeGlobalState() == false) { - featureStatesToRestore.clear(); } final List featuresNotOnThisNode = featureStatesToRestore.keySet().stream() diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index fb325498eae90..67c0ad22b8371 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -19,8 +19,37 @@ package org.elasticsearch.snapshots; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; +import static org.elasticsearch.cluster.SnapshotsInProgress.completed; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -92,36 +121,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Deque; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executor; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableList; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; -import static org.elasticsearch.cluster.SnapshotsInProgress.completed; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; /** * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and @@ -142,6 +143,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status"; + public static final String NO_FEATURE_STATES_VALUE = "none"; + private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; @@ -261,12 +264,23 @@ public ClusterState execute(ClusterState currentState) { List indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request)); List featureStates = Collections.emptyList(); - if (request.includeGlobalState() || request.featureStates().length > 0) { - final Set featureStatesSet = new HashSet<>(); - if (request.includeGlobalState() && request.featureStates().length == 0) { - featureStatesSet.addAll(systemIndexDescriptorMap.keySet()); + final List requestedStates = Arrays.asList(request.featureStates()); + if (request.includeGlobalState() || requestedStates.isEmpty() == false) { + final Set featureStatesSet; + if (request.includeGlobalState() && requestedStates.isEmpty()) { + // If we're including global state and feature states aren't specified, include all of them + featureStatesSet = new HashSet<>(systemIndexDescriptorMap.keySet()); + } else if (requestedStates.size() == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedStates.get(0))) { + // If there's exactly one value and it's "none", include no states + featureStatesSet = Collections.emptySet(); } else { - featureStatesSet.addAll(Arrays.asList(request.featureStates())); + // Otherwise, check for "none" then use the list of requested states + if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { + throw new IllegalArgumentException("the feature_states value [" + SnapshotsService.NO_FEATURE_STATES_VALUE + + "] indicates that no feature states should be snapshotted, but other feature states were requested: " + + requestedStates); + } + featureStatesSet = new HashSet<>(requestedStates); } featureStates = systemIndexDescriptorMap.keySet().stream() From d677aeb0411403469834fb7317cd82c95a851f7d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 20 Jan 2021 14:58:00 -0700 Subject: [PATCH 083/107] Don't include feature_states in restore request JSON if it's empty If we include it, some BWC tests use new_version request classes to make requests against old_version clusters. --- .../admin/cluster/snapshots/restore/RestoreSnapshotRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 8ab87164cf6ac..fe65e4b02b705 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -564,7 +564,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } - if (featureStates != null) { + if (featureStates != null && featureStates.length > 0) { builder.startArray("feature_states"); for (String plugin : featureStates) { builder.value(plugin); From 1476d8d5cbf68faf9d2b2b5abd1de5569df83e0f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 20 Jan 2021 16:56:41 -0700 Subject: [PATCH 084/107] Exercise feature_states in HLRC tests --- .../java/org/elasticsearch/client/SnapshotIT.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index c6b76e1ee1571..5e6104d937777 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -19,6 +19,8 @@ package org.elasticsearch.client; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; @@ -28,6 +30,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.elasticsearch.ElasticsearchException; @@ -164,6 +167,14 @@ public void testCreateSnapshot() throws Exception { } request.partial(randomBoolean()); request.includeGlobalState(randomBoolean()); + final List featureStates = randomFrom( + List.of( + Collections.emptyList(), + Collections.singletonList(TASKS_FEATURE_NAME), + Collections.singletonList(NO_FEATURE_STATES_VALUE) + ) + ); + request.featureStates(featureStates); CreateSnapshotResponse response = createTestSnapshot(request); assertEquals(waitForCompletion ? RestStatus.OK : RestStatus.ACCEPTED, response.status()); @@ -280,6 +291,10 @@ public void testRestoreSnapshot() throws IOException { request.waitForCompletion(true); request.renamePattern(testIndex); request.renameReplacement(restoredIndex); + if (randomBoolean()) { + request.includeGlobalState(true); + request.featureStates(Collections.singletonList(NO_FEATURE_STATES_VALUE)); + } RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore, highLevelClient().snapshot()::restoreAsync); From 884983c6f31a1c64daf157fc4cac2b701da1a2e5 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 21 Jan 2021 17:02:12 -0500 Subject: [PATCH 085/107] Add integration test for feature states and security index --- .../SecurityFeatureStateIntegTests.java | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java new file mode 100644 index 0000000000000..b41a99b6d8839 --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.integration; + +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.nio.file.Path; + +import static org.elasticsearch.test.SecuritySettingsSource.TEST_SUPERUSER; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class SecurityFeatureStateIntegTests extends AbstractPrivilegeTestCase { + + private static final String LOCAL_TEST_USER_NAME = "feature_state_user"; + private static final String LOCAL_TEST_USER_PASSWORD = "my_password"; + private static Path repositoryLocation; + + @BeforeClass + public static void setupRepositoryPath() { + repositoryLocation = createTempDir(); + } + + @AfterClass + public static void cleanupRepositoryPath() { + repositoryLocation = null; + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(super.nodeSettings()) + .put("path.repo", repositoryLocation) + .build(); + } + + /** + * Test that, when the security system index is restored as a feature state, + * the security plugin's listeners detect the state change and reload native + * realm privileges. + * + * We use the admin client to handle snapshots and the rest API to manage + * security roles and users. We use the native realm instead of the file + * realm because this test relies on dynamically changing privileges. + */ + public void testSecurityFeatureStateSnapshotAndRestore() throws Exception { + // set up a snapshot repository + final String repositoryName = "test-repo"; + client().admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put("location", repositoryLocation)) + .get(); + + // create a new role + final String roleName = "extra_role"; + final Request createRoleRequest = new Request("PUT", "/_security/role/" + roleName); + createRoleRequest.addParameter("refresh", "wait_for"); + createRoleRequest.setJsonEntity("{" + + " \"indices\": [" + + " {" + + " \"names\": [ \"test_index\" ]," + + " \"privileges\" : [ \"create\", \"create_index\", \"create_doc\" ]" + + " }" + + " ]" + + "}"); + performSuperuserRequest(createRoleRequest); + + // create a test user + final Request createUserRequest = new Request("PUT", "/_security/user/" + LOCAL_TEST_USER_NAME); + createUserRequest.addParameter("refresh", "wait_for"); + createUserRequest.setJsonEntity("{" + + " \"password\": \"" + LOCAL_TEST_USER_PASSWORD + "\"," + + " \"roles\": [ \"" + roleName + "\" ]" + + "}"); + performSuperuserRequest(createUserRequest); + + // test user posts a document + final Request postTestDocument1 = new Request("POST", "/test_index/_doc"); + postTestDocument1.setJsonEntity("{\"message\": \"before snapshot\"}"); + performTestUserRequest(postTestDocument1); + + // snapshot state + final String snapshotName = "security-state"; + client().admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName) + .setIndices("test_index") + .setFeatureStates("LocalStateSecurity") + .get(); + waitForSnapshotToFinish(repositoryName, snapshotName); + + // modify user's roles + final Request modifyUserRequest = new Request("PUT", "/_security/user/" + LOCAL_TEST_USER_NAME); + modifyUserRequest.addParameter("refresh", "wait_for"); + modifyUserRequest.setJsonEntity("{\"roles\": [] }"); + performSuperuserRequest(modifyUserRequest); + + // new user has lost privileges and can't post a document + final Request postDocumentRequest2 = new Request("POST", "/test_index/_doc"); + postDocumentRequest2.setJsonEntity("{\"message\": \"between snapshot and restore\"}"); + ResponseException exception = expectThrows(ResponseException.class, () -> performTestUserRequest(postDocumentRequest2)); + + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(exception.getMessage(), + containsString("action [" + IndexAction.NAME + "] is unauthorized for user [" + LOCAL_TEST_USER_NAME + "]")); + + client().admin().indices().prepareClose("test_index").get(); + // client().admin().indices().prepareClose(".security-7").get(); // shouldn't have to do this + + // restore state + client().admin().cluster().prepareRestoreSnapshot(repositoryName, snapshotName) + .setFeatureStates("LocalStateSecurity") + .setIndices("test_index") + .setWaitForCompletion(true) + .get(); + + // user has privileges again + final Request postDocumentRequest3 = new Request("POST", "/test_index/_doc"); + postDocumentRequest3.setJsonEntity("{\"message\": \"after restore\"}"); + performTestUserRequest(postDocumentRequest3); + } + + private Response performSuperuserRequest(Request request) throws Exception { + String token = UsernamePasswordToken.basicAuthHeaderValue( + TEST_SUPERUSER, new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())); + return performAuthenticatedRequest(request, token); + } + + private Response performTestUserRequest(Request request) throws Exception { + String token = UsernamePasswordToken.basicAuthHeaderValue( + LOCAL_TEST_USER_NAME, new SecureString(LOCAL_TEST_USER_PASSWORD.toCharArray())); + return performAuthenticatedRequest(request, token); + } + + private Response performAuthenticatedRequest(Request request, String token) throws Exception { + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Authorization", token); + request.setOptions(options); + return getRestClient().performRequest(request); + } + + private void waitForSnapshotToFinish(String repo, String snapshot) throws Exception { + assertBusy(() -> { + SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot).get(); + assertThat(response.getSnapshots().get(0).getState(), is(SnapshotsInProgress.State.SUCCESS)); + // The status of the snapshot in the repository can become SUCCESS before it is fully finalized in the cluster state so wait for + // it to disappear from the cluster state as well + SnapshotsInProgress snapshotsInProgress = + client().admin().cluster().state(new ClusterStateRequest()).get().getState().custom(SnapshotsInProgress.TYPE); + assertThat(snapshotsInProgress.entries(), Matchers.empty()); + }); + } +} From 1dc1a80b51344f072d43119ddb83cca5e5c6c6fd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 21 Jan 2021 15:09:44 -0700 Subject: [PATCH 086/107] Add a test to ensure system indices can be renamed if restored by name --- .../snapshots/SystemIndicesSnapshotIT.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 86edfe35dc891..b086e70102f40 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.snapshots; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -719,6 +720,36 @@ public void testNoneFeatureStateOnRestore() { assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); } + /** + * This test checks a piece of BWC logic, and so should be removed when we block restoring system indices by name. + * + * This test checks whether it's possible to change the name of a system index when it's restored by name (rather than by feature state) + */ + public void testCanRenameSystemIndicesIfRestoredByIndexName() { + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including our system index + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(false) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Now restore it with a rename + clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(SystemIndexTestPlugin.SYSTEM_INDEX_NAME) + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setFeatureStates(NO_FEATURE_STATES_VALUE) + .setRenamePattern(".test-(.+)") + .setRenameReplacement("restored-$1") + .get(); + + assertTrue("The renamed system index should be present", indexExists("restored-system-idx")); + assertTrue("The original index should still be present", indexExists(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), From 1e37dc6fb2d4387162c9256023b97967497527bd Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 21 Jan 2021 17:32:03 -0500 Subject: [PATCH 087/107] Add UUID comparison to security index state change listeners --- .../integration/SecurityFeatureStateIntegTests.java | 1 - .../xpack/security/authc/AuthenticationService.java | 4 +++- .../xpack/security/authc/esnative/NativeRealm.java | 5 ++++- .../authc/support/mapper/NativeRoleMappingStore.java | 6 ++++-- .../xpack/security/authz/store/CompositeRolesStore.java | 6 ++++-- .../xpack/security/authz/store/NativePrivilegeStore.java | 5 ++++- .../xpack/security/support/CacheInvalidatorRegistry.java | 2 ++ .../xpack/security/support/SecurityIndexManager.java | 9 ++++++--- .../xpack/security/authc/AuthenticationServiceTests.java | 2 +- .../xpack/security/authc/esnative/NativeRealmTests.java | 2 +- .../support/mapper/NativeRoleMappingStoreTests.java | 2 +- .../security/authz/store/CompositeRolesStoreTests.java | 2 +- .../security/authz/store/NativePrivilegeStoreTests.java | 3 +-- .../security/support/CacheInvalidatorRegistryTests.java | 2 +- .../security/support/SecurityIndexManagerTests.java | 2 +- 15 files changed, 34 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java index b41a99b6d8839..1c0d40d0c1996 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java @@ -126,7 +126,6 @@ public void testSecurityFeatureStateSnapshotAndRestore() throws Exception { containsString("action [" + IndexAction.NAME + "] is unauthorized for user [" + LOCAL_TEST_USER_NAME + "]")); client().admin().indices().prepareClose("test_index").get(); - // client().admin().indices().prepareClose(".security-7").get(); // shouldn't have to do this // restore state client().admin().cluster().prepareRestoreSnapshot(repositoryName, snapshotName) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java index c125c35f728c4..45cbaaba1d05a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java @@ -205,7 +205,9 @@ public void expireAll() { public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { if (lastSuccessfulAuthCache != null) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState)) { + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false) { expireAll(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java index 53e171ff4f321..7207dba192d84 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; @@ -42,7 +43,9 @@ protected void doAuthenticate(UsernamePasswordToken token, ActionListener> listener, List> listener) { } public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) || - previousState.isIndexUpToDate != currentState.isIndexUpToDate) { + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false + || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { invalidateAll(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index 04ea4e20fb2e1..625f5b5812610 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -56,6 +56,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; @@ -418,7 +419,9 @@ private static String toDocId(String application, String name) { } public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { invalidateAll(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java index 6c3ef32b7323d..487a7b5a14555 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java @@ -8,6 +8,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; @@ -33,6 +34,7 @@ public void registerCacheInvalidator(String name, CacheInvalidator cacheInvalida public void onSecurityIndexStageChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { cacheInvalidators.values().forEach(CacheInvalidator::invalidateAll); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 9b34726d2b203..6218b55eb213e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -219,8 +219,9 @@ public void clusterChanged(ClusterChangedEvent event) { final IndexRoutingTable routingTable = event.state().getRoutingTable().index(indexMetadata.getIndex()); indexHealth = new ClusterIndexHealth(indexMetadata, routingTable).getStatus(); } + final String indexUUID = indexMetadata != null ? indexMetadata.getIndexUUID() : null; final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, - concreteIndexName, indexHealth, indexState); + concreteIndexName, indexHealth, indexState, indexUUID); this.indexState = newState; if (newState.equals(previousState) == false) { @@ -441,7 +442,7 @@ private static Tuple parseMappingAndSettingsFromTemplateBytes( * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null); + public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null); public final Instant creationTime; public final boolean isIndexUpToDate; public final boolean indexAvailable; @@ -450,10 +451,11 @@ public static class State { public final String concreteIndexName; public final ClusterHealthStatus indexHealth; public final IndexMetadata.State indexState; + public final String indexUUID; public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, - IndexMetadata.State indexState) { + IndexMetadata.State indexState, String indexUUID) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; @@ -462,6 +464,7 @@ public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailab this.concreteIndexName = concreteIndexName; this.indexHealth = indexHealth; this.indexState = indexState; + this.indexUUID = indexUUID; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 4d8db20a50d8a..03cf4f1c198ce 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2028,6 +2028,6 @@ private void setCompletedToTrue(AtomicBoolean completed) { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid"); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 0b0b4d2c0d30f..76234a188af1a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -30,7 +30,7 @@ public class NativeRealmTests extends ESTestCase { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, "my_uuid"); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 8fc3734cd3ac0..89a8b36e21035 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -150,7 +150,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid"); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 581923d8c2ec6..c3263b85482b4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -809,7 +809,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN); + Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid"); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 7e71b193d3d17..da5d304a060a1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -610,8 +610,7 @@ private SecurityIndexManager.State dummyState( String concreteSecurityIndexName, boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( Instant.now(), isIndexUpToDate, true, true, null, - concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN - ); + concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, "my_uuid"); } private SearchHit[] buildHits(List sourcePrivileges) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index a60829ba63f99..32cdfae009a0f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -48,7 +48,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( final SecurityIndexManager.State previousState = SecurityIndexManager.State.UNRECOVERED_STATE; final SecurityIndexManager.State currentState = new SecurityIndexManager.State( Instant.now(), true, true, true, Version.CURRENT, - ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN); + ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, "my_uuid"); cacheInvalidatorRegistry.onSecurityIndexStageChange(previousState, currentState); verify(invalidator1).invalidateAll(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 4ba306f318fb4..2fed9d09ad842 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -107,7 +107,7 @@ void doExecute(ActionType action, Request request, ActionListener mappingSourceSupplier = () -> { throw new RuntimeException(); }; From 8f038ee190eefe9fb240178f38b9167a6be3bada Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 25 Jan 2021 12:45:01 -0700 Subject: [PATCH 088/107] Exclude all feature states in `createSnapshot` Otherwise some tests can fail because the snapshot includes a different number of shards than the test expects, due to system indices being implicitly included. --- .../elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 1a36b355438a8..1712735986114 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -89,6 +89,7 @@ import java.util.function.Predicate; import java.util.stream.StreamSupport; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -373,6 +374,7 @@ protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, Li .prepareCreateSnapshot(repositoryName, snapshot) .setIndices(indices.toArray(Strings.EMPTY_ARRAY)) .setWaitForCompletion(true) + .setFeatureStates(NO_FEATURE_STATES_VALUE) // Exclude all feature states to ensure only specified indices are included .get(); final SnapshotInfo snapshotInfo = response.getSnapshotInfo(); From 1e65d125efd7b8f31539c19c9ace482b17d4ae00 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 15:33:04 -0700 Subject: [PATCH 089/107] Undo import changes (thanks IntelliJ) --- .../elasticsearch/client/SnapshotClient.java | 8 +- .../client/SnapshotRequestConverters.java | 4 +- .../GetSnapshottableFeaturesResponse.java | 6 +- .../org/elasticsearch/client/SnapshotIT.java | 28 +++---- ...GetSnapshottableFeaturesResponseTests.java | 14 ++-- .../indices/TestSystemIndexPlugin.java | 6 +- .../snapshots/SystemIndicesSnapshotIT.java | 34 ++++----- .../create/CreateSnapshotRequest.java | 24 +++--- .../GetSnapshottableFeaturesResponse.java | 10 +-- .../restore/RestoreSnapshotRequest.java | 28 +++---- .../elasticsearch/indices/IndicesService.java | 74 +++++++++---------- .../elasticsearch/indices/SystemIndices.java | 24 +++--- .../java/org/elasticsearch/node/Node.java | 57 +++++++------- .../RestSnapshottableFeaturesAction.java | 6 +- .../snapshots/RestoreService.java | 71 +++++++++--------- .../snapshots/SnapshotsService.java | 65 ++++++++-------- .../core/async/AsyncTaskServiceTests.java | 12 +-- .../xpack/security/Security.java | 2 +- 18 files changed, 235 insertions(+), 238 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index 87d0287bcf8da..17b038b2b7c1d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -19,10 +19,6 @@ package org.elasticsearch.client; -import static java.util.Collections.emptySet; - -import java.io.IOException; - import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -46,6 +42,10 @@ import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; +import java.io.IOException; + +import static java.util.Collections.emptySet; + /** * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Snapshot API. *

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 83af683423f7f..cad5ffbf7cf28 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -19,8 +19,6 @@ package org.elasticsearch.client; -import java.io.IOException; - import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -39,6 +37,8 @@ import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; import org.elasticsearch.common.Strings; +import java.io.IOException; + final class SnapshotRequestConverters { private SnapshotRequestConverters() {} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java index ed33c3d1492c6..62abbb100d64a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java @@ -19,14 +19,14 @@ package org.elasticsearch.client.snapshots; -import java.util.List; -import java.util.Objects; - import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; +import java.util.List; +import java.util.Objects; + public class GetSnapshottableFeaturesResponse { private final List features; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 5e6104d937777..060ec8f139e7a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -19,20 +19,6 @@ package org.elasticsearch.client; -import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; -import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -63,6 +49,20 @@ import org.elasticsearch.snapshots.RestoreInfo; import org.mockito.internal.util.collections.Sets; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + public class SnapshotIT extends ESRestHighLevelClientTestCase { private AcknowledgedResponse createTestRepository(String repository, String type, String settings) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java index 6b9612d975c29..19f4fc478954f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -19,18 +19,18 @@ package org.elasticsearch.client.snapshots; -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.in; -import static org.hamcrest.Matchers.is; +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Map; import java.util.stream.Collectors; -import org.elasticsearch.client.AbstractResponseTestCase; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.is; public class GetSnapshottableFeaturesResponseTests extends AbstractResponseTestCase< org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java index d89ee3804ef68..3e2c7d5cbdad4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java @@ -19,13 +19,13 @@ package org.elasticsearch.indices; -import java.util.Collection; -import java.util.List; - import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; +import java.util.Collection; +import java.util.List; + /** * Just a test plugin to allow the test descriptor to be installed in the cluster. */ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index b086e70102f40..e2228ecebc08c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -19,23 +19,6 @@ package org.elasticsearch.snapshots; -import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.in; -import static org.hamcrest.Matchers.not; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -48,6 +31,23 @@ import org.elasticsearch.test.MockLogAppender; import org.junit.Before; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.not; + public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { public static final String REPO_NAME = "test-repo"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index d11f56997f27e..3e01cade1ab0f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -19,18 +19,6 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; -import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.common.Strings.EMPTY_ARRAY; -import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; -import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -46,6 +34,18 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.Strings.EMPTY_ARRAY; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + /** * Create snapshot request *

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index af21d0b88b062..d96045eb7be5c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -19,11 +19,6 @@ package org.elasticsearch.action.admin.cluster.snapshots.features; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -31,6 +26,11 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + public class GetSnapshottableFeaturesResponse extends ActionResponse implements ToXContentObject { private final List snapshottableFeatures; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index fe65e4b02b705..2203f0e2e849e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -19,20 +19,6 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore; -import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; -import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; -import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; - import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -45,6 +31,20 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; + /** * Restore snapshot request */ diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 1d2d47af2e716..75d7c17d41646 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -19,43 +19,6 @@ package org.elasticsearch.indices; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; -import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; -import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX; -import static org.elasticsearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; -import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; -import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.LongSupplier; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -176,6 +139,43 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.threadpool.ThreadPool; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; +import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX; +import static org.elasticsearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; +import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; + public class IndicesService extends AbstractLifecycleComponent implements IndicesClusterStateService.AllocatedIndices, IndexService.ShardStoreDeleter { private static final Logger logger = LogManager.getLogger(IndicesService.class); diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 9a46ffef07d43..fde2150dc103c 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -19,9 +19,15 @@ package org.elasticsearch.indices; -import static java.util.stream.Collectors.toUnmodifiableList; -import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; -import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.Index; +import org.elasticsearch.snapshots.SnapshotsService; import java.util.Collection; import java.util.Collections; @@ -32,15 +38,9 @@ import java.util.Optional; import java.util.stream.Collectors; -import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.MinimizationOperations; -import org.apache.lucene.util.automaton.Operations; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.index.Index; -import org.elasticsearch.snapshots.SnapshotsService; +import static java.util.stream.Collectors.toUnmodifiableList; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; +import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; /** * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index da96792363302..df409505f3019 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -19,35 +19,6 @@ package org.elasticsearch.node; -import static java.util.stream.Collectors.toList; - -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.net.ssl.SNIHostName; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; @@ -204,6 +175,34 @@ import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; +import javax.net.ssl.SNIHostName; +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; + /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used * in order to use a {@link Client} to perform actions/operations against the cluster. diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java index 76190221ade30..748d3a9d308b7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java @@ -19,9 +19,6 @@ package org.elasticsearch.rest.action.admin.cluster; -import java.io.IOException; -import java.util.List; - import org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesRequest; import org.elasticsearch.action.admin.cluster.snapshots.features.SnapshottableFeaturesAction; import org.elasticsearch.client.node.NodeClient; @@ -29,6 +26,9 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import java.io.IOException; +import java.util.List; + public class RestSnapshottableFeaturesAction extends BaseRestHandler { @Override public List routes() { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c424d87c438af..c73d1ca99108d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -18,38 +18,10 @@ */ package org.elasticsearch.snapshots; -import static java.util.Collections.unmodifiableSet; -import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; -import static org.elasticsearch.common.util.set.Sets.newHashSet; -import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; -import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - +import com.carrotsearch.hppc.IntHashSet; +import com.carrotsearch.hppc.IntSet; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -111,10 +83,37 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; -import com.carrotsearch.hppc.IntHashSet; -import com.carrotsearch.hppc.IntSet; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Collections.unmodifiableSet; +import static org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest.FEATURE_STATES_VERSION; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; +import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; /** * Service responsible for restoring snapshots diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d3c2647e2f900..afdfac9c61e58 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -19,37 +19,8 @@ package org.elasticsearch.snapshots; -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableList; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; -import static org.elasticsearch.cluster.SnapshotsInProgress.completed; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Deque; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executor; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -121,8 +92,36 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; +import static org.elasticsearch.cluster.SnapshotsInProgress.completed; /** * Service responsible for creating snapshots. This service runs all the steps executed on the master node during snapshot creation and diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java index 3efa4d113854b..b9aa464f5c621 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java @@ -5,12 +5,6 @@ */ package org.elasticsearch.xpack.core.async; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.delete.DeleteResponse; @@ -32,6 +26,12 @@ import org.elasticsearch.xpack.core.security.user.User; import org.junit.Before; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + // TODO: test CRUD operations public class AsyncTaskServiceTests extends ESSingleNodeTestCase { private AsyncTaskIndexService indexService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index b8468b7bbb8ff..68b6f4dee8209 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -214,10 +214,10 @@ import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; +import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorPrivileges; import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService; -import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction; From fda3a9c7b560d00161f4c7a7262228caba5a4501 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 15:39:28 -0700 Subject: [PATCH 090/107] ! -> == false --- .../java/org/elasticsearch/cluster/SnapshotsInProgress.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 5e84fac064688..b36bc20eb338c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -424,7 +424,7 @@ public boolean equals(Object o) { if (version.equals(entry.version) == false) return false; if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; - if (!featureStates.equals(entry.featureStates)) return false; + if (featureStates.equals(entry.featureStates) == false) return false; return true; } From 455f76270308c0242dfe2ea76efcdb1730d83376 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 15:43:59 -0700 Subject: [PATCH 091/107] DRY up test per review --- .../elasticsearch/snapshots/SystemIndicesSnapshotIT.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index e2228ecebc08c..e8447929923d0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -76,11 +76,7 @@ public void testRestoreSystemIndicesAsGlobalState() { refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); // run a snapshot including global state - CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") - .setWaitForCompletion(true) - .setIncludeGlobalState(true) - .get(); - assertSnapshotSuccess(createSnapshotResponse); + createFullSnapshot(REPO_NAME, "test-snap"); // add another document indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); From ba8aba300fb77b133f1639c69d4b9738bf5e302b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 15:44:38 -0700 Subject: [PATCH 092/107] Don't pre-size arrays per review --- .../admin/cluster/snapshots/create/CreateSnapshotRequest.java | 2 +- .../admin/cluster/snapshots/restore/RestoreSnapshotRequest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 3e01cade1ab0f..f76e4bd5704a2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -379,7 +379,7 @@ public CreateSnapshotRequest featureStates(String[] featureStates) { * @param featureStates The plugin states to be included in the snapshot */ public CreateSnapshotRequest featureStates(List featureStates) { - return featureStates(featureStates.toArray(new String[featureStates.size()])); + return featureStates(featureStates.toArray(EMPTY_ARRAY)); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 2203f0e2e849e..89192dedccb30 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -479,7 +479,7 @@ public RestoreSnapshotRequest featureStates(String[] featureStates) { * @param featureStates The feature states to be included in the snapshot */ public RestoreSnapshotRequest featureStates(List featureStates) { - return featureStates(featureStates.toArray(new String[featureStates.size()])); + return featureStates(featureStates.toArray(Strings.EMPTY_ARRAY)); } /** From 4c5ce561d51fc359156298a9feb255f8115aac89 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 15:59:47 -0700 Subject: [PATCH 093/107] Rearrange `startedEntry` params per review --- .../java/org/elasticsearch/cluster/SnapshotsInProgress.java | 5 ++--- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 4 ++-- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index b36bc20eb338c..ef28af44377e0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -96,9 +96,8 @@ public String toString() { * will be in state {@link State#SUCCESS} right away otherwise it will be in state {@link State#STARTED}. */ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, boolean partial, List indices, - List dataStreams, List featureStates, long repositoryStateId, - ImmutableOpenMap shards, Map userMetadata, - Version version, long startTime) { + List dataStreams, long startTime, long repositoryStateId, ImmutableOpenMap shards, + Map userMetadata, Version version, List featureStates) { return new SnapshotsInProgress.Entry(snapshot, includeGlobalState, partial, completed(shards.values()) ? State.SUCCESS : State.STARTED, indices, dataStreams, featureStates, startTime, repositoryStateId, shards, null, userMetadata, version); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index afdfac9c61e58..ab880aa1363c9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -329,8 +329,8 @@ public ClusterState execute(ClusterState currentState) { } newEntry = SnapshotsInProgress.startedEntry( new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), - indexIds, dataStreams, featureStates, repositoryData.getGenId(), shards, userMeta, version, - threadPool.absoluteTimeInMillis()); + indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, + userMeta, version, featureStates); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(CollectionUtils.appendToCopy(runningSnapshots, newEntry))).build(); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 527375b0f0bcc..6e5fb95b9c9e5 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -390,8 +390,8 @@ private static ClusterState applyUpdates(ClusterState state, SnapshotsService.Sh private static SnapshotsInProgress.Entry snapshotEntry(Snapshot snapshot, List indexIds, ImmutableOpenMap shards) { - return SnapshotsInProgress.startedEntry(snapshot, randomBoolean(), randomBoolean(), indexIds, Collections.emptyList(), - Collections.emptyList(), randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT, 1L); + return SnapshotsInProgress.startedEntry(snapshot, randomBoolean(), randomBoolean(), indexIds, Collections.emptyList(), 1L, + randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT, Collections.emptyList()); } private static SnapshotsInProgress.Entry cloneEntry( From eeda4e123a64be29069d52d86110e834c1918b0c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 2 Feb 2021 16:02:32 -0700 Subject: [PATCH 094/107] Move `FEATURE_STATES_VERSION` to `SnapshotsService` per review --- .../cluster/snapshots/create/CreateSnapshotRequest.java | 6 +++--- .../cluster/snapshots/restore/RestoreSnapshotRequest.java | 2 +- .../java/org/elasticsearch/cluster/SnapshotsInProgress.java | 2 +- .../java/org/elasticsearch/snapshots/RestoreService.java | 2 +- .../main/java/org/elasticsearch/snapshots/SnapshotInfo.java | 2 +- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 2 ++ 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index f76e4bd5704a2..16b4fd77dd8b8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.util.Arrays; @@ -64,7 +65,6 @@ public class CreateSnapshotRequest extends MasterNodeRequest Date: Tue, 2 Feb 2021 16:45:29 -0700 Subject: [PATCH 095/107] "Partial" fix in SnapshotsService#finalizeSnapshotEntry --- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 12b15e5a64800..94520d66107b7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1281,7 +1281,9 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met entry.partial() ? entry.dataStreams().stream() .filter(metaForSnapshot.dataStreams()::containsKey) .collect(Collectors.toList()) : entry.dataStreams(), - entry.featureStates(), // TODO: Fix this + entry.partial() ? entry.featureStates().stream() + .filter(state -> state.getIndices().stream().allMatch(metaForSnapshot::hasIndex)) + .collect(Collectors.toList()) : entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime()); From 56fbb20496153a40d74a1f7e6a1dc448f190ae4d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 3 Feb 2021 15:14:23 -0700 Subject: [PATCH 096/107] New license headers that didn't get auto-updated --- .../GetSnapshottableFeaturesRequest.java | 21 +++++-------------- .../GetSnapshottableFeaturesResponse.java | 21 +++++-------------- ...GetSnapshottableFeaturesResponseTests.java | 21 +++++-------------- .../snapshots/SystemIndicesSnapshotIT.java | 21 +++++-------------- .../GetSnapshottableFeaturesRequest.java | 21 +++++-------------- .../GetSnapshottableFeaturesResponse.java | 21 +++++-------------- .../features/SnapshottableFeaturesAction.java | 21 +++++-------------- .../TransportSnapshottableFeaturesAction.java | 21 +++++-------------- .../RestSnapshottableFeaturesAction.java | 21 +++++-------------- .../snapshots/SnapshotFeatureInfo.java | 21 +++++-------------- ...GetSnapshottableFeaturesResponseTests.java | 21 +++++-------------- .../snapshots/SnapshotFeatureInfoTests.java | 21 +++++-------------- .../SecurityFeatureStateIntegTests.java | 5 +++-- 13 files changed, 63 insertions(+), 194 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java index b50abf31b6611..458c3f5720440 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.snapshots; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java index 62abbb100d64a..3d19dc5b26772 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.snapshots; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java index 19f4fc478954f..5fe9809e839de 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.snapshots; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index e8447929923d0..24d4b7e83be4b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.snapshots; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java index 9741b5b411cb2..545f5c7fbdd7a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index d96045eb7be5c..7792f8be417d9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java index 9d0bc8c1d0e9b..38bf7afd6b505 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java index ef284b0bef81c..62f32bc460f0e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java index 748d3a9d308b7..50092106dd32b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.rest.action.admin.cluster; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index d553922394afb..4d0ec29baf3be 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.snapshots; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java index 3c073cfd959b8..40702a8b51a71 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.action.admin.cluster.snapshots.features; diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java index 061116b4dc76d..b16aa56293cf9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.snapshots; diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java index 1c0d40d0c1996..c3967c799c224 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.integration; From 12d1b25775cefb66d682e4d9b62af2826ba635d3 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 4 Feb 2021 16:57:46 -0700 Subject: [PATCH 097/107] Don't put Descriptor patterns through IndexNameExpressionResolver SystemIndexDescriptor patterns can use full Lucene regex syntax, so we can't feed them to IndexNameExpressionResolver anymore. This commit introduces a method to SystemIndexDescriptor to resolve all indices matching a descriptor. --- .../indices/SystemIndexDescriptor.java | 25 ++++++++ .../java/org/elasticsearch/node/Node.java | 2 +- .../snapshots/RestoreService.java | 60 ++++++++++--------- .../snapshots/SnapshotsService.java | 4 +- .../snapshots/SnapshotResiliencyTests.java | 4 +- 5 files changed, 60 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index c76d3600c68ee..0a63135b4a7bb 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -14,10 +14,15 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Objects; @@ -188,6 +193,26 @@ public boolean matchesIndexPattern(String index) { return indexPatternAutomaton.run(index); } + /** + * Retrieves a list of all indices which match this descriptor's pattern. + * + * This cannot be done via {@link org.elasticsearch.cluster.metadata.IndexNameExpressionResolver} because that class can only handle + * simple wildcard expressions, but system index name patterns may use full Lucene regular expression syntax, + * + * @param metadata The current metadata to get the list of matching indices from + * @return A list of index names that match this descriptor + */ + public List getMatchingIndices(Metadata metadata) { + ArrayList matchingIndices = new ArrayList<>(); + metadata.indices().keysIt().forEachRemaining(indexName -> { + if (matchesIndexPattern(indexName)) { + matchingIndices.add(indexName); + } + }); + + return Collections.unmodifiableList(matchingIndices); + } + /** * @return A short description of the purpose of this system index. */ diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 71f5b33a8c795..5a1dabb283977 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -604,7 +604,7 @@ protected Node(final Environment initialEnvironment, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), metadataCreateIndexService, clusterModule.getMetadataDeleteIndexService(), indexMetadataVerifier, - shardLimitValidator, systemIndices, clusterModule.getIndexNameExpressionResolver()); + shardLimitValidator, systemIndices); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state, clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, rerouteService); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 0fb2f32fa38f2..b8d499bd00830 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -34,13 +34,12 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; -import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; @@ -91,8 +90,6 @@ import java.util.stream.Stream; import static java.util.Collections.unmodifiableSet; -import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; @@ -102,6 +99,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; +import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; /** @@ -165,15 +163,18 @@ public class RestoreService implements ClusterStateApplier { private final SystemIndices systemIndices; - private final IndexNameExpressionResolver indexNameExpressionResolver; - private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(); - public RestoreService(ClusterService clusterService, RepositoriesService repositoriesService, - AllocationService allocationService, MetadataCreateIndexService createIndexService, - MetadataDeleteIndexService metadataDeleteIndexService, IndexMetadataVerifier indexMetadataVerifier, - ShardLimitValidator shardLimitValidator, SystemIndices systemIndices, - IndexNameExpressionResolver indexNameExpressionResolver) { + public RestoreService( + ClusterService clusterService, + RepositoriesService repositoriesService, + AllocationService allocationService, + MetadataCreateIndexService createIndexService, + MetadataDeleteIndexService metadataDeleteIndexService, + IndexMetadataVerifier indexMetadataVerifier, + ShardLimitValidator shardLimitValidator, + SystemIndices systemIndices + ) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; this.allocationService = allocationService; @@ -186,7 +187,6 @@ public RestoreService(ClusterService clusterService, RepositoriesService reposit this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; this.systemIndices = systemIndices; - this.indexNameExpressionResolver = indexNameExpressionResolver; } /** @@ -322,7 +322,10 @@ public ClusterState execute(ClusterState currentState) { } // Clear out all existing indices which fall within a system index pattern being restored - final Set systemIndicesToDelete = resolveIndicesToDelete(currentState, featureStatesToRestore); + final Set systemIndicesToDelete = resolveSystemIndicesToDelete( + currentState, + featureStatesToRestore.keySet() + ); currentState = metadataDeleteIndexService.deleteIndices(currentState, systemIndicesToDelete); // Updating cluster state @@ -724,28 +727,29 @@ private Map> getFeatureStatesToRestore(RestoreSnapshotReque return featureStatesToRestore; } - private Set resolveIndicesToDelete(ClusterState currentState, Map> featureStatesToRestore) { + /** + * Resolves a set of index names that currently exist in the cluster that are part of a feature state which is about to be restored, + * and should therefore be removed prior to restoring those feature states from the snapshot. + * + * @param currentState The current cluster state + * @param featureStatesToRestore A set of feature state names that are about to be restored + * @return A set of index names that should be removed based on the feature states being restored + */ + private Set resolveSystemIndicesToDelete(ClusterState currentState, Set featureStatesToRestore) { if (featureStatesToRestore == null) { return Collections.emptySet(); } - final String[] indexPatternsToDelete = featureStatesToRestore.keySet().stream() + return featureStatesToRestore.stream() .map(featureName -> systemIndices.getFeatures().get(featureName)) .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` .flatMap(feature -> feature.getIndexDescriptors().stream()) - .map(descriptor -> descriptor.getIndexPattern()) - .toArray(String[]::new); - - if (indexPatternsToDelete.length == 0) { - // If this is empty, it'll resolve to all indices, so explicitly return an empty set here. - return Collections.emptySet(); - } - - final String[] indexNamesToDelete = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, - LENIENT_EXPAND_OPEN_CLOSED, indexPatternsToDelete); - return Stream.of(indexNamesToDelete) - .map(idxName -> currentState.metadata().index(idxName).getIndex()) - .collect(Collectors.toSet()); + .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream()) + .map(indexName -> { + assert currentState.metadata().hasIndex(indexName) : "index [" + indexName + "] not found in metadata but must be present"; + return currentState.metadata().getIndices().get(indexName).getIndex(); + }) + .collect(Collectors.toUnmodifiableSet()); } //visible for testing diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index aac7591a73ae7..04e6486fb5ad9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -353,9 +353,7 @@ private List resolveFeatureIndexNames(ClusterState currentState, String final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); return feature.getIndexDescriptors().stream() - .map(SystemIndexDescriptor::getIndexPattern) - .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, - LENIENT_EXPAND_OPEN_CLOSED, pattern))) + .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream()) .collect(Collectors.toList()); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 4ae38e8f8415f..155824dfb682d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1569,9 +1569,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi indexScopedSettings, null), shardLimitValidator, - systemIndices, - indexNameExpressionResolver - ); + systemIndices); actions.put(PutMappingAction.INSTANCE, new TransportPutMappingAction(transportService, clusterService, threadPool, metadataMappingService, actionFilters, indexNameExpressionResolver, new RequestValidators<>(Collections.emptyList()), From 699c2aef1eea47df677f0436ca99a29d94efb35c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 4 Feb 2021 17:06:52 -0700 Subject: [PATCH 098/107] Unused imports --- .../java/org/elasticsearch/indices/SystemIndexDescriptor.java | 1 - .../main/java/org/elasticsearch/snapshots/SnapshotsService.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 0a63135b4a7bb..0651e29fb6392 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Objects; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 04e6486fb5ad9..3cf8f6b52e1d9 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -67,7 +67,6 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; @@ -108,7 +107,6 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; -import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; From d503868f626644fba39769fc01f6351e5ca6db11 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 5 Feb 2021 12:30:08 -0700 Subject: [PATCH 099/107] Only use feature_states if all nodes are on a version that can handle it (BWC) --- .../snapshots/SnapshotsService.java | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 3cf8f6b52e1d9..f09ea8faa70eb 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -258,40 +258,55 @@ public ClusterState execute(ClusterState currentState) { List featureStates = Collections.emptyList(); final List requestedStates = Arrays.asList(request.featureStates()); - if (request.includeGlobalState() || requestedStates.isEmpty() == false) { - final Set featureStatesSet; - if (request.includeGlobalState() && requestedStates.isEmpty()) { - // If we're including global state and feature states aren't specified, include all of them - featureStatesSet = new HashSet<>(systemIndexDescriptorMap.keySet()); - } else if (requestedStates.size() == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedStates.get(0))) { - // If there's exactly one value and it's "none", include no states - featureStatesSet = Collections.emptySet(); - } else { - // Otherwise, check for "none" then use the list of requested states - if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { - throw new IllegalArgumentException("the feature_states value [" + SnapshotsService.NO_FEATURE_STATES_VALUE + - "] indicates that no feature states should be snapshotted, but other feature states were requested: " + - requestedStates); + + // We should only use the feature states logic if we're sure we'll be able to finish the snapshot without a lower-version + // node taking over and causing problems. Therefore, if we're in a mixed cluster with versions that don't know how to handle + // feature states, skip all feature states logic, and if `feature_states` is explicitly configured, throw an exception. + if (currentState.nodes().getMinNodeVersion().onOrAfter(FEATURE_STATES_VERSION)) { + if (request.includeGlobalState() || requestedStates.isEmpty() == false) { + final Set featureStatesSet; + if (request.includeGlobalState() && requestedStates.isEmpty()) { + // If we're including global state and feature states aren't specified, include all of them + featureStatesSet = new HashSet<>(systemIndexDescriptorMap.keySet()); + } else if (requestedStates.size() == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedStates.get(0))) { + // If there's exactly one value and it's "none", include no states + featureStatesSet = Collections.emptySet(); + } else { + // Otherwise, check for "none" then use the list of requested states + if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { + throw new IllegalArgumentException("the feature_states value [" + SnapshotsService.NO_FEATURE_STATES_VALUE + + "] indicates that no feature states should be snapshotted, but other feature states were requested: " + + requestedStates); + } + featureStatesSet = new HashSet<>(requestedStates); } - featureStatesSet = new HashSet<>(requestedStates); - } - featureStates = systemIndexDescriptorMap.keySet().stream() - .filter(feature -> featureStatesSet.contains(feature)) - .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) - .filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates - .collect(Collectors.toList()); - final Stream featureStateIndices = featureStates.stream().flatMap(feature -> feature.getIndices().stream()); - - final Stream associatedIndices = systemIndexDescriptorMap.keySet().stream() - .filter(feature -> featureStatesSet.contains(feature)) - .flatMap(feature -> resolveAssociatedIndices(currentState, feature).stream()); - - // Add all resolved indices from the feature states to the list of indices - indices = Stream.of(indices.stream(), featureStateIndices, associatedIndices) - .flatMap(s -> s) - .distinct() - .collect(Collectors.toList()); + featureStates = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature)) + .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) + .filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates + .collect(Collectors.toList()); + final Stream featureStateIndices = featureStates.stream().flatMap(feature -> feature.getIndices().stream()); + + final Stream associatedIndices = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature)) + .flatMap(feature -> resolveAssociatedIndices(currentState, feature).stream()); + + // Add all resolved indices from the feature states to the list of indices + indices = Stream.of(indices.stream(), featureStateIndices, associatedIndices) + .flatMap(s -> s) + .distinct() + .collect(Collectors.toList()); + } + } else if (requestedStates.isEmpty() == false) { + throw new SnapshotException( + new Snapshot(repositoryName, snapshotId), + "feature_states can only be used when all nodes in cluster are version [" + + FEATURE_STATES_VERSION + + "] or higher, but at least one node in this cluster is on version [" + + currentState.nodes().getMinNodeVersion() + + "]" + ); } final List dataStreams = From f2d7bbe390db9e79434a3a165acf6a60f157e83d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 5 Feb 2021 17:01:14 -0700 Subject: [PATCH 100/107] If any feature state indices are partial, drop associated feature state --- .../snapshots/SystemIndicesSnapshotIT.java | 82 ++++++++++++++++++- .../snapshots/SnapshotsService.java | 27 +++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 24d4b7e83be4b..6dcf39bd33050 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -10,13 +10,17 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockLogAppender; import org.junit.Before; @@ -25,6 +29,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; @@ -34,13 +39,18 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { public static final String REPO_NAME = "test-repo"; + private List dataNodes = null; + @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); @@ -52,7 +62,8 @@ protected Collection> nodePlugins() { @Before public void setup() { - createRepository(REPO_NAME, "fs"); + internalCluster().startMasterOnlyNodes(2); + dataNodes = internalCluster().startDataOnlyNodes(2); } /** @@ -60,6 +71,7 @@ public void setup() { * with no reference to feature state, the system indices are restored too. */ public void testRestoreSystemIndicesAsGlobalState() { + createRepository(REPO_NAME, "fs"); // put a document in a system index indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -88,6 +100,7 @@ public void testRestoreSystemIndicesAsGlobalState() { * If we take a snapshot with includeGlobalState set to false, are system indices included? */ public void testSnapshotWithoutGlobalState() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc"); indexDoc("not-a-system-index", "1", "purpose", "non system index doc"); @@ -115,6 +128,7 @@ public void testSnapshotWithoutGlobalState() { * Test that we can snapshot feature states by name. */ public void testSnapshotByFeature() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -155,6 +169,7 @@ public void testSnapshotByFeature() { * exception. */ public void testDefaultRestoreOnlyRegularIndices() { + createRepository(REPO_NAME, "fs"); final String regularIndex = "test-idx"; indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); @@ -188,6 +203,7 @@ public void testDefaultRestoreOnlyRegularIndices() { * Take a snapshot with global state but restore features by state. */ public void testRestoreByFeature() { + createRepository(REPO_NAME, "fs"); final String regularIndex = "test-idx"; indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); @@ -233,6 +249,7 @@ public void testRestoreByFeature() { * when that feature state is selected. */ public void testSnapshotAndRestoreAssociatedIndices() { + createRepository(REPO_NAME, "fs"); final String regularIndex = "regular-idx"; // put documents into a regular index as well as the system index and associated index of a feature @@ -286,6 +303,7 @@ public void testSnapshotAndRestoreAssociatedIndices() { * Check that if we request a feature not in the snapshot, we get an error. */ public void testRestoreFeatureNotInSnapshot() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -313,6 +331,7 @@ public void testRestoreFeatureNotInSnapshot() { * @throws IllegalAccessException if something goes wrong with the mock log appender */ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessException { + createRepository(REPO_NAME, "fs"); // put a document in system index indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -357,6 +376,7 @@ public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessExc * Check that if a system index matches a rename pattern in a restore request, it's not renamed */ public void testSystemIndicesCannotBeRenamed() { + createRepository(REPO_NAME, "fs"); final String nonSystemIndex = ".test-non-system-index"; indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(nonSystemIndex, "1", "purpose", "pre-snapshot doc"); @@ -393,6 +413,7 @@ public void testSystemIndicesCannotBeRenamed() { * all feature states should be restored. */ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -428,6 +449,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { * "all indices." */ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() { + createRepository(REPO_NAME, "fs"); String regularIndex = "my-index"; indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); @@ -467,6 +489,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() * saying that the system index must be closed, because here it is included in "all indices." */ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoIndicesSpecified() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -497,6 +520,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoI * However, other feature states should be unaffected. */ public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { + createRepository(REPO_NAME, "fs"); // Create a system index we'll snapshot and restore final String systemIndexInSnapshot = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; indexDoc(systemIndexInSnapshot, "1", "purpose", "pre-snapshot doc"); @@ -546,6 +570,7 @@ public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { } public void testSystemIndexAliasesAreAlwaysRestored() { + createRepository(REPO_NAME, "fs"); // Create a system index final String systemIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; indexDoc(systemIndexName, "1", "purpose", "pre-snapshot doc"); @@ -597,6 +622,7 @@ public void testSystemIndexAliasesAreAlwaysRestored() { * feature state names, and an error occurs if it's tried. */ public void testNoneFeatureStateMustBeAlone() { + createRepository(REPO_NAME, "fs"); // put a document in a system index indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -644,6 +670,7 @@ public void testNoneFeatureStateMustBeAlone() { * Tests that using the special "none" feature state value creates a snapshot with no feature states included */ public void testNoneFeatureStateOnCreation() { + createRepository(REPO_NAME, "fs"); final String regularIndex = "test-idx"; indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); @@ -669,6 +696,7 @@ public void testNoneFeatureStateOnCreation() { } public void testNoneFeatureStateOnRestore() { + createRepository(REPO_NAME, "fs"); final String regularIndex = "test-idx"; indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); @@ -711,6 +739,7 @@ public void testNoneFeatureStateOnRestore() { * This test checks whether it's possible to change the name of a system index when it's restored by name (rather than by feature state) */ public void testCanRenameSystemIndicesIfRestoredByIndexName() { + createRepository(REPO_NAME, "fs"); indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); @@ -735,6 +764,57 @@ public void testCanRenameSystemIndicesIfRestoredByIndexName() { assertTrue("The original index should still be present", indexExists(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)); } + /** + * Ensures that if we can only capture a partial snapshot of a system index, then the feature state associated with that index is + * not included in the snapshot, because it would not be safe to restore that feature state. + */ + public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Exception { + final String partialIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; + + createRepositoryNoVerify(REPO_NAME, "mock"); + + // Creating the index that we'll get a partial snapshot of with a bunch of shards + assertAcked(prepareCreate(partialIndexName, 0, indexSettingsNoReplicas(6))); + indexDoc(partialIndexName, "1", "purpose", "pre-snapshot doc"); + // And another one with the default + indexDoc(fullIndexName, "1", "purpose", "pre-snapshot doc"); + ensureGreen(); + + // Stop a random data node so we lose a shard from the partial index + internalCluster().stopRandomDataNode(); + assertBusy(() -> assertEquals(ClusterHealthStatus.RED, client().admin().cluster().prepareHealth().get().getStatus()), + 30, TimeUnit.SECONDS); + + // Get ready to block + blockMasterFromFinalizingSnapshotOnIndexFile(REPO_NAME); + + // Start a snapshot and wait for it to hit the block, then kill the master to force a failover + final String partialSnapName = "test-partial-snap"; + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, partialSnapName) + .setIncludeGlobalState(true) + .setWaitForCompletion(false) + .setPartial(true) + .get(); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + waitForBlock(internalCluster().getMasterName(), REPO_NAME); + internalCluster().stopCurrentMasterNode(); + + // Now get the snapshot and do our checks + assertBusy(() -> { + GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() + .prepareGetSnapshots(REPO_NAME).setSnapshots(partialSnapName).get(); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(REPO_NAME).get(0); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.failedShards(), lessThan(snapshotInfo.totalShards())); + List statesInSnapshot = snapshotInfo.featureStates().stream() + .map(SnapshotFeatureInfo::getPluginName) + .collect(Collectors.toList()); + assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); + }); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index f09ea8faa70eb..d691ffe86f463 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1281,9 +1281,7 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met entry.partial() ? entry.dataStreams().stream() .filter(metaForSnapshot.dataStreams()::containsKey) .collect(Collectors.toList()) : entry.dataStreams(), - entry.partial() ? entry.featureStates().stream() - .filter(state -> state.getIndices().stream().allMatch(metaForSnapshot::hasIndex)) - .collect(Collectors.toList()) : entry.featureStates(), + entry.partial() ? onlySuccessfulFeatureStates(entry) : entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime()); @@ -1308,6 +1306,29 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met } } + /** + * Removes all feature states which have missing or failed shards, as they are no longer safely restorable. + * @param entry The "in progress" entry with a list of feature states and one or more failed shards. + * @return The list of feature states which were completed successfully in the given entry. + */ + private List onlySuccessfulFeatureStates(SnapshotsInProgress.Entry entry) { + assert entry.partial() : "should not try to filter feature states from a non-partial entry"; + + // Figure out which indices have unsuccessful shards + Set indicesWithUnsuccessfulShards = new HashSet<>(); + entry.shards().keysIt().forEachRemaining(shardId -> { + final ShardState shardState = entry.shards().get(shardId).state(); + if (shardState.failed() || shardState.completed() == false) { + indicesWithUnsuccessfulShards.add(shardId.getIndexName()); + } + }); + + // Now remove any feature states which contain any of those indices, as the feature state is not intact and not safely restorable + return entry.featureStates().stream() + .filter(stateInfo -> stateInfo.getIndices().stream().anyMatch(indicesWithUnsuccessfulShards::contains) == false) + .collect(Collectors.toList()); + } + /** * Remove a snapshot from {@link #endingSnapshots} set and return its completion listeners that must be resolved. */ From 6d9edd800362f7a982e5e019f37fe981c3816e9c Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 5 Feb 2021 17:06:23 -0700 Subject: [PATCH 101/107] Unused import --- .../org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 6dcf39bd33050..a8db5ddf07981 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -39,7 +39,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; From 38022ddf0fb242b40f239c7398e887f2ee05edf0 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 8 Feb 2021 16:22:10 -0700 Subject: [PATCH 102/107] Check indices in snapshot as well as failed shards --- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index d691ffe86f463..ae305b7497bc1 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1313,6 +1313,9 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met */ private List onlySuccessfulFeatureStates(SnapshotsInProgress.Entry entry) { assert entry.partial() : "should not try to filter feature states from a non-partial entry"; + final Set indicesInSnapshot = entry.indices().stream() + .map(IndexId::getName) + .collect(Collectors.toUnmodifiableSet()); // Figure out which indices have unsuccessful shards Set indicesWithUnsuccessfulShards = new HashSet<>(); @@ -1325,6 +1328,7 @@ private List onlySuccessfulFeatureStates(SnapshotsInProgres // Now remove any feature states which contain any of those indices, as the feature state is not intact and not safely restorable return entry.featureStates().stream() + .filter(stateInfo -> indicesInSnapshot.containsAll(stateInfo.getIndices())) .filter(stateInfo -> stateInfo.getIndices().stream().anyMatch(indicesWithUnsuccessfulShards::contains) == false) .collect(Collectors.toList()); } From 873de63e27f1606e49552a0ddf39283f344a4294 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 8 Feb 2021 16:26:49 -0700 Subject: [PATCH 103/107] Remove extraneous line break --- .../client/snapshots/GetSnapshottableFeaturesResponseTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java index 5fe9809e839de..0b899af725c7b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -38,7 +38,6 @@ protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshott ) ) ); - } @Override From 6dd31f182ed8736809c848aa8b3e807f6a0b88c8 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 8 Feb 2021 16:28:12 -0700 Subject: [PATCH 104/107] `!` -> `== false` --- .../client/snapshots/GetSnapshottableFeaturesResponse.java | 4 ++-- .../snapshots/features/GetSnapshottableFeaturesResponse.java | 4 ++-- .../java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java index 3d19dc5b26772..049eba6b051b8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java @@ -46,7 +46,7 @@ public static GetSnapshottableFeaturesResponse parse(XContentParser parser) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof GetSnapshottableFeaturesResponse)) return false; + if ((o instanceof GetSnapshottableFeaturesResponse) == false) return false; GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; return getFeatures().equals(that.getFeatures()); } @@ -95,7 +95,7 @@ public String getDescription() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SnapshottableFeature)) return false; + if ((o instanceof SnapshottableFeature) == false) return false; SnapshottableFeature feature = (SnapshottableFeature) o; return Objects.equals(getFeatureName(), feature.getFeatureName()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java index 7792f8be417d9..a2048bab29c58 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -59,7 +59,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof GetSnapshottableFeaturesResponse)) return false; + if ((o instanceof GetSnapshottableFeaturesResponse) == false) return false; GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; return snapshottableFeatures.equals(that.snapshottableFeatures); } @@ -110,7 +110,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SnapshottableFeature)) return false; + if ((o instanceof SnapshottableFeature) == false) return false; SnapshottableFeature feature = (SnapshottableFeature) o; return Objects.equals(getFeatureName(), feature.getFeatureName()); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java index 4d0ec29baf3be..419d75a7226a5 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -89,7 +89,7 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SnapshotFeatureInfo)) return false; + if ((o instanceof SnapshotFeatureInfo) == false) return false; SnapshotFeatureInfo that = (SnapshotFeatureInfo) o; return getPluginName().equals(that.getPluginName()) && getIndices().equals(that.getIndices()); From dd8889ffacf796962bff9f2cfea17a1fda5c5e39 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 10 Feb 2021 18:36:57 -0700 Subject: [PATCH 105/107] Add test - but this test doesn't fail if logic is removed This test does not get a state where the SnapshotsInProgress.Entry has a feature state but not the index in that feature state --- .../snapshots/SystemIndicesSnapshotIT.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index a8db5ddf07981..b2308b0e05550 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.snapshots; import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -814,6 +815,68 @@ public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Except }); } + public void testParallelIndexDeleteRemovesFeatureState() throws Exception { + final String indexToBeDeleted = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; + + createRepositoryNoVerify(REPO_NAME, "mock"); + + // Creating the index that we'll get a partial snapshot of with a bunch of shards + assertAcked(prepareCreate(indexToBeDeleted, 0, indexSettingsNoReplicas(6))); + indexDoc(indexToBeDeleted, "1", "purpose", "pre-snapshot doc"); + // And another one with the default + indexDoc(fullIndexName, "1", "purpose", "pre-snapshot doc"); + refresh(); + ensureGreen(); + + logger.info("--> Created indices, blocking repo..."); + blockNodeOnAnyFiles(REPO_NAME, internalCluster().getMasterName()); +// blockAllDataNodes(REPO_NAME); + + // Start a snapshot - need to do this async because some blocks will block this call + logger.info("--> Blocked repo, starting snapshot..."); + final String partialSnapName = "test-partial-snap"; + ActionFuture createSnapshotFuture = clusterAdmin().prepareCreateSnapshot(REPO_NAME, partialSnapName) + .setIncludeGlobalState(true) + .setWaitForCompletion(false) + .setPartial(true) + .execute(); + + logger.info("--> Started snapshot, waiting for block..."); + waitForBlock(internalCluster().getMasterName(), REPO_NAME); +// waitForBlockOnAnyDataNode(REPO_NAME); + + logger.info("--> Repo hit block, deleting the index..."); + assertAcked(cluster().client().admin().indices().prepareDelete(indexToBeDeleted)); + + logger.info("--> Index deleted, unblocking repo..."); + unblockNode(REPO_NAME, internalCluster().getMasterName()); +// unblockAllDataNodes(REPO_NAME); + + logger.info("--> Repo unblocked, checking that snapshot started..."); + CreateSnapshotResponse createSnapshotResponse = createSnapshotFuture.actionGet(); + logger.info(createSnapshotResponse.toString()); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + + logger.info("--> Snapshot was started sucessfully, waiting for all operations to complete..."); + awaitNoMoreRunningOperations(); + + logger.info("--> All operations complete, running assertions"); + // Now get the snapshot and do our checks + assertBusy(() -> { + GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() + .prepareGetSnapshots(REPO_NAME).setSnapshots(partialSnapName).get(); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(REPO_NAME).get(0); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.indices(), not(hasItem(indexToBeDeleted))); + List statesInSnapshot = snapshotInfo.featureStates().stream() + .map(SnapshotFeatureInfo::getPluginName) + .collect(Collectors.toList()); + assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); + }); + } + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), From d3005866c05e13a7ec7df20824e1680b39a737b9 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 10 Feb 2021 18:37:16 -0700 Subject: [PATCH 106/107] Remove commented lines --- .../org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index b2308b0e05550..69e642897e66f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -831,7 +831,6 @@ public void testParallelIndexDeleteRemovesFeatureState() throws Exception { logger.info("--> Created indices, blocking repo..."); blockNodeOnAnyFiles(REPO_NAME, internalCluster().getMasterName()); -// blockAllDataNodes(REPO_NAME); // Start a snapshot - need to do this async because some blocks will block this call logger.info("--> Blocked repo, starting snapshot..."); @@ -844,14 +843,12 @@ public void testParallelIndexDeleteRemovesFeatureState() throws Exception { logger.info("--> Started snapshot, waiting for block..."); waitForBlock(internalCluster().getMasterName(), REPO_NAME); -// waitForBlockOnAnyDataNode(REPO_NAME); logger.info("--> Repo hit block, deleting the index..."); assertAcked(cluster().client().admin().indices().prepareDelete(indexToBeDeleted)); logger.info("--> Index deleted, unblocking repo..."); unblockNode(REPO_NAME, internalCluster().getMasterName()); -// unblockAllDataNodes(REPO_NAME); logger.info("--> Repo unblocked, checking that snapshot started..."); CreateSnapshotResponse createSnapshotResponse = createSnapshotFuture.actionGet(); From 6930e000a51bd2d393e8353714acbba18084bc21 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 11 Feb 2021 10:04:58 -0700 Subject: [PATCH 107/107] Fix test and use real final indices list --- .../snapshots/SystemIndicesSnapshotIT.java | 51 ++++++++++--------- .../snapshots/SnapshotsService.java | 17 +++---- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java index 69e642897e66f..a617b672a57ca 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -818,6 +818,11 @@ public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Except public void testParallelIndexDeleteRemovesFeatureState() throws Exception { final String indexToBeDeleted = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String nonsystemIndex = "nonsystem-idx"; + + // Stop one data node so we only have one data node to start with + internalCluster().stopNode(dataNodes.get(1)); + dataNodes.remove(1); createRepositoryNoVerify(REPO_NAME, "mock"); @@ -826,52 +831,52 @@ public void testParallelIndexDeleteRemovesFeatureState() throws Exception { indexDoc(indexToBeDeleted, "1", "purpose", "pre-snapshot doc"); // And another one with the default indexDoc(fullIndexName, "1", "purpose", "pre-snapshot doc"); + + // Now start up a new node and create an index that should get allocated to it + dataNodes.add(internalCluster().startDataOnlyNode()); + createIndexWithContent( + nonsystemIndex, + indexSettingsNoReplicas(2).put("index.routing.allocation.require._name", dataNodes.get(1)).build() + ); refresh(); ensureGreen(); - logger.info("--> Created indices, blocking repo..."); - blockNodeOnAnyFiles(REPO_NAME, internalCluster().getMasterName()); + logger.info("--> Created indices, blocking repo on new data node..."); + blockDataNode(REPO_NAME, dataNodes.get(1)); // Start a snapshot - need to do this async because some blocks will block this call logger.info("--> Blocked repo, starting snapshot..."); final String partialSnapName = "test-partial-snap"; ActionFuture createSnapshotFuture = clusterAdmin().prepareCreateSnapshot(REPO_NAME, partialSnapName) + .setIndices(nonsystemIndex) .setIncludeGlobalState(true) - .setWaitForCompletion(false) + .setWaitForCompletion(true) .setPartial(true) .execute(); logger.info("--> Started snapshot, waiting for block..."); - waitForBlock(internalCluster().getMasterName(), REPO_NAME); + waitForBlock(dataNodes.get(1), REPO_NAME); logger.info("--> Repo hit block, deleting the index..."); assertAcked(cluster().client().admin().indices().prepareDelete(indexToBeDeleted)); logger.info("--> Index deleted, unblocking repo..."); - unblockNode(REPO_NAME, internalCluster().getMasterName()); + unblockNode(REPO_NAME, dataNodes.get(1)); - logger.info("--> Repo unblocked, checking that snapshot started..."); + logger.info("--> Repo unblocked, checking that snapshot finished..."); CreateSnapshotResponse createSnapshotResponse = createSnapshotFuture.actionGet(); logger.info(createSnapshotResponse.toString()); - assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); - - logger.info("--> Snapshot was started sucessfully, waiting for all operations to complete..."); - awaitNoMoreRunningOperations(); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.OK)); logger.info("--> All operations complete, running assertions"); - // Now get the snapshot and do our checks - assertBusy(() -> { - GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() - .prepareGetSnapshots(REPO_NAME).setSnapshots(partialSnapName).get(); - SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(REPO_NAME).get(0); - assertNotNull(snapshotInfo); - assertThat(snapshotInfo.indices(), not(hasItem(indexToBeDeleted))); - List statesInSnapshot = snapshotInfo.featureStates().stream() - .map(SnapshotFeatureInfo::getPluginName) - .collect(Collectors.toList()); - assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); - assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); - }); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.indices(), not(hasItem(indexToBeDeleted))); + List statesInSnapshot = snapshotInfo.featureStates().stream() + .map(SnapshotFeatureInfo::getPluginName) + .collect(Collectors.toList()); + assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); } private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 8916bf6408720..7f849ba68c043 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -67,7 +67,6 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.SystemIndices; @@ -77,7 +76,6 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -1299,12 +1297,15 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met } metadataListener.whenComplete(meta -> { final Metadata metaForSnapshot = metadataForSnapshot(entry, meta); + final List finalIndices = shardGenerations.indices().stream() + .map(IndexId::getName) + .collect(Collectors.toList()); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), - shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), + finalIndices, entry.partial() ? entry.dataStreams().stream() .filter(metaForSnapshot.dataStreams()::containsKey) .collect(Collectors.toList()) : entry.dataStreams(), - entry.partial() ? onlySuccessfulFeatureStates(entry) : entry.featureStates(), + entry.partial() ? onlySuccessfulFeatureStates(entry, finalIndices) : entry.featureStates(), failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, entry.includeGlobalState(), entry.userMetadata(), entry.startTime()); @@ -1332,13 +1333,11 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met /** * Removes all feature states which have missing or failed shards, as they are no longer safely restorable. * @param entry The "in progress" entry with a list of feature states and one or more failed shards. + * @param finalIndices The final list of indices in the snapshot, after any indices that were concurrently deleted are removed. * @return The list of feature states which were completed successfully in the given entry. */ - private List onlySuccessfulFeatureStates(SnapshotsInProgress.Entry entry) { + private List onlySuccessfulFeatureStates(SnapshotsInProgress.Entry entry, List finalIndices) { assert entry.partial() : "should not try to filter feature states from a non-partial entry"; - final Set indicesInSnapshot = entry.indices().stream() - .map(IndexId::getName) - .collect(Collectors.toUnmodifiableSet()); // Figure out which indices have unsuccessful shards Set indicesWithUnsuccessfulShards = new HashSet<>(); @@ -1351,7 +1350,7 @@ private List onlySuccessfulFeatureStates(SnapshotsInProgres // Now remove any feature states which contain any of those indices, as the feature state is not intact and not safely restorable return entry.featureStates().stream() - .filter(stateInfo -> indicesInSnapshot.containsAll(stateInfo.getIndices())) + .filter(stateInfo -> finalIndices.containsAll(stateInfo.getIndices())) .filter(stateInfo -> stateInfo.getIndices().stream().anyMatch(indicesWithUnsuccessfulShards::contains) == false) .collect(Collectors.toList()); }