INFO logging of snapshot restore and completion

docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc

@@ -144,6 +144,10 @@ backing indices are configured using the data stream's matching index template.
 If `true`, the request restores aliases for any restored data streams and
 indices. If `false`, the request doesn't restore aliases. Defaults to `true`.
 
+`silent`::
+(Optional, Boolean)
+If `true`, logging happens at a lower level. Defaults to `false`.
+
 [[restore-snapshot-api-include-global-state]]
 `include_global_state`::
 +

server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

@@ -727,6 +727,7 @@ public ClusterState.Custom randomCreate(String name) {
                             UUIDs.randomBase64UUID(),
                             new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())),
                             RestoreInProgress.State.fromValue((byte) randomIntBetween(0, 3)),
+                            randomBoolean(),
                             emptyList(),
                             ImmutableOpenMap.of()
                         )

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java

@@ -48,6 +48,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
+    private boolean silent = false;
     private Settings indexSettings = Settings.EMPTY;
     private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
 
@@ -83,6 +84,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
         includeGlobalState = in.readBoolean();
         partial = in.readBoolean();
         includeAliases = in.readBoolean();
+        silent = in.readBoolean();
         indexSettings = readSettingsFromStream(in);
         ignoreIndexSettings = in.readStringArray();
         snapshotUuid = in.readOptionalString();
@@ -102,6 +104,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeGlobalState);
         out.writeBoolean(partial);
         out.writeBoolean(includeAliases);
+        out.writeBoolean(silent);
         writeSettingsToStream(indexSettings, out);
         out.writeStringArray(ignoreIndexSettings);
         out.writeOptionalString(snapshotUuid);
@@ -383,6 +386,26 @@ public boolean includeAliases() {
         return includeAliases;
     }
 
+    /**
+     * If set to true the logging should be at a lower level
+     *
+     * @param silent true if logging should be at a lower level
+     * @return this request
+     */
+    public RestoreSnapshotRequest silent(boolean silent) {
+        this.silent = silent;
+        return this;
+    }
+
+    /**
+     * Returns a boolean for whether logging should be at a lower or higher level
+     *
+     * @return true if logging should be at a lower level
+     */
+    public boolean silent() {
+        return silent;
+    }
+
     /**
      * Sets settings that should be added/changed in all restored indices
      */
@@ -505,6 +528,8 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
                 includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             } else if (name.equals("include_aliases")) {
                 includeAliases = nodeBooleanValue(entry.getValue(), "include_aliases");
+            } else if (name.equals("silent")) {
+                silent = nodeBooleanValue(entry.getValue(), "silent");
             } else if (name.equals("rename_pattern")) {
                 if (entry.getValue() instanceof String) {
                     renamePattern((String) entry.getValue());
@@ -573,6 +598,7 @@ private void toXContentFragment(XContentBuilder builder, Params params) throws I
         builder.field("include_global_state", includeGlobalState);
         builder.field("partial", partial);
         builder.field("include_aliases", includeAliases);
+        builder.field("silent", silent);
         if (indexSettings != null) {
             builder.startObject("index_settings");
             if (indexSettings.isEmpty() == false) {
@@ -601,6 +627,7 @@ public boolean equals(Object o) {
             && includeGlobalState == that.includeGlobalState
             && partial == that.partial
             && includeAliases == that.includeAliases
+            && silent == that.silent
             && Objects.equals(snapshot, that.snapshot)
             && Objects.equals(repository, that.repository)
             && Arrays.equals(indices, that.indices)
@@ -626,6 +653,7 @@ public int hashCode() {
             includeGlobalState,
             partial,
             includeAliases,
+            silent,
             indexSettings,
             snapshotUuid,
             skipOperatorOnlyState

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java

@@ -162,6 +162,17 @@ public RestoreSnapshotRequestBuilder setIncludeAliases(boolean restoreAliases) {
         return this;
     }
 
+    /**
+     * If set to true the logging should be at a lower level
+     *
+     * @param silent true if logging should be at a lower level
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSilent(boolean silent) {
+        request.silent(silent);
+        return this;
+    }
+
     /**
      * Sets index settings that should be added or replaced during restore
      *

server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java

@@ -104,19 +104,35 @@ public RestoreInProgress build() {
     /**
      * Restore metadata
      */
-    public record Entry(String uuid, Snapshot snapshot, State state, List<String> indices, Map<ShardId, ShardRestoreStatus> shards) {
+    public record Entry(
+        String uuid,
+        Snapshot snapshot,
+        State state,
+        boolean silent,
+        List<String> indices,
+        Map<ShardId, ShardRestoreStatus> shards
+    ) {
         /**
          * Creates new restore metadata
          *
          * @param uuid     uuid of the restore
          * @param snapshot snapshot
          * @param state    current state of the restore process
+         * @param silent   boolean for whether logging should be at a lower or higher level
          * @param indices  list of indices being restored
          * @param shards   map of shards being restored to their current restore status
          */
-        public Entry(String uuid, Snapshot snapshot, State state, List<String> indices, Map<ShardId, ShardRestoreStatus> shards) {
+        public Entry(
+            String uuid,
+            Snapshot snapshot,
+            State state,
+            boolean silent,
+            List<String> indices,
+            Map<ShardId, ShardRestoreStatus> shards
+        ) {
             this.snapshot = Objects.requireNonNull(snapshot);
             this.state = Objects.requireNonNull(state);
+            this.silent = Objects.requireNonNull(silent);
             this.indices = Objects.requireNonNull(indices);
             if (shards == null) {
                 this.shards = Map.of();
@@ -342,10 +358,18 @@ public RestoreInProgress(StreamInput in) throws IOException {
             uuid = in.readString();
             Snapshot snapshot = new Snapshot(in);
             State state = State.fromValue(in.readByte());
+            boolean silent = in.readBoolean();
             List<String> indices = in.readImmutableList(StreamInput::readString);
             entriesBuilder.put(
                 uuid,
-                new Entry(uuid, snapshot, state, indices, in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus))
+                new Entry(
+                    uuid,
+                    snapshot,
+                    state,
+                    silent,
+                    indices,
+                    in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus)
+                )
             );
         }
         this.entries = Collections.unmodifiableMap(entriesBuilder);
@@ -357,6 +381,7 @@ public void writeTo(StreamOutput out) throws IOException {
             o.writeString(entry.uuid);
             entry.snapshot().writeTo(o);
             o.writeByte(entry.state().value());
+            o.writeBoolean(entry.silent());
             o.writeStringCollection(entry.indices);
             o.writeMap(entry.shards);
         });
@@ -383,6 +408,7 @@ public static void toXContent(Entry entry, XContentBuilder builder) throws IOExc
         builder.field("snapshot", entry.snapshot().getSnapshotId().getName());
         builder.field("repository", entry.snapshot().getRepository());
         builder.field("state", entry.state());
+        builder.field("silent", entry.silent());
         builder.startArray("indices");
         {
             for (String index : entry.indices()) {

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -238,7 +238,6 @@ public void restoreSnapshot(
         final BiConsumer<ClusterState, Metadata.Builder> updater
     ) {
         try {
-
             // Try and fill in any missing repository UUIDs in case they're needed during the restore
             final StepListener<Void> repositoryUuidRefreshListener = new StepListener<>();
             refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, repositoryUuidRefreshListener);
@@ -737,6 +736,7 @@ public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInPr
                         entry.uuid(),
                         entry.snapshot(),
                         overallState(RestoreInProgress.State.STARTED, shards),
+                        entry.silent(),
                         entry.indices(),
                         shards
                     )
@@ -873,7 +873,9 @@ public RestoreInProgress applyChanges(final RestoreInProgress oldRestore) {
 
                         Map<ShardId, ShardRestoreStatus> shards = Map.copyOf(shardsBuilder);
                         RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards);
-                        builder.add(new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.indices(), shards));
+                        builder.add(
+                            new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.silent(), entry.indices(), shards)
+                        );
                     } else {
                         builder.add(entry);
                     }
@@ -1051,6 +1053,7 @@ public ClusterState execute(ClusterState currentState) {
                 boolean changed = false;
                 for (RestoreInProgress.Entry entry : currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)) {
                     if (entry.state().completed()) {
+                        logger.log(entry.silent() ? Level.DEBUG : Level.INFO, "restore of snapshot [{}] completed", entry.snapshot());
                         changed = true;
                     } else {
                         restoreInProgressBuilder.add(entry);
@@ -1373,6 +1376,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) {
                             restoreUUID,
                             snapshot,
                             overallState(RestoreInProgress.State.INIT, shards),
+                            request.silent(),
                             List.copyOf(indicesToRestore.keySet()),
                             Map.copyOf(shards)
                         )
@@ -1569,6 +1573,12 @@ public void onFailure(Exception e) {
 
         @Override
         public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
+            logger.log(
+                request.silent() ? Level.DEBUG : Level.INFO,
+                "restore of snapshot [{}] started for indices {}",
+                snapshot,
+                snapshotInfo.indices()
+            );
             listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo));
         }
     }

server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java

@@ -64,6 +64,7 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
         }
         instance.partial(randomBoolean());
         instance.includeAliases(randomBoolean());
+        instance.silent(randomBoolean());
 
         if (randomBoolean()) {
             Map<String, Object> indexSettings = new HashMap<>();

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java

@@ -344,6 +344,7 @@ private static ClusterState addRestoredIndex(final String index, final int numSh
             "_uuid",
             snapshot,
             RestoreInProgress.State.INIT,
+            false,
             Collections.singletonList(index),
             shardsBuilder.build()
         );

server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java

@@ -414,6 +414,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations(
                 restoreUUID,
                 snapshot,
                 RestoreInProgress.State.INIT,
+                false,
                 new ArrayList<>(snapshotIndices),
                 restoreShards.build()
             );

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java

@@ -1199,7 +1199,7 @@ public void testDiskThresholdWithSnapshotShardSizes() {
         shards.put(shardId, new RestoreInProgress.ShardRestoreStatus("node1"));
 
         final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add(
-            new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, List.of("test"), shards.build())
+            new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, false, List.of("test"), shards.build())
         );
 
         ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName()))

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java

@@ -148,6 +148,7 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() {
             recoverySource.restoreUUID(),
             snapshot,
             restoreState,
+            false,
             singletonList("test"),
             shards.build()
         );

server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java

@@ -137,6 +137,7 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception {
                         UUIDs.randomBase64UUID(),
                         new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())),
                         RestoreInProgress.State.STARTED,
+                        false,
                         Collections.singletonList("index_name"),
                         ImmutableOpenMap.of()
                     )

server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java

@@ -451,6 +451,7 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String
                 recoverySource.restoreUUID(),
                 recoverySource.snapshot(),
                 RestoreInProgress.State.INIT,
+                false,
                 Collections.singletonList(indexName),
                 shards.build()
             )

x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java

@@ -158,7 +158,8 @@ public void testThatRepositoryRecoversEmptyIndexBasedOnLeaderSettings() throws I
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
             .masterNodeTimeout(TimeValue.MAX_VALUE)
-            .indexSettings(settingsBuilder);
+            .indexSettings(settingsBuilder)
+            .silent(true);
 
         PlainActionFuture<RestoreInfo> future = PlainActionFuture.newFuture();
         restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future));
@@ -230,7 +231,8 @@ public void testDocsAreRecovered() throws Exception {
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
             .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS))
-            .indexSettings(settingsBuilder);
+            .indexSettings(settingsBuilder)
+            .silent(true);
 
         PlainActionFuture<RestoreInfo> future = PlainActionFuture.newFuture();
         restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future));
@@ -298,7 +300,8 @@ public void testRateLimitingIsEmployed() throws Exception {
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
             .masterNodeTimeout(TimeValue.MAX_VALUE)
-            .indexSettings(settingsBuilder);
+            .indexSettings(settingsBuilder)
+            .silent(true);
 
         PlainActionFuture<RestoreInfo> future = PlainActionFuture.newFuture();
         restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future));
@@ -364,7 +367,8 @@ public void testIndividualActionsTimeout() throws Exception {
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
             .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS))
-            .indexSettings(settingsBuilder);
+            .indexSettings(settingsBuilder)
+            .silent(true);
 
         try {
             final RestoreService restoreService = getFollowerCluster().getCurrentMasterNodeInstance(RestoreService.class);
@@ -427,7 +431,8 @@ public void testFollowerMappingIsUpdated() throws IOException {
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
             .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS))
-            .indexSettings(settingsBuilder);
+            .indexSettings(settingsBuilder)
+            .silent(true);
 
         List<MockTransportService> transportServices = new ArrayList<>();
         CountDownLatch latch = new CountDownLatch(1);
@@ -586,7 +591,8 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats()
                 Settings.builder()
                     .put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, followerIndex)
                     .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)
-            );
+            )
+            .silent(true);
         restoreService.restoreSnapshot(restoreRequest, PlainActionFuture.newFuture());
 
         waitForRestoreInProgress.get(30L, TimeUnit.SECONDS);

x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java

@@ -157,7 +157,8 @@ private RestoreSnapshotRequest setUpRestoreSnapshotRequest(
             .indicesOptions(indicesOptions)
             .renamePattern("^(.*)$")
             .renameReplacement(followerIndex)
-            .masterNodeTimeout(TimeValue.MAX_VALUE);
+            .masterNodeTimeout(TimeValue.MAX_VALUE)
+            .silent(true);
     }
 
     public void testRetentionLeaseIsTakenAtTheStartOfRecovery() throws Exception {

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java

@@ -194,7 +194,8 @@ private void createFollowerIndex(
             .renamePattern("^(.*)$")
             .renameReplacement(request.getFollowerIndex())
             .masterNodeTimeout(request.masterNodeTimeout())
-            .indexSettings(overrideSettings);
+            .indexSettings(overrideSettings)
+            .silent(true);
 
         final Client clientWithHeaders = CcrLicenseChecker.wrapClient(
             this.client,

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java

@@ -272,7 +272,9 @@ protected void masterOperation(
                         // Pass through the master-node timeout
                         .masterNodeTimeout(request.masterNodeTimeout())
                         // Fail the restore if the snapshot found above is swapped out from under us before the restore happens
-                        .snapshotUuid(snapshotId.getUUID()),
+                        .snapshotUuid(snapshotId.getUUID())
+                        // Log snapshot restore at a lower level
+                        .silent(true),
                     listener
                 );
         }, listener::onFailure), threadPool.executor(ThreadPool.Names.SNAPSHOT_META), null);