Take a snapshot for the policy when the SLM policy is triggered

This commit fills in `SnapshotLifecycleTask` to actually perform the
snapshotting when the policy is triggered. Currently there is no handling of the
results (other than logging) as that will be added in subsequent work.

This also adds unit tests and an integration test that schedules a policy and
ensures that a snapshot is correctly taken.

Relates to elastic#38461

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@@ -34,7 +34,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -391,8 +390,8 @@ public CreateSnapshotRequest source(Map<String, Object> source) {
             if (name.equals("indices")) {
                 if (entry.getValue() instanceof String) {
                     indices(Strings.splitStringByCommaToArray((String) entry.getValue()));
-                } else if (entry.getValue() instanceof ArrayList) {
-                    indices((ArrayList<String>) entry.getValue());
+                } else if (entry.getValue() instanceof List) {
+                    indices((List<String>) entry.getValue());
                 } else {
                     throw new IllegalArgumentException("malformed indices section, should be an array of strings");
                 }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -541,7 +541,7 @@ boolean isPatternMatchingAllIndices(MetaData metaData, String[] indicesOrAliases
         return false;
     }
 
-    static final class Context {
+    public static class Context {
 
         private final ClusterState state;
         private final IndicesOptions options;
@@ -561,7 +561,7 @@ static final class Context {
            this(state, options, startTime, false, false);
         }
 
-        Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) {
+        protected Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex) {
             this.state = state;
             this.options = options;
             this.startTime = startTime;
@@ -817,7 +817,7 @@ private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options
         }
     }
 
-    static final class DateMathExpressionResolver implements ExpressionResolver {
+    public static final class DateMathExpressionResolver implements ExpressionResolver {
 
         private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
         private static final String EXPRESSION_LEFT_BOUND = "<";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -57,9 +57,9 @@
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
-import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
 import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
 import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
 import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction;
@@ -186,6 +186,10 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
+import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecycleMetadata;
+import org.elasticsearch.xpack.core.snapshotlifecycle.action.DeleteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.snapshotlifecycle.action.GetSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.snapshotlifecycle.action.PutSnapshotLifecycleAction;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -363,6 +367,10 @@ public List<Action<? extends ActionResponse>> getClientActions() {
                 RemoveIndexLifecyclePolicyAction.INSTANCE,
                 MoveToStepAction.INSTANCE,
                 RetryAction.INSTANCE,
+                PutSnapshotLifecycleAction.INSTANCE,
+                GetSnapshotLifecycleAction.INSTANCE,
+                DeleteSnapshotLifecycleAction.INSTANCE,
+                // Freeze
                 TransportFreezeIndexAction.FreezeIndexAction.INSTANCE
         );
     }
@@ -431,6 +439,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
                     IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new),
+                new NamedWriteableRegistry.Entry(MetaData.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new),
+                new NamedWriteableRegistry.Entry(NamedDiff.class, SnapshotLifecycleMetadata.TYPE,
+                    SnapshotLifecycleMetadata.SnapshotLifecycleMetadataDiff::new),
                 // ILM - LifecycleTypes
                 new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
                     (in) -> TimeseriesLifecycleType.INSTANCE),

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecycleMetadata.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecycleMetadata.java

@@ -4,26 +4,31 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle;
+package org.elasticsearch.xpack.core.snapshotlifecycle;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.XPackPlugin.XPackMetaDataCustom;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Custom cluster state metadata that stores all the snapshot lifecycle
@@ -32,6 +37,20 @@
 public class SnapshotLifecycleMetadata implements XPackMetaDataCustom {
 
     public static final String TYPE = "snapshot_lifecycle";
+    public static final ParseField POLICIES_FIELD = new ParseField("policies");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<SnapshotLifecycleMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE,
+        a -> new SnapshotLifecycleMetadata(
+            ((List<SnapshotLifecyclePolicyMetadata>) a[0]).stream()
+                .collect(Collectors.toMap(m -> m.getPolicy().getId(), Function.identity()))));
+
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> SnapshotLifecyclePolicyMetadata.parse(p, n),
+            v -> {
+                throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported");
+            }, POLICIES_FIELD);
+    }
 
     private final Map<String, SnapshotLifecyclePolicyMetadata> snapshotConfigurations;
 
@@ -75,7 +94,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("policies", this.snapshotConfigurations);
+        builder.field(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations);
         return builder;
     }
 
@@ -93,6 +112,12 @@ public static class SnapshotLifecycleMetadataDiff implements NamedDiff<MetaData.
                 DiffableUtils.getStringKeySerializer());
         }
 
+        public SnapshotLifecycleMetadataDiff(StreamInput in) throws IOException {
+            this.lifecycles = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                SnapshotLifecyclePolicyMetadata::new,
+                SnapshotLifecycleMetadataDiff::readLifecyclePolicyDiffFrom);
+        }
+
         @Override
         public MetaData.Custom apply(MetaData.Custom part) {
             TreeMap<String, SnapshotLifecyclePolicyMetadata> newLifecycles = new TreeMap<>(
@@ -110,8 +135,8 @@ public void writeTo(StreamOutput out) throws IOException {
             lifecycles.writeTo(out);
         }
 
-        static Diff<SnapshotLifecyclePolicy> readLifecyclePolicyDiffFrom(StreamInput in) throws IOException {
-            return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicy::new, in);
+        static Diff<SnapshotLifecyclePolicyMetadata> readLifecyclePolicyDiffFrom(StreamInput in) throws IOException {
+            return AbstractDiffable.readDiffFrom(SnapshotLifecyclePolicyMetadata::new, in);
         }
     }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicy.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicy.java

@@ -4,13 +4,18 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle;
+package org.elasticsearch.xpack.core.snapshotlifecycle;
 
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -21,6 +26,9 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -42,6 +50,8 @@ public class SnapshotLifecyclePolicy extends AbstractDiffable<SnapshotLifecycleP
     private static final ParseField SCHEDULE = new ParseField("schedule");
     private static final ParseField REPOSITORY = new ParseField("repository");
     private static final ParseField CONFIG = new ParseField("config");
+    private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_RESOLVER =
+        new IndexNameExpressionResolver.DateMathExpressionResolver();
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<SnapshotLifecyclePolicy, String> PARSER =
@@ -103,8 +113,29 @@ public ValidationException validate() {
         return null;
     }
 
+    /**
+     * Since snapshots need to be uniquely named, this method will resolve any date math used in
+     * the provided name, as well as appending a unique identifier so expressions that may overlap
+     * still result in unique snapshot names.
+     */
+    public String generateSnapshotName(Context context) {
+        List<String> candidates = DATE_MATH_RESOLVER.resolve(context, Collections.singletonList(this.name));
+        if (candidates.size() != 1) {
+            throw new IllegalStateException("resolving snapshot name " + this.name + " generated more than one candidate: " + candidates);
+        }
+        // TODO: we are breaking the rules of UUIDs by lowercasing this here, find an alternative (snapshot names must be lowercase)
+        return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
+    }
+
+    /**
+     * Generate a new create snapshot request from this policy. The name of the snapshot is
+     * generated at this time based on any date math expressions in the "name" field.
+     */
     public CreateSnapshotRequest toRequest() {
-        throw new UnsupportedOperationException("implement me");
+        CreateSnapshotRequest req = new CreateSnapshotRequest(repository, generateSnapshotName(new ResolverContext()));
+        req.source(configuration);
+        req.waitForCompletion(false);
+        return req;
     }
 
     public static SnapshotLifecyclePolicy parse(XContentParser parser, String id) {
@@ -157,4 +188,29 @@ public boolean equals(Object obj) {
     public String toString() {
         return Strings.toString(this);
     }
+
+    /**
+     * This is a context for the DateMathExpressionResolver, which does not require
+     * {@code IndicesOptions} or {@code ClusterState} since it only uses the start
+     * time to resolve expressions
+     */
+    public static final class ResolverContext extends Context {
+        public ResolverContext() {
+            this(System.currentTimeMillis());
+        }
+
+        public ResolverContext(long startTime) {
+            super(null, null, startTime, false, false);
+        }
+
+        @Override
+        public ClusterState getState() {
+            throw new UnsupportedOperationException("should never be called");
+        }
+
+        @Override
+        public IndicesOptions getOptions() {
+            throw new UnsupportedOperationException("should never be called");
+        }
+    }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadata.java

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle;
+package org.elasticsearch.xpack.core.snapshotlifecycle;
 
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diffable;
@@ -59,6 +59,10 @@ public class SnapshotLifecyclePolicyMetadata extends AbstractDiffable<SnapshotLi
         PARSER.declareString(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_STRING);
     }
 
+    public static SnapshotLifecyclePolicyMetadata parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
     public SnapshotLifecyclePolicyMetadata(SnapshotLifecyclePolicy policy, Map<String, String> headers, long version, long modifiedDate) {
         this.policy = policy;
         this.headers = headers;

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/DeleteSnapshotLifecycleAction.java

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle.action;
+package org.elasticsearch.xpack.core.snapshotlifecycle.action;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -26,19 +26,18 @@ protected DeleteSnapshotLifecycleAction() {
     }
 
     @Override
-    public DeleteSnapshotLifecycleAction.Response newResponse() {
+    public Response newResponse() {
         return new Response();
     }
 
-    public static class Request extends AcknowledgedRequest<DeleteSnapshotLifecycleAction.Request> {
+    public static class Request extends AcknowledgedRequest<Request> {
 
         private String lifecycleId;
 
-        Request(String lifecycleId) {
-            this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null");
-        }
+        public Request() { }
 
-        Request() {
+        public Request(String lifecycleId) {
+            this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null");
         }
 
         public String getLifecycleId() {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/GetSnapshotLifecycleAction.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/GetSnapshotLifecycleAction.java

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle.action;
+package org.elasticsearch.xpack.core.snapshotlifecycle.action;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -17,8 +17,8 @@
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicy;
-import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicyMetadata;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -47,11 +47,11 @@ public static class Request extends AcknowledgedRequest<GetSnapshotLifecycleActi
 
         private String[] lifecycleIds;
 
-        Request(String... lifecycleIds) {
+        public Request(String... lifecycleIds) {
             this.lifecycleIds = Objects.requireNonNull(lifecycleIds, "ids may not be null");
         }
 
-        Request() {
+        public Request() {
             this.lifecycleIds = Strings.EMPTY_ARRAY;
         }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/snapshotlifecycle/action/PutSnapshotLifecycleAction.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/snapshotlifecycle/action/PutSnapshotLifecycleAction.java

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.snapshotlifecycle.action;
+package org.elasticsearch.xpack.core.snapshotlifecycle.action;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.snapshotlifecycle.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.snapshotlifecycle.SnapshotLifecyclePolicy;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -45,12 +45,12 @@ public static class Request extends AcknowledgedRequest<Request> implements ToXC
         private String lifecycleId;
         private SnapshotLifecyclePolicy lifecycle;
 
-        Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) {
+        public Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) {
             this.lifecycleId = lifecycleId;
             this.lifecycle = lifecycle;
         }
 
-        Request() { }
+        public Request() { }
 
         public String getLifecycleId() {
             return this.lifecycleId;