Skip to content

Commit

Permalink
[7.14] Resolve date math expressions before looking up index metadata (…
Browse files Browse the repository at this point in the history 
…#75314) (#75477)
  • Loading branch information
@danhermann
danhermann authored Jul 19, 2021
1 parent d923b0d commit 0b3dad9
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 32 deletions.
24 changes: 24 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1340,4 +1340,28 @@ static String resolveExpression(String expression, final Context context) {
return beforePlaceHolderSb.toString();
}
}

/**
* This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
* since it uses only the start time to resolve expressions.
*/
public static final class ResolverContext extends Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}
}
22 changes: 19 additions & 3 deletions server/src/main/java/org/elasticsearch/ingest/IngestService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.elasticsearch.cluster.ClusterStateApplier;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
Expand Down Expand Up @@ -75,6 +76,8 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
public static final String INGEST_ORIGIN = "ingest";

private static final Logger logger = LogManager.getLogger(IngestService.class);
private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_EXPRESSION_RESOLVER =
new IndexNameExpressionResolver.DateMathExpressionResolver();

private final ClusterService clusterService;
private final ScriptService scriptService;
Expand Down Expand Up @@ -123,7 +126,12 @@ private static Map<String, Processor.Factory> processorFactories(List<IngestPlug
}

public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final IndexRequest indexRequest,
final Metadata metadata) {
final Metadata metadata) {
return resolvePipelines(originalRequest, indexRequest, metadata, System.currentTimeMillis());
}

public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final IndexRequest indexRequest,
final Metadata metadata, final long epochMillis) {
if (indexRequest.isPipelineResolved() == false) {
final String requestPipeline = indexRequest.getPipeline();
indexRequest.setPipeline(NOOP_PIPELINE_NAME);
Expand All @@ -133,7 +141,7 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
IndexMetadata indexMetadata = null;
// start to look for default or final pipelines via settings found in the index meta data
if (originalRequest != null) {
indexMetadata = metadata.indices().get(originalRequest.index());
indexMetadata = metadata.indices().get(resolveIndexName(originalRequest.index(), epochMillis));
}
// check the alias for the index request (this is how normal index requests are modeled)
if (indexMetadata == null && indexRequest.index() != null) {
Expand Down Expand Up @@ -219,12 +227,20 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
indexRequest.isPipelineResolved(true);
}


// return whether this index request has a pipeline
return NOOP_PIPELINE_NAME.equals(indexRequest.getPipeline()) == false
|| NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false;
}

private static String resolveIndexName(final String unresolvedIndexName, final long epochMillis) {
List<String> resolvedNames = DATE_MATH_EXPRESSION_RESOLVER.resolve(
new IndexNameExpressionResolver.ResolverContext(epochMillis),
org.elasticsearch.core.List.of(unresolvedIndexName)
);
assert resolvedNames.size() == 1;
return resolvedNames.get(0);
}

public ClusterService getClusterService() {
return clusterService;
}
Expand Down
19 changes: 19 additions & 0 deletions server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
Expand Down Expand Up @@ -1399,6 +1400,24 @@ public void testResolveFinalPipeline() {
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveFinalPipelineWithDateMathExpression() {
final long epochMillis = randomLongBetween(1, System.currentTimeMillis());
final DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
IndexMetadata.Builder builder = IndexMetadata.builder("idx-" + dateFormatter.formatMillis(epochMillis))
.settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline"))
.numberOfShards(1)
.numberOfReplicas(0);
Metadata metadata = Metadata.builder().put(builder).build();

// index name matches with IDM:
IndexRequest indexRequest = new IndexRequest("<idx-{now/d}>");
boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata, epochMillis);
assertThat(result, is(true));
assertThat(indexRequest.isPipelineResolved(), is(true));
assertThat(indexRequest.getPipeline(), equalTo("_none"));
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveRequestOrDefaultPipelineAndFinalPipeline() {
// no pipeline:
{
Expand Down
29 changes: 1 addition & 28 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
Expand All @@ -18,7 +17,6 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.index.Index;
import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;

import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -117,7 +115,7 @@ public boolean equals(Object obj) {
* still result in unique snapshot names.
*/
public static String generateSnapshotName(String name) {
return generateSnapshotName(name, new ResolverContext());
return generateSnapshotName(name, new IndexNameExpressionResolver.ResolverContext());
}

public static String generateSnapshotName(String name, IndexNameExpressionResolver.Context context) {
Expand All @@ -129,31 +127,6 @@ public static String generateSnapshotName(String name, IndexNameExpressionResolv
return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
}

/**
* This is a context for the DateMathExpressionResolver, which does not require
* {@code IndicesOptions} or {@code ClusterState} since it only uses the start
* time to resolve expressions
*/
public static final class ResolverContext extends IndexNameExpressionResolver.Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}

@Nullable
public static ActionRequestValidationException validateGeneratedSnapshotName(String snapshotPrefix, String snapshotName) {
ActionRequestValidationException err = new ActionRequestValidationException();
Expand Down
3 changes: 2 additions & 1 deletion ...in/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStepTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.Strings;

Expand Down Expand Up @@ -86,7 +87,7 @@ public void testNameGeneration() {
assertThat(generateSnapshotName("name"), startsWith("name-"));
assertThat(generateSnapshotName("name").length(), greaterThan("name-".length()));

GenerateSnapshotNameStep.ResolverContext resolverContext = new GenerateSnapshotNameStep.ResolverContext(time);
IndexNameExpressionResolver.ResolverContext resolverContext = new IndexNameExpressionResolver.ResolverContext(time);
assertThat(generateSnapshotName("<name-{now}>", resolverContext), startsWith("name-2019.03.15-"));
assertThat(generateSnapshotName("<name-{now}>", resolverContext).length(), greaterThan("name-2019.03.15-".length()));

Expand Down

0 comments on commit 0b3dad9

Please sign in to comment.