Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolve date math expressions before looking up index metadata #75314

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1340,4 +1340,28 @@ static String resolveExpression(String expression, final Context context) {
return beforePlaceHolderSb.toString();
}
}

/**
* This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
* since it uses only the start time to resolve expressions.
*/
public static final class ResolverContext extends Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}
}
22 changes: 19 additions & 3 deletions server/src/main/java/org/elasticsearch/ingest/IngestService.java
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.elasticsearch.cluster.ClusterStateApplier;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -74,6 +75,8 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
public static final String INGEST_ORIGIN = "ingest";

private static final Logger logger = LogManager.getLogger(IngestService.class);
private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_EXPRESSION_RESOLVER =
new IndexNameExpressionResolver.DateMathExpressionResolver();

private final ClusterService clusterService;
private final ScriptService scriptService;
@@ -122,7 +125,12 @@ private static Map<String, Processor.Factory> processorFactories(List<IngestPlug
}

public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final IndexRequest indexRequest,
final Metadata metadata) {
final Metadata metadata) {
return resolvePipelines(originalRequest, indexRequest, metadata, System.currentTimeMillis());
}

public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final IndexRequest indexRequest,
final Metadata metadata, final long epochMillis) {
if (indexRequest.isPipelineResolved() == false) {
final String requestPipeline = indexRequest.getPipeline();
indexRequest.setPipeline(NOOP_PIPELINE_NAME);
@@ -132,7 +140,7 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
IndexMetadata indexMetadata = null;
// start to look for default or final pipelines via settings found in the index meta data
if (originalRequest != null) {
indexMetadata = metadata.indices().get(originalRequest.index());
indexMetadata = metadata.indices().get(resolveIndexName(originalRequest.index(), epochMillis));
}
// check the alias for the index request (this is how normal index requests are modeled)
if (indexMetadata == null && indexRequest.index() != null) {
@@ -218,12 +226,20 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
indexRequest.isPipelineResolved(true);
}


// return whether this index request has a pipeline
return NOOP_PIPELINE_NAME.equals(indexRequest.getPipeline()) == false
|| NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false;
}

private static String resolveIndexName(final String unresolvedIndexName, final long epochMillis) {
List<String> resolvedNames = DATE_MATH_EXPRESSION_RESOLVER.resolve(
new IndexNameExpressionResolver.ResolverContext(epochMillis),
List.of(unresolvedIndexName)
);
assert resolvedNames.size() == 1;
return resolvedNames.get(0);
}

public ClusterService getClusterService() {
return clusterService;
}
Original file line number Diff line number Diff line change
@@ -37,6 +37,7 @@
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
@@ -1376,6 +1377,24 @@ public void testResolveFinalPipeline() {
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveFinalPipelineWithDateMathExpression() {
final long epochMillis = randomLongBetween(1, System.currentTimeMillis());
final DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
IndexMetadata.Builder builder = IndexMetadata.builder("idx-" + dateFormatter.formatMillis(epochMillis))
.settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline"))
.numberOfShards(1)
.numberOfReplicas(0);
Metadata metadata = Metadata.builder().put(builder).build();

// index name matches with IDM:
IndexRequest indexRequest = new IndexRequest("<idx-{now/d}>");
boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata, epochMillis);
assertThat(result, is(true));
assertThat(indexRequest.isPipelineResolved(), is(true));
assertThat(indexRequest.getPipeline(), equalTo("_none"));
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveRequestOrDefaultPipelineAndFinalPipeline() {
// no pipeline:
{
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -18,7 +17,6 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.index.Index;
import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;

import java.util.Collections;
import java.util.List;
@@ -117,7 +115,7 @@ public boolean equals(Object obj) {
* still result in unique snapshot names.
*/
public static String generateSnapshotName(String name) {
return generateSnapshotName(name, new ResolverContext());
return generateSnapshotName(name, new IndexNameExpressionResolver.ResolverContext());
}

public static String generateSnapshotName(String name, IndexNameExpressionResolver.Context context) {
@@ -129,31 +127,6 @@ public static String generateSnapshotName(String name, IndexNameExpressionResolv
return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
}

/**
* This is a context for the DateMathExpressionResolver, which does not require
* {@code IndicesOptions} or {@code ClusterState} since it only uses the start
* time to resolve expressions
*/
public static final class ResolverContext extends IndexNameExpressionResolver.Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}

@Nullable
public static ActionRequestValidationException validateGeneratedSnapshotName(String snapshotPrefix, String snapshotName) {
ActionRequestValidationException err = new ActionRequestValidationException();
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.Strings;

@@ -86,7 +87,7 @@ public void testNameGeneration() {
assertThat(generateSnapshotName("name"), startsWith("name-"));
assertThat(generateSnapshotName("name").length(), greaterThan("name-".length()));

GenerateSnapshotNameStep.ResolverContext resolverContext = new GenerateSnapshotNameStep.ResolverContext(time);
IndexNameExpressionResolver.ResolverContext resolverContext = new IndexNameExpressionResolver.ResolverContext(time);
assertThat(generateSnapshotName("<name-{now}>", resolverContext), startsWith("name-2019.03.15-"));
assertThat(generateSnapshotName("<name-{now}>", resolverContext).length(), greaterThan("name-2019.03.15-".length()));