Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolve date math expressions before looking up index metadata #75314

Merged
merged 3 commits into from
Jul 15, 2021
Merged

Resolve date math expressions before looking up index metadata #75314

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d663462
Move ResolverContext class to permit reuse
danhermann Jul 13, 2021
52ada9e
resolve date math expressions before retrieving index metadata
danhermann Jul 13, 2021
b934eb4
use single time for unit test
danhermann Jul 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1340,4 +1340,28 @@ static String resolveExpression(String expression, final Context context) {
return beforePlaceHolderSb.toString();
}
}

/**
* This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
* since it uses only the start time to resolve expressions.
*/
public static final class ResolverContext extends Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}
}
15 changes: 13 additions & 2 deletions server/src/main/java/org/elasticsearch/ingest/IngestService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.elasticsearch.cluster.ClusterStateApplier;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
Expand Down Expand Up @@ -74,6 +75,8 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
public static final String INGEST_ORIGIN = "ingest";

private static final Logger logger = LogManager.getLogger(IngestService.class);
private static final IndexNameExpressionResolver.DateMathExpressionResolver DATE_MATH_EXPRESSION_RESOLVER =
new IndexNameExpressionResolver.DateMathExpressionResolver();

private final ClusterService clusterService;
private final ScriptService scriptService;
Expand Down Expand Up @@ -132,7 +135,7 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
IndexMetadata indexMetadata = null;
// start to look for default or final pipelines via settings found in the index meta data
if (originalRequest != null) {
indexMetadata = metadata.indices().get(originalRequest.index());
indexMetadata = metadata.indices().get(resolveIndexName(originalRequest.index()));
}
// check the alias for the index request (this is how normal index requests are modeled)
if (indexMetadata == null && indexRequest.index() != null) {
Expand Down Expand Up @@ -218,12 +221,20 @@ public static boolean resolvePipelines(final DocWriteRequest<?> originalRequest,
indexRequest.isPipelineResolved(true);
}


// return whether this index request has a pipeline
return NOOP_PIPELINE_NAME.equals(indexRequest.getPipeline()) == false
|| NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false;
}

private static String resolveIndexName(String unresolvedIndexName) {
List<String> resolvedNames = DATE_MATH_EXPRESSION_RESOLVER.resolve(
new IndexNameExpressionResolver.ResolverContext(),
List.of(unresolvedIndexName)
);
assert resolvedNames.size() == 1;
return resolvedNames.get(0);
}

public ClusterService getClusterService() {
return clusterService;
}
Expand Down
18 changes: 18 additions & 0 deletions server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
Expand Down Expand Up @@ -1376,6 +1377,23 @@ public void testResolveFinalPipeline() {
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveFinalPipelineWithDateMathExpression() {
final DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
IndexMetadata.Builder builder = IndexMetadata.builder("idx-" + dateFormatter.formatMillis(System.currentTimeMillis()))
.settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline"))
.numberOfShards(1)
.numberOfReplicas(0);
Metadata metadata = Metadata.builder().put(builder).build();

// index name matches with IDM:
IndexRequest indexRequest = new IndexRequest("<idx-{now/d}>");
boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this will very rarely fail when the first call to System.currentTimeMillis() on line 1382 is called on one day, then this is called on the next day (across the day delineation I mean).

Is it worth adding a parameter that allows overriding the time used when resolving for this test? (I only say since we've had failures related to this with data streams and dates in index names)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, worth it. Added in b934eb4

assertThat(result, is(true));
assertThat(indexRequest.isPipelineResolved(), is(true));
assertThat(indexRequest.getPipeline(), equalTo("_none"));
assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
}

public void testResolveRequestOrDefaultPipelineAndFinalPipeline() {
// no pipeline:
{
Expand Down
29 changes: 1 addition & 28 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
Expand All @@ -18,7 +17,6 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.index.Index;
import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;

import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -117,7 +115,7 @@ public boolean equals(Object obj) {
* still result in unique snapshot names.
*/
public static String generateSnapshotName(String name) {
return generateSnapshotName(name, new ResolverContext());
return generateSnapshotName(name, new IndexNameExpressionResolver.ResolverContext());
}

public static String generateSnapshotName(String name, IndexNameExpressionResolver.Context context) {
Expand All @@ -129,31 +127,6 @@ public static String generateSnapshotName(String name, IndexNameExpressionResolv
return candidates.get(0) + "-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
}

/**
* This is a context for the DateMathExpressionResolver, which does not require
* {@code IndicesOptions} or {@code ClusterState} since it only uses the start
* time to resolve expressions
*/
public static final class ResolverContext extends IndexNameExpressionResolver.Context {
public ResolverContext() {
this(System.currentTimeMillis());
}

public ResolverContext(long startTime) {
super(null, null, startTime, false, false, false, false, SystemIndexAccessLevel.ALL, name -> false, name -> false);
}

@Override
public ClusterState getState() {
throw new UnsupportedOperationException("should never be called");
}

@Override
public IndicesOptions getOptions() {
throw new UnsupportedOperationException("should never be called");
}
}

@Nullable
public static ActionRequestValidationException validateGeneratedSnapshotName(String snapshotPrefix, String snapshotName) {
ActionRequestValidationException err = new ActionRequestValidationException();
Expand Down
3 changes: 2 additions & 1 deletion ...in/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStepTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.Strings;

Expand Down Expand Up @@ -86,7 +87,7 @@ public void testNameGeneration() {
assertThat(generateSnapshotName("name"), startsWith("name-"));
assertThat(generateSnapshotName("name").length(), greaterThan("name-".length()));

GenerateSnapshotNameStep.ResolverContext resolverContext = new GenerateSnapshotNameStep.ResolverContext(time);
IndexNameExpressionResolver.ResolverContext resolverContext = new IndexNameExpressionResolver.ResolverContext(time);
assertThat(generateSnapshotName("<name-{now}>", resolverContext), startsWith("name-2019.03.15-"));
assertThat(generateSnapshotName("<name-{now}>", resolverContext).length(), greaterThan("name-2019.03.15-".length()));

Expand Down