From 0d7a30630ca1c604ccf0404a9556895c3c91ba9e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 1 Dec 2020 09:57:19 +0000 Subject: [PATCH 01/27] Correctly handle mixed object paths in XContentMapValues (#65539) When we have an object that looks like this: ``` { "foo" : { "cat": "meow" }, "foo.bar" : "baz" ``` where the inner objects of foo are defined both as json objects and via dot notation, then XContentMapValues.extractValue() will only descend through the json object and will not collect values from the dot notated paths. In these cases, the foo.bar values will not be returned in the fields section of a search response. This commit adds the ability to check both paths, ensuring that all relevant values get returned as part of fields. Fixes #65499 --- .../xcontent/support/XContentMapValues.java | 54 ++++--- .../support/XContentMapValuesTests.java | 34 +++++ .../fetch/subphase/FieldFetcherTests.java | 144 +++++++++--------- 3 files changed, 134 insertions(+), 98 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index fa21a88fbee49..d5d71b8c3e470 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -60,25 +60,26 @@ private static void extractRawValues(List values, Map part, Stri } String key = pathElements[index]; - Object currentValue = part.get(key); + Object currentValue; int nextIndex = index + 1; - while (currentValue == null && nextIndex != pathElements.length) { - key += "." + pathElements[nextIndex]; + + while (true) { currentValue = part.get(key); + if (currentValue != null) { + if (currentValue instanceof Map) { + extractRawValues(values, (Map) currentValue, pathElements, nextIndex); + } else if (currentValue instanceof List) { + extractRawValues(values, (List) currentValue, pathElements, nextIndex); + } else { + values.add(currentValue); + } + } + if (nextIndex == pathElements.length) { + return; + } + key += "." + pathElements[nextIndex]; nextIndex++; } - - if (currentValue == null) { - return; - } - - if (currentValue instanceof Map) { - extractRawValues(values, (Map) currentValue, pathElements, nextIndex); - } else if (currentValue instanceof List) { - extractRawValues(values, (List) currentValue, pathElements, nextIndex); - } else { - values.add(currentValue); - } } @SuppressWarnings({"unchecked"}) @@ -163,19 +164,24 @@ private static Object extractValue(String[] pathElements, if (currentValue instanceof Map) { Map map = (Map) currentValue; String key = pathElements[index]; - Object mapValue = map.get(key); int nextIndex = index + 1; - while (mapValue == null && nextIndex != pathElements.length) { + while (true) { + if (map.containsKey(key)) { + Object mapValue = map.get(key); + if (mapValue == null) { + return nullValue; + } + Object val = extractValue(pathElements, nextIndex, mapValue, nullValue); + if (val != null) { + return val; + } + } + if (nextIndex == pathElements.length) { + return null; + } key += "." + pathElements[nextIndex]; - mapValue = map.get(key); nextIndex++; } - - if (map.containsKey(key) == false) { - return null; - } - - return extractValue(pathElements, nextIndex, mapValue, nullValue); } return null; } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index 4ab28aa9a80c5..d730372047fbf 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -195,6 +195,17 @@ public void testExtractValueWithNullValue() throws Exception { assertEquals("NULL", XContentMapValues.extractValue("object1.object2.field", map, "NULL")); } + public void testExtractValueMixedObjects() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .startObject("foo").field("cat", "meow").endObject() + .field("foo.bar", "baz") + .endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { + Map map = parser.map(); + assertThat(XContentMapValues.extractValue("foo.bar", map), equalTo("baz")); + } + } + public void testExtractRawValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test", "value") @@ -269,6 +280,29 @@ public void testExtractRawValueLeafOnly() throws IOException { assertThat(XContentMapValues.extractRawValues("path1.path2", map), Matchers.contains("value")); } + public void testExtractRawValueMixedObjects() throws IOException { + { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .startObject("foo").field("cat", "meow").endObject() + .field("foo.bar", "baz") + .endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { + Map map = parser.map(); + assertThat(XContentMapValues.extractRawValues("foo.bar", map), Matchers.contains("baz")); + } + } + { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .startObject("foo").field("bar", "meow").endObject() + .field("foo.bar", "baz") + .endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { + Map map = parser.map(); + assertThat(XContentMapValues.extractRawValues("foo.bar", map), Matchers.containsInAnyOrder("meow", "baz")); + } + } + } + public void testPrefixedNamesFilteringTest() { Map map = new HashMap<>(); map.put("obj", "value"); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index c50ab2acf3053..96dadaa8c71e6 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -21,17 +21,18 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SourceLookup; -import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.List; @@ -43,7 +44,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; -public class FieldFetcherTests extends ESSingleNodeTestCase { +public class FieldFetcherTests extends MapperServiceTestCase { public void testLeafValues() throws IOException { MapperService mapperService = createMapperService(); @@ -89,6 +90,24 @@ public void testObjectValues() throws IOException { assertThat(rangeField.getValue(), equalTo(Map.of("gte", 0.0f, "lte", 2.718f))); } + public void testMixedObjectValues() throws IOException { + MapperService mapperService = createMapperService(); + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .startObject("foo").field("cat", "meow").endObject() + .field("foo.bar", "baz") + .endObject(); + + ParsedDocument doc = mapperService.documentMapper().parse(source(Strings.toString(source))); + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + + Map fields = fetchFields(mapperService, source, "foo.bar"); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("foo.bar"); + assertThat(field.getValues().size(), equalTo(1)); + assertThat(field.getValue(), equalTo("baz")); + } + public void testNonExistentField() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() @@ -182,7 +201,7 @@ public void testArrayValueMappers() throws IOException { } public void testFieldNamesWithWildcard() throws IOException { - MapperService mapperService = createMapperService();; + MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "first", "second") .field("integer_field", 333) @@ -232,17 +251,10 @@ public void testDateFormat() throws IOException { } public void testIgnoreAbove() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", "keyword") - .field("ignore_above", 20) - .endObject() - .endObject() - .endObject(); - - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - MapperService mapperService = indexService.mapperService(); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "keyword"); + b.field("ignore_above", 20); + })); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "value", "other_value", "really_really_long_value") @@ -259,18 +271,15 @@ public void testIgnoreAbove() throws IOException { } public void testFieldAliases() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field").field("type", "keyword").endObject() - .startObject("alias_field") - .field("type", "alias") - .field("path", "field") - .endObject() - .endObject() - .endObject(); - - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - MapperService mapperService = indexService.mapperService(); + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("field").field("type", "keyword").endObject(); + b.startObject("alias_field"); + { + b.field("type", "alias"); + b.field("path", "field"); + } + b.endObject(); + })); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .field("field", "value") @@ -291,19 +300,14 @@ public void testFieldAliases() throws IOException { } public void testMultiFields() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", "integer") - .startObject("fields") - .startObject("keyword").field("type", "keyword").endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - MapperService mapperService = indexService.mapperService(); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "integer"); + b.startObject("fields"); + { + b.startObject("keyword").field("type", "keyword").endObject(); + } + b.endObject(); + })); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .field("field", 42) @@ -324,24 +328,22 @@ public void testMultiFields() throws IOException { } public void testCopyTo() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", "keyword") - .endObject() - .startObject("other_field") - .field("type", "integer") - .field("copy_to", "field") - .endObject() - .startObject("yet_another_field") - .field("type", "keyword") - .field("copy_to", "field") - .endObject() - .endObject() - .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - MapperService mapperService = indexService.mapperService(); + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("field").field("type", "keyword").endObject(); + b.startObject("other_field"); + { + b.field("type", "integer"); + b.field("copy_to", "field"); + } + b.endObject(); + b.startObject("yet_another_field"); + { + b.field("type", "keyword"); + b.field("copy_to", "field"); + } + b.endObject(); + })); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "one", "two", "three") @@ -358,7 +360,7 @@ public void testCopyTo() throws IOException { } public void testObjectFields() throws IOException { - MapperService mapperService = createMapperService();; + MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "first", "second") .startObject("object") @@ -371,18 +373,11 @@ public void testObjectFields() throws IOException { } public void testTextSubFields() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field") - .field("type", "text") - .startObject("index_prefixes").endObject() - .field("index_phrases", true) - .endObject() - .endObject() - .endObject(); - - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - MapperService mapperService = indexService.mapperService(); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "text"); + b.startObject("index_prefixes").endObject(); + b.field("index_phrases", true); + })); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "some text") @@ -411,12 +406,13 @@ private static Map fetchFields(MapperService mapperServic SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldFetcher fieldFetcher = FieldFetcher.create(createQueryShardContext(mapperService), null, fields); + FieldFetcher fieldFetcher = FieldFetcher.create(newQueryShardContext(mapperService), null, fields); return fieldFetcher.fetch(sourceLookup, Set.of()); } public MapperService createMapperService() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") .startObject("properties") .startObject("field").field("type", "keyword").endObject() .startObject("integer_field").field("type", "integer").endObject() @@ -430,13 +426,13 @@ public MapperService createMapperService() throws IOException { .endObject() .startObject("field_that_does_not_match").field("type", "keyword").endObject() .endObject() + .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, mapping); - return indexService.mapperService(); + return createMapperService(mapping); } - private static QueryShardContext createQueryShardContext(MapperService mapperService) { + private static QueryShardContext newQueryShardContext(MapperService mapperService) { Settings settings = Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) From 161463a1bd345f41b97c9113fbcaee720d25cc6d Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 1 Dec 2020 10:09:36 +0000 Subject: [PATCH 02/27] [ML] Remove unnecessary call to get datafeed stats in preview (#65660) The datafeed timing stats are not required in datafeed preview --- .../TransportPreviewDatafeedAction.java | 56 +++++++++---------- .../TransportPreviewDatafeedActionTests.java | 4 +- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java index 710d301897a65..2b30bb949ad8d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java @@ -20,13 +20,13 @@ import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction; import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import java.io.BufferedReader; import java.io.InputStream; @@ -44,21 +44,19 @@ public class TransportPreviewDatafeedAction extends HandledTransportAction { previewDatafeed.setHeaders(filterSecurityHeaders(threadPool.getThreadContext().getHeaders())); - jobResultsProvider.datafeedTimingStats( - jobBuilder.getId(), - timingStats -> { - // NB: this is using the client from the transport layer, NOT the internal client. - // This is important because it means the datafeed search will fail if the user - // requesting the preview doesn't have permission to search the relevant indices. - DataExtractorFactory.create( - client, - previewDatafeed.build(), - jobBuilder.build(), - xContentRegistry, - // Fake DatafeedTimingStatsReporter that does not have access to results index - new DatafeedTimingStatsReporter(timingStats, (ts, refreshPolicy) -> {}), - new ActionListener<>() { - @Override - public void onResponse(DataExtractorFactory dataExtractorFactory) { - DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); - threadPool.generic().execute(() -> previewDatafeed(dataExtractor, listener)); - } + // NB: this is using the client from the transport layer, NOT the internal client. + // This is important because it means the datafeed search will fail if the user + // requesting the preview doesn't have permission to search the relevant indices. + DataExtractorFactory.create( + client, + previewDatafeed.build(), + jobBuilder.build(), + xContentRegistry, + // Fake DatafeedTimingStatsReporter that does not have access to results index + new DatafeedTimingStatsReporter(new DatafeedTimingStats(datafeedConfig.getJobId()), (ts, refreshPolicy) -> { + }), + new ActionListener<>() { + @Override + public void onResponse(DataExtractorFactory dataExtractorFactory) { + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); + threadPool.generic().execute(() -> previewDatafeed(dataExtractor, listener)); + } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - }, - listener::onFailure); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); }); }, listener::onFailure)); @@ -143,7 +137,7 @@ static void previewDatafeed(DataExtractor dataExtractor, ActionListener() { @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { + public Void answer(InvocationOnMock invocationOnMock) { PreviewDatafeedAction.Response response = (PreviewDatafeedAction.Response) invocationOnMock.getArguments()[0]; capturedResponse = response.toString(); return null; @@ -61,7 +61,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { doAnswer(new Answer() { @Override - public Void answer(InvocationOnMock invocationOnMock) throws Throwable { + public Void answer(InvocationOnMock invocationOnMock) { capturedFailure = (Exception) invocationOnMock.getArguments()[0]; return null; } From a44f11d5962325799ed21983d288b56f92b56749 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 1 Dec 2020 11:28:16 +0100 Subject: [PATCH 03/27] Deduplicate Index Meta Generations when Deserializing (#65619) These strings are quite long individually and will be repeated potentially up to the number of snapshots in the repository times. Since these make up more than half of the size of the repository metadata and are likely the same for all snapshots the savings from deduplicating them can make up for more than half the size of `RepositoryData` easily in most real-world cases. --- .../java/org/elasticsearch/repositories/RepositoryData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index f8fa653255409..8198173d53b72 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -608,6 +608,7 @@ private static void parseSnapshots(XContentParser parser, Map snapshotVersions, Map> indexMetaLookup) throws IOException { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); + final Map stringDeduplicator = new HashMap<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { String name = null; String uuid = null; @@ -628,7 +629,7 @@ private static void parseSnapshots(XContentParser parser, Map stringDeduplicator.computeIfAbsent(p.text(), Function.identity())); break; case VERSION: version = Version.fromString(parser.text()); From c0ff8ecf20596720809dbed99af254c4a7f17d22 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 1 Dec 2020 14:35:09 +0200 Subject: [PATCH 04/27] [ML] Resume reassigning datafeed from loaded model snapshot (#65630) This fixes a long outstanding issue with the resume behaviour of a datafeed that has been reassigned on a node. When we resume a datafeed, we start from the latest result time or the latest record time, whichever is greater. This makes sense when a job had been closed gracefully previously. In the scenario when a job is being reassigned, it is highly likely that the job has seen data after the latest snapshot was persisted. This means that with the current behaviour, we load that snapshot and then we resume sending data from the point the job had reached previously. This results to the model having a knowledge gap. This commit addresses this by checking whether a reassining datafeed has gone past its job's model snapshot and reverting the job to that snapshot while deleting intervening results. Then we resume the datafeed from the latest result or latest record of the snapshot. Fixes #63400 --- .../test/InternalTestCluster.java | 19 ++- .../ml/action/RevertModelSnapshotAction.java | 29 +++- ...RevertModelSnapshotActionRequestTests.java | 3 + .../integration/MlDistributedFailureIT.java | 128 +++++++++++++++++- .../xpack/ml/MachineLearning.java | 11 +- .../TransportRevertModelSnapshotAction.java | 2 +- .../xpack/ml/datafeed/DatafeedContext.java | 73 ++++++++-- .../ml/datafeed/DatafeedContextProvider.java | 115 ++++++++++++++++ .../xpack/ml/datafeed/DatafeedManager.java | 56 +------- .../persistence/JobDataCountsPersister.java | 1 + .../ml/job/persistence/RestartTimeInfo.java | 18 +++ .../autodetect/AutodetectProcessManager.java | 2 +- .../task/OpenJobPersistentTasksExecutor.java | 49 ++++++- .../ml/datafeed/DatafeedJobBuilderTests.java | 48 +++---- .../ml/datafeed/DatafeedManagerTests.java | 53 +++----- .../OpenJobPersistentTasksExecutorTests.java | 37 +++-- 16 files changed, 489 insertions(+), 155 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextProvider.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 85419c6974389..88ed8b9ad1c29 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -36,14 +36,13 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; -import org.elasticsearch.cluster.coordination.NoMasterBlockService; -import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.coordination.ClusterBootstrapService; +import org.elasticsearch.cluster.coordination.NoMasterBlockService; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -82,6 +81,7 @@ import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineTestCase; @@ -126,6 +126,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.TreeMap; @@ -1524,6 +1525,20 @@ public synchronized boolean stopRandomDataNode() throws IOException { return false; } + /** + * Stops a specific node in the cluster. Returns true if the node was found to stop, false otherwise. + */ + public synchronized boolean stopNode(String nodeName) throws IOException { + ensureOpen(); + Optional nodeToStop = nodes.values().stream().filter(n -> n.getName().equals(nodeName)).findFirst(); + if (nodeToStop.isPresent()) { + logger.info("Closing node [{}]", nodeToStop.get().name); + stopNodesAndClient(nodeToStop.get()); + return true; + } + return false; + } + /** * Stops a random node in the cluster that applies to the given filter. Does nothing if none of the nodes match the * filter. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java index 60e5c3180ce81..8f60540659b9e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ml.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -39,12 +40,14 @@ public static class Request extends AcknowledgedRequest implements ToXC public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id"); public static final ParseField DELETE_INTERVENING = new ParseField("delete_intervening_results"); + private static final ParseField FORCE = new ParseField("force"); private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); static { PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); PARSER.declareString((request, snapshotId) -> request.snapshotId = snapshotId, SNAPSHOT_ID); PARSER.declareBoolean(Request::setDeleteInterveningResults, DELETE_INTERVENING); + PARSER.declareBoolean(Request::setForce, FORCE); } public static Request parseRequest(String jobId, String snapshotId, XContentParser parser) { @@ -61,6 +64,7 @@ public static Request parseRequest(String jobId, String snapshotId, XContentPars private String jobId; private String snapshotId; private boolean deleteInterveningResults; + private boolean force; public Request() { } @@ -70,6 +74,11 @@ public Request(StreamInput in) throws IOException { jobId = in.readString(); snapshotId = in.readString(); deleteInterveningResults = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + force = in.readBoolean(); + } else { + force = false; + } } public Request(String jobId, String snapshotId) { @@ -93,6 +102,14 @@ public void setDeleteInterveningResults(boolean deleteInterveningResults) { this.deleteInterveningResults = deleteInterveningResults; } + public boolean isForce() { + return force; + } + + public void setForce(boolean force) { + this.force = force; + } + @Override public ActionRequestValidationException validate() { return null; @@ -104,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); out.writeString(snapshotId); out.writeBoolean(deleteInterveningResults); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(force); + } } @Override @@ -112,13 +132,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Job.ID.getPreferredName(), jobId); builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); builder.field(DELETE_INTERVENING.getPreferredName(), deleteInterveningResults); + builder.field(FORCE.getPreferredName(), force); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(jobId, snapshotId, deleteInterveningResults); + return Objects.hash(jobId, snapshotId, deleteInterveningResults, force); } @Override @@ -130,8 +151,10 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(jobId, other.jobId) && Objects.equals(snapshotId, other.snapshotId) - && Objects.equals(deleteInterveningResults, other.deleteInterveningResults); + return Objects.equals(jobId, other.jobId) + && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(deleteInterveningResults, other.deleteInterveningResults) + && force == other.force; } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java index 46b6999defcd2..bcac20fbdcd15 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java @@ -19,6 +19,9 @@ protected Request createTestInstance() { if (randomBoolean()) { request.setDeleteInterveningResults(randomBoolean()); } + if (randomBoolean()) { + request.setForce(randomBoolean()); + } return request; } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index f8989322caa47..7f174bcaa24db 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -5,10 +5,13 @@ */ package org.elasticsearch.xpack.ml.integration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -21,9 +24,12 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.persistent.PersistentTaskResponse; @@ -44,11 +50,15 @@ import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; +import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; +import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.process.autodetect.BlackHoleAutodetectProcess; @@ -56,6 +66,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -66,6 +77,8 @@ import static org.elasticsearch.test.NodeRoles.onlyRole; import static org.elasticsearch.test.NodeRoles.onlyRoles; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -147,7 +160,7 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception { String jobId = "test-lose-ml-node"; String datafeedId = jobId + "-datafeed"; setupJobAndDatafeed(jobId, datafeedId, TimeValue.timeValueHours(1)); - waitForDatafeed(jobId, numDocs1); + waitForJobToHaveProcessedExactly(jobId, numDocs1); // stop the only ML node ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index @@ -224,7 +237,7 @@ public void testCloseUnassignedFailedJobAndStopUnassignedStoppingDatafeed() thro String jobId = "test-stop-unassigned-datafeed-for-failed-job"; String datafeedId = jobId + "-datafeed"; setupJobAndDatafeed(jobId, datafeedId, TimeValue.timeValueHours(1)); - waitForDatafeed(jobId, numDocs1); + waitForJobToHaveProcessedExactly(jobId, numDocs1); // Job state should be opened here GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId); @@ -331,7 +344,7 @@ public void testStopAndForceStopDatafeed() throws Exception { String jobId = "test-stop-and-force-stop"; String datafeedId = jobId + "-datafeed"; setupJobAndDatafeed(jobId, datafeedId, TimeValue.timeValueHours(1)); - waitForDatafeed(jobId, numDocs1); + waitForJobToHaveProcessedExactly(jobId, numDocs1); GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(datafeedId); GetDatafeedsStatsAction.Response datafeedStatsResponse = @@ -416,6 +429,65 @@ public void testJobRelocationIsMemoryAware() throws Exception { }); } + public void testClusterWithTwoMlNodes_RunsDatafeed_GivenOriginalNodeGoesDown() throws Exception { + internalCluster().ensureAtMostNumDataNodes(0); + logger.info("Starting dedicated master node..."); + internalCluster().startMasterOnlyNode(); + logger.info("Starting ml and data node..."); + internalCluster().startNode(onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE, MachineLearning.ML_ROLE))); + logger.info("Starting another ml and data node..."); + internalCluster().startNode(onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE, MachineLearning.ML_ROLE))); + ensureStableCluster(); + + // index some datafeed data + client().admin().indices().prepareCreate("data") + .setMapping("time", "type=date") + .get(); + long numDocs = 80000; + long now = System.currentTimeMillis(); + long weekAgo = now - 604800000; + long twoWeeksAgo = weekAgo - 604800000; + indexDocs(logger, "data", numDocs, twoWeeksAgo, weekAgo); + + String jobId = "test-node-goes-down-while-running-job"; + String datafeedId = jobId + "-datafeed"; + + Job.Builder job = createScheduledJob(jobId); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job); + client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet(); + + DatafeedConfig config = createDatafeed(datafeedId, job.getId(), Collections.singletonList("data"), TimeValue.timeValueHours(1)); + PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(config); + client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).actionGet(); + + client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())); + + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(JobState.OPENED, statsResponse.getResponse().results().get(0).getState()); + }, 30, TimeUnit.SECONDS); + + DiscoveryNode nodeRunningJob = client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())) + .actionGet().getResponse().results().get(0).getNode(); + + setMlIndicesDelayedNodeLeftTimeoutToZero(); + + StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(config.getId(), 0L); + startDatafeedRequest.getParams().setEndTime("now"); + client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get(); + + waitForJobToHaveProcessedAtLeast(jobId, 1000); + + internalCluster().stopNode(nodeRunningJob.getName()); + + waitForJobClosed(jobId); + + DataCounts dataCounts = getJobStats(jobId).getDataCounts(); + assertThat(dataCounts.getProcessedRecordCount(), greaterThanOrEqualTo(numDocs)); + assertThat(dataCounts.getOutOfOrderTimeStampCount(), equalTo(0L)); + } + private void setupJobWithoutDatafeed(String jobId, ByteSizeValue modelMemoryLimit) throws Exception { Job.Builder job = createFareQuoteJob(jobId, modelMemoryLimit); PutJobAction.Request putJobRequest = new PutJobAction.Request(job); @@ -462,7 +534,12 @@ private void run(String jobId, CheckedRunnable disrupt) throws Except indexDocs(logger, "data", numDocs1, twoWeeksAgo, weekAgo); setupJobAndDatafeed(jobId, "data_feed_id", TimeValue.timeValueSeconds(1)); - waitForDatafeed(jobId, numDocs1); + waitForJobToHaveProcessedExactly(jobId, numDocs1); + + // At this point the lookback has completed and normally there would be a model snapshot persisted. + // We manually index a model snapshot document to imitate this behaviour and to avoid the job + // having to recover after reassignment. + indexModelSnapshotFromCurrentJobStats(jobId); client().admin().indices().prepareFlush().get(); @@ -507,7 +584,7 @@ private void run(String jobId, CheckedRunnable disrupt) throws Except long numDocs2 = randomIntBetween(2, 64); long now2 = System.currentTimeMillis(); indexDocs(logger, "data", numDocs2, now2 + 5000, now2 + 6000); - waitForDatafeed(jobId, numDocs1 + numDocs2); + waitForJobToHaveProcessedExactly(jobId, numDocs1 + numDocs2); } // Get datacounts from index instead of via job stats api, @@ -532,7 +609,7 @@ private static DataCounts getDataCountsFromIndex(String jobId) { } } - private void waitForDatafeed(String jobId, long numDocs) throws Exception { + private void waitForJobToHaveProcessedExactly(String jobId, long numDocs) throws Exception { assertBusy(() -> { DataCounts dataCounts = getDataCountsFromIndex(jobId); assertEquals(numDocs, dataCounts.getProcessedRecordCount()); @@ -540,7 +617,46 @@ private void waitForDatafeed(String jobId, long numDocs) throws Exception { }, 30, TimeUnit.SECONDS); } + private void waitForJobToHaveProcessedAtLeast(String jobId, long numDocs) throws Exception { + assertBusy(() -> { + DataCounts dataCounts = getDataCountsFromIndex(jobId); + assertThat(dataCounts.getProcessedRecordCount(), greaterThanOrEqualTo(numDocs)); + assertEquals(0L, dataCounts.getOutOfOrderTimeStampCount()); + }, 30, TimeUnit.SECONDS); + } + + private void waitForJobClosed(String jobId) throws Exception { + assertBusy(() -> { + JobStats jobStats = getJobStats(jobId); + assertEquals(jobStats.getState(), JobState.CLOSED); + }, 30, TimeUnit.SECONDS); + } + private void ensureStableCluster() { ensureStableCluster(internalCluster().getNodeNames().length, TimeValue.timeValueSeconds(60)); } + + private void indexModelSnapshotFromCurrentJobStats(String jobId) throws IOException { + JobStats jobStats = getJobStats(jobId); + DataCounts dataCounts = jobStats.getDataCounts(); + + ModelSnapshot modelSnapshot = new ModelSnapshot.Builder(jobId) + .setLatestRecordTimeStamp(dataCounts.getLatestRecordTimeStamp()) + .setMinVersion(Version.CURRENT) + .setSnapshotId(jobId + "_mock_snapshot") + .setTimestamp(new Date()) + .build(); + + try (XContentBuilder xContentBuilder = JsonXContent.contentBuilder()) { + modelSnapshot.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.id(ModelSnapshot.documentId(modelSnapshot)); + indexRequest.source(xContentBuilder); + client().index(indexRequest).actionGet(); + } + + JobUpdate jobUpdate = new JobUpdate.Builder(jobId).setModelSnapshotId(modelSnapshot.getSnapshotId()).build(); + client().execute(UpdateJobAction.INSTANCE, new UpdateJobAction.Request(jobId, jobUpdate)).actionGet(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 9805cd41d18f4..e79115127d148 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -228,6 +228,7 @@ import org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingDeciderService; import org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingNamedWritableProvider; import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigAutoUpdater; +import org.elasticsearch.xpack.ml.datafeed.DatafeedContextProvider; import org.elasticsearch.xpack.ml.datafeed.DatafeedJobBuilder; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; @@ -330,11 +331,11 @@ import org.elasticsearch.xpack.ml.rest.job.RestPostDataAction; import org.elasticsearch.xpack.ml.rest.job.RestPostJobUpdateAction; import org.elasticsearch.xpack.ml.rest.job.RestPutJobAction; -import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestUpgradeJobModelSnapshotAction; import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestDeleteModelSnapshotAction; import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestGetModelSnapshotsAction; import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestRevertModelSnapshotAction; import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestUpdateModelSnapshotAction; +import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestUpgradeJobModelSnapshotAction; import org.elasticsearch.xpack.ml.rest.results.RestGetBucketsAction; import org.elasticsearch.xpack.ml.rest.results.RestGetCategoriesAction; import org.elasticsearch.xpack.ml.rest.results.RestGetInfluencersAction; @@ -504,6 +505,7 @@ public Set getRoles() { private final boolean enabled; private final SetOnce autodetectProcessManager = new SetOnce<>(); + private final SetOnce datafeedContextProvider = new SetOnce<>(); private final SetOnce datafeedManager = new SetOnce<>(); private final SetOnce dataFrameAnalyticsManager = new SetOnce<>(); private final SetOnce dataFrameAnalyticsAuditor = new SetOnce<>(); @@ -723,9 +725,11 @@ public Collection createComponents(Client client, ClusterService cluster jobResultsPersister, settings, clusterService.getNodeName()); + DatafeedContextProvider datafeedContextProvider = new DatafeedContextProvider(jobConfigProvider, datafeedConfigProvider, + jobResultsProvider); + this.datafeedContextProvider.set(datafeedContextProvider); DatafeedManager datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder, - System::currentTimeMillis, anomalyDetectionAuditor, autodetectProcessManager, jobConfigProvider, datafeedConfigProvider, - jobResultsProvider); + System::currentTimeMillis, anomalyDetectionAuditor, autodetectProcessManager, datafeedContextProvider); this.datafeedManager.set(datafeedManager); // Inference components @@ -831,6 +835,7 @@ public List> getPersistentTasksExecutor(ClusterServic new OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get(), + datafeedContextProvider.get(), memoryTracker.get(), client, expressionResolver), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index c896ded6a0090..4e35154dd4ff7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -85,7 +85,7 @@ protected void masterOperation(Task task, RevertModelSnapshotAction.Request requ PersistentTasksCustomMetadata tasks = state.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); JobState jobState = MlTasks.getJobState(request.getJobId(), tasks); - if (jobState.equals(JobState.CLOSED) == false) { + if (request.isForce() == false && jobState.equals(JobState.CLOSED) == false) { listener.onFailure(ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT))); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContext.java index 4ae622f89403b..cf8819f3c7a72 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContext.java @@ -6,50 +6,100 @@ package org.elasticsearch.xpack.ml.datafeed; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Nullable; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.persistence.RestartTimeInfo; import java.util.Objects; -class DatafeedContext { +public class DatafeedContext { + private static final Logger logger = LogManager.getLogger(DatafeedContext.class); + + private final long datafeedStartTimeMs; private final DatafeedConfig datafeedConfig; private final Job job; private final RestartTimeInfo restartTimeInfo; private final DatafeedTimingStats timingStats; + @Nullable + private final ModelSnapshot modelSnapshot; - DatafeedContext(DatafeedConfig datafeedConfig, Job job, RestartTimeInfo restartTimeInfo, - DatafeedTimingStats timingStats) { + private DatafeedContext(long datafeedStartTimeMs, DatafeedConfig datafeedConfig, Job job, RestartTimeInfo restartTimeInfo, + DatafeedTimingStats timingStats, ModelSnapshot modelSnapshot) { + this.datafeedStartTimeMs = datafeedStartTimeMs; this.datafeedConfig = Objects.requireNonNull(datafeedConfig); this.job = Objects.requireNonNull(job); this.restartTimeInfo = Objects.requireNonNull(restartTimeInfo); this.timingStats = Objects.requireNonNull(timingStats); + this.modelSnapshot = modelSnapshot; } - - DatafeedConfig getDatafeedConfig() { + public DatafeedConfig getDatafeedConfig() { return datafeedConfig; } - Job getJob() { + public Job getJob() { return job; } - RestartTimeInfo getRestartTimeInfo() { + public RestartTimeInfo getRestartTimeInfo() { return restartTimeInfo; } - DatafeedTimingStats getTimingStats() { + public DatafeedTimingStats getTimingStats() { return timingStats; } + @Nullable + public ModelSnapshot getModelSnapshot() { + return modelSnapshot; + } + + public boolean shouldRecoverFromCurrentSnapshot() { + if (modelSnapshot == null) { + logger.debug("[{}] checking whether recovery is required; job latest result timestamp [{}]; " + + "job latest record timestamp [{}]; snapshot is [null]; datafeed start time [{}]", datafeedConfig.getJobId(), + restartTimeInfo.getLatestFinalBucketTimeMs(), restartTimeInfo.getLatestRecordTimeMs(), datafeedStartTimeMs); + } else { + logger.debug("[{}] checking whether recovery is required; job latest result timestamp [{}]; " + + "job latest record timestamp [{}]; snapshot latest result timestamp [{}]; snapshot latest record timestamp [{}]; " + + "datafeed start time [{}]", + datafeedConfig.getJobId(), + restartTimeInfo.getLatestFinalBucketTimeMs(), + restartTimeInfo.getLatestRecordTimeMs(), + modelSnapshot.getLatestResultTimeStamp() == null ? null : modelSnapshot.getLatestResultTimeStamp().getTime(), + modelSnapshot.getLatestRecordTimeStamp() == null ? null : modelSnapshot.getLatestRecordTimeStamp().getTime(), + datafeedStartTimeMs); + } + + if (restartTimeInfo.isAfter(datafeedStartTimeMs)) { + return restartTimeInfo.haveSeenDataPreviously() && + (modelSnapshot == null || restartTimeInfo.isAfterModelSnapshot(modelSnapshot)); + } + // If the datafeed start time is past the job checkpoint we should not attempt to recover + return false; + } + + static Builder builder(long datafeedStartTimeMs) { + return new Builder(datafeedStartTimeMs); + } + static class Builder { + private final long datafeedStartTimeMs; private volatile DatafeedConfig datafeedConfig; private volatile Job job; private volatile RestartTimeInfo restartTimeInfo; private volatile DatafeedTimingStats timingStats; + private volatile ModelSnapshot modelSnapshot; + + Builder(long datafeedStartTimeMs) { + this.datafeedStartTimeMs = datafeedStartTimeMs; + } Builder setDatafeedConfig(DatafeedConfig datafeedConfig) { this.datafeedConfig = datafeedConfig; @@ -75,8 +125,13 @@ Builder setTimingStats(DatafeedTimingStats timingStats) { return this; } + Builder setModelSnapshot(ModelSnapshot modelSnapshot) { + this.modelSnapshot = modelSnapshot; + return this; + } + DatafeedContext build() { - return new DatafeedContext(datafeedConfig, job, restartTimeInfo, timingStats); + return new DatafeedContext(datafeedStartTimeMs, datafeedConfig, job, restartTimeInfo, timingStats, modelSnapshot); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextProvider.java new file mode 100644 index 0000000000000..7e5f271fa9f69 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextProvider.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.core.ml.job.results.Result; +import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; +import org.elasticsearch.xpack.ml.job.persistence.RestartTimeInfo; + +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; + +public class DatafeedContextProvider { + + private final JobConfigProvider jobConfigProvider; + private final DatafeedConfigProvider datafeedConfigProvider; + private final JobResultsProvider resultsProvider; + + public DatafeedContextProvider(JobConfigProvider jobConfigProvider, DatafeedConfigProvider datafeedConfigProvider, + JobResultsProvider jobResultsProvider) { + this.jobConfigProvider = Objects.requireNonNull(jobConfigProvider); + this.datafeedConfigProvider = Objects.requireNonNull(datafeedConfigProvider); + this.resultsProvider = Objects.requireNonNull(jobResultsProvider); + } + + public void buildDatafeedContext(String datafeedId, long datafeedStartTimeMs, ActionListener listener) { + DatafeedContext.Builder context = DatafeedContext.builder(datafeedStartTimeMs); + + Consumer> modelSnapshotListener = resultSnapshot -> { + context.setModelSnapshot(resultSnapshot == null ? null : resultSnapshot.result); + listener.onResponse(context.build()); + }; + + Consumer timingStatsListener = timingStats -> { + context.setTimingStats(timingStats); + resultsProvider.getModelSnapshot( + context.getJob().getId(), + context.getJob().getModelSnapshotId(), + modelSnapshotListener, + listener::onFailure); + }; + + ActionListener restartTimeInfoListener = ActionListener.wrap( + restartTimeInfo -> { + context.setRestartTimeInfo(restartTimeInfo); + resultsProvider.datafeedTimingStats(context.getJob().getId(), timingStatsListener, listener::onFailure); + }, + listener::onFailure + ); + + ActionListener jobConfigListener = ActionListener.wrap( + jobBuilder -> { + context.setJob(jobBuilder.build()); + resultsProvider.getRestartTimeInfo(jobBuilder.getId(), restartTimeInfoListener); + }, + listener::onFailure + ); + + ActionListener datafeedListener = ActionListener.wrap( + datafeedConfigBuilder -> { + DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); + context.setDatafeedConfig(datafeedConfig); + jobConfigProvider.getJob(datafeedConfig.getJobId(), jobConfigListener); + }, + listener::onFailure + ); + + datafeedConfigProvider.getDatafeedConfig(datafeedId, datafeedListener); + } + + public void buildDatafeedContextForJob(String jobId, ClusterState clusterState, ActionListener listener) { + ActionListener> datafeedListener = ActionListener.wrap( + datafeeds -> { + assert datafeeds.size() <= 1; + if (datafeeds.isEmpty()) { + // This job has no datafeed attached to it + listener.onResponse(null); + return; + } + + String datafeedId = datafeeds.iterator().next(); + PersistentTasksCustomMetadata tasks = clusterState.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); + PersistentTasksCustomMetadata.PersistentTask datafeedTask = MlTasks.getDatafeedTask(datafeedId, tasks); + if (datafeedTask == null) { + // This datafeed is not started + listener.onResponse(null); + return; + } + + @SuppressWarnings("unchecked") + StartDatafeedAction.DatafeedParams taskParams = (StartDatafeedAction.DatafeedParams) datafeedTask.getParams(); + buildDatafeedContext(datafeedId, taskParams.getStartTime(), listener); + }, + listener::onFailure + ); + + datafeedConfigProvider.findDatafeedsForJobIds(Collections.singleton(jobId), datafeedListener); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java index 91571fb237b4d..e831e89e19931 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java @@ -26,20 +26,13 @@ import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction; -import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; -import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; -import org.elasticsearch.xpack.ml.job.persistence.RestartTimeInfo; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor; @@ -73,14 +66,11 @@ public class DatafeedManager { private final DatafeedJobBuilder datafeedJobBuilder; private final TaskRunner taskRunner = new TaskRunner(); private final AutodetectProcessManager autodetectProcessManager; - private final JobConfigProvider jobConfigProvider; - private final DatafeedConfigProvider datafeedConfigProvider; - private final JobResultsProvider resultsProvider; + private final DatafeedContextProvider datafeedContextProvider; public DatafeedManager(ThreadPool threadPool, Client client, ClusterService clusterService, DatafeedJobBuilder datafeedJobBuilder, Supplier currentTimeSupplier, AnomalyDetectionAuditor auditor, - AutodetectProcessManager autodetectProcessManager, JobConfigProvider jobConfigProvider, - DatafeedConfigProvider datafeedConfigProvider, JobResultsProvider resultsProvider) { + AutodetectProcessManager autodetectProcessManager, DatafeedContextProvider datafeedContextProvider) { this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.threadPool = Objects.requireNonNull(threadPool); @@ -88,9 +78,7 @@ public DatafeedManager(ThreadPool threadPool, Client client, ClusterService clus this.auditor = Objects.requireNonNull(auditor); this.datafeedJobBuilder = Objects.requireNonNull(datafeedJobBuilder); this.autodetectProcessManager = Objects.requireNonNull(autodetectProcessManager); - this.jobConfigProvider = Objects.requireNonNull(jobConfigProvider); - this.datafeedConfigProvider = Objects.requireNonNull(datafeedConfigProvider); - this.resultsProvider = Objects.requireNonNull(resultsProvider); + this.datafeedContextProvider = Objects.requireNonNull(datafeedContextProvider); clusterService.addListener(taskRunner); } @@ -125,43 +113,7 @@ public void onFailure(Exception e) { finishHandler ); - buildDatafeedContext(task, datafeedContextListener); - } - - private void buildDatafeedContext(TransportStartDatafeedAction.DatafeedTask task, ActionListener listener) { - DatafeedContext.Builder context = new DatafeedContext.Builder(); - - Consumer timingStatsListener = timingStats -> { - context.setTimingStats(timingStats); - listener.onResponse(context.build()); - }; - - ActionListener restartTimeInfoListener = ActionListener.wrap( - restartTimeInfo -> { - context.setRestartTimeInfo(restartTimeInfo); - resultsProvider.datafeedTimingStats(context.getJob().getId(), timingStatsListener, listener::onFailure); - }, - listener::onFailure - ); - - ActionListener jobConfigListener = ActionListener.wrap( - jobBuilder -> { - context.setJob(jobBuilder.build()); - resultsProvider.getRestartTimeInfo(jobBuilder.getId(), restartTimeInfoListener); - }, - listener::onFailure - ); - - ActionListener datafeedListener = ActionListener.wrap( - datafeedConfigBuilder -> { - DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); - context.setDatafeedConfig(datafeedConfig); - jobConfigProvider.getJob(datafeedConfig.getJobId(), jobConfigListener); - }, - listener::onFailure - ); - - datafeedConfigProvider.getDatafeedConfig(task.getDatafeedId(), datafeedListener); + datafeedContextProvider.buildDatafeedContext(task.getDatafeedId(), task.getDatafeedStartTime(), datafeedContextListener); } public void stopDatafeed(TransportStartDatafeedAction.DatafeedTask task, String reason, TimeValue timeout) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java index 0f6c51d17caa4..89c51b8b14f0f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java @@ -91,6 +91,7 @@ public void persistDataCountsAsync(String jobId, DataCounts counts, ActionListen final IndexRequest request = new IndexRequest(AnomalyDetectorsIndex.resultsWriteAlias(jobId)) .id(DataCounts.documentId(jobId)) .setRequireAlias(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(content); executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, request, new ActionListener<>() { @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfo.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfo.java index fe588ef3301f5..f606a74cbea92 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfo.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfo.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.job.persistence; import org.elasticsearch.common.Nullable; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; public class RestartTimeInfo { @@ -33,4 +34,21 @@ public Long getLatestRecordTimeMs() { public boolean haveSeenDataPreviously() { return haveSeenDataPreviously; } + + public boolean isAfter(long timestampMs) { + long jobLatestResultTime = latestFinalBucketTimeMs == null ? 0 : latestFinalBucketTimeMs; + long jobLatestRecordTime = latestRecordTimeMs == null ? 0 : latestRecordTimeMs; + return Math.max(jobLatestResultTime, jobLatestRecordTime) > timestampMs; + } + + public boolean isAfterModelSnapshot(ModelSnapshot modelSnapshot) { + assert modelSnapshot != null; + long jobLatestResultTime = latestFinalBucketTimeMs == null ? 0 : latestFinalBucketTimeMs; + long jobLatestRecordTime = latestRecordTimeMs == null ? 0 : latestRecordTimeMs; + long modelLatestResultTime = modelSnapshot.getLatestResultTimeStamp() == null ? + 0 : modelSnapshot.getLatestResultTimeStamp().getTime(); + long modelLatestRecordTime = modelSnapshot.getLatestRecordTimeStamp() == null ? + 0 : modelSnapshot.getLatestRecordTimeStamp().getTime(); + return jobLatestResultTime > modelLatestResultTime || jobLatestRecordTime > modelLatestRecordTime; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index ca7a7b70bc384..29d141c5a34f8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -603,7 +603,7 @@ AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodet String jobId = jobTask.getJobId(); notifyLoadingSnapshot(jobId, autodetectParams); - if (autodetectParams.dataCounts().getProcessedRecordCount() > 0) { + if (autodetectParams.dataCounts().getLatestRecordTimeStamp() != null) { if (autodetectParams.modelSnapshot() == null) { String msg = "No model snapshot could be found for a job with processed records"; logger.warn("[{}] {}", jobId, msg); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java index 9ad169fa518c9..df0d18bbbf7eb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java @@ -31,12 +31,16 @@ import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.datafeed.DatafeedContext; +import org.elasticsearch.xpack.ml.datafeed.DatafeedContextProvider; import org.elasticsearch.xpack.ml.job.JobNodeSelector; import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -65,16 +69,19 @@ public static String[] indicesOfInterest(String resultsIndex) { } private final AutodetectProcessManager autodetectProcessManager; + private final DatafeedContextProvider datafeedContextProvider; private final Client client; private final JobResultsProvider jobResultsProvider; private volatile ClusterState clusterState; public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterService, - AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker, - Client client, IndexNameExpressionResolver expressionResolver) { + AutodetectProcessManager autodetectProcessManager, + DatafeedContextProvider datafeedContextProvider, MlMemoryTracker memoryTracker, Client client, + IndexNameExpressionResolver expressionResolver) { super(MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME, settings, clusterService, memoryTracker, expressionResolver); this.autodetectProcessManager = Objects.requireNonNull(autodetectProcessManager); + this.datafeedContextProvider = Objects.requireNonNull(datafeedContextProvider); this.client = Objects.requireNonNull(client); this.jobResultsProvider = new JobResultsProvider(client, settings, expressionResolver); clusterService.addListener(event -> clusterState = event.state()); @@ -155,7 +162,7 @@ public void validate(OpenJobAction.JobParams params, ClusterState clusterState) // simply because there are no ml nodes in the cluster then we fail quickly here: PersistentTasksCustomMetadata.Assignment assignment = getAssignment(params, clusterState); if (assignment.equals(AWAITING_UPGRADE)) { - throw makeCurrentlyBeingUpgradedException(logger, params.getJobId(), assignment.getExplanation()); + throw makeCurrentlyBeingUpgradedException(logger, params.getJobId()); } if (assignment.getExecutorNode() == null && assignment.equals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT) == false) { @@ -193,6 +200,38 @@ private void runJob(JobTask jobTask, JobState jobState, OpenJobAction.JobParams return; } + ActionListener datafeedContextListener = ActionListener.wrap( + datafeedContext -> { + if (datafeedContext != null && datafeedContext.shouldRecoverFromCurrentSnapshot()) { + // This job has a datafeed attached to it and the job had advanced past the current model snapshot. + // In order to prevent gaps in the model we revert to the current snapshot deleting intervening results. + ModelSnapshot modelSnapshot = datafeedContext.getModelSnapshot() == null ? + ModelSnapshot.emptySnapshot(jobTask.getJobId()) : datafeedContext.getModelSnapshot(); + logger.info("[{}] job had advanced past its current model snapshot [{}]; performing recovery", + jobTask.getJobId(), modelSnapshot.getSnapshotId()); + revertToSnapshot(modelSnapshot, ActionListener.wrap( + response -> openJob(jobTask), + jobTask::markAsFailed + )); + } else { + openJob(jobTask); + } + }, + jobTask::markAsFailed + ); + + datafeedContextProvider.buildDatafeedContextForJob(jobTask.getJobId(), clusterState, datafeedContextListener); + } + + private void revertToSnapshot(ModelSnapshot modelSnapshot, ActionListener listener) { + RevertModelSnapshotAction.Request request = new RevertModelSnapshotAction.Request(modelSnapshot.getJobId(), + modelSnapshot.getSnapshotId()); + request.setForce(true); + request.setDeleteInterveningResults(true); + executeAsyncWithOrigin(client, ML_ORIGIN, RevertModelSnapshotAction.INSTANCE, request, listener); + } + + private void openJob(JobTask jobTask) { String jobId = jobTask.getJobId(); autodetectProcessManager.openJob(jobTask, clusterState, (e2, shouldFinalizeJob) -> { if (e2 == null) { @@ -235,7 +274,7 @@ public static Optional checkAssignmentState(PersistentTa && assignment.isAssigned() == false) { // Assignment has failed on the master node despite passing our "fast fail" validation if (assignment.equals(AWAITING_UPGRADE)) { - return Optional.of(makeCurrentlyBeingUpgradedException(logger, jobId, assignment.getExplanation())); + return Optional.of(makeCurrentlyBeingUpgradedException(logger, jobId)); } else if (assignment.getExplanation().contains("[" + EnableAssignmentDecider.ALLOCATION_NONE_EXPLANATION + "]")) { return Optional.of(makeAssignmentsNotAllowedException(logger, jobId)); } else { @@ -260,7 +299,7 @@ static ElasticsearchException makeAssignmentsNotAllowedException(Logger logger, return new ElasticsearchStatusException(msg, RestStatus.TOO_MANY_REQUESTS); } - static ElasticsearchException makeCurrentlyBeingUpgradedException(Logger logger, String jobId, String explanation) { + static ElasticsearchException makeCurrentlyBeingUpgradedException(Logger logger, String jobId) { String msg = "Cannot open jobs when upgrade mode is enabled"; logger.warn("[{}] {}", jobId, msg); return new ElasticsearchStatusException(msg, RestStatus.TOO_MANY_REQUESTS); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java index f35964f8f0f92..0f1ddf607777d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilderTests.java @@ -89,12 +89,12 @@ public void testBuild_GivenScrollDatafeedAndNewJob() throws Exception { }, e -> fail() ); - DatafeedContext datafeedContext = new DatafeedContext( - datafeed.build(), - jobBuilder.build(), - new RestartTimeInfo(null, null, false), - new DatafeedTimingStats(jobBuilder.getId()) - ); + DatafeedContext datafeedContext = DatafeedContext.builder(0L) + .setDatafeedConfig(datafeed.build()) + .setJob(jobBuilder.build()) + .setRestartTimeInfo(new RestartTimeInfo(null, null, false)) + .setTimingStats(new DatafeedTimingStats(jobBuilder.getId())) + .build(); TransportStartDatafeedAction.DatafeedTask datafeedTask = newDatafeedTask("datafeed1"); @@ -121,12 +121,12 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestRecordTimestampAfter }, e -> fail() ); - DatafeedContext datafeedContext = new DatafeedContext( - datafeed.build(), - jobBuilder.build(), - new RestartTimeInfo(3_600_000L, 7_200_000L, false), - new DatafeedTimingStats(jobBuilder.getId()) - ); + DatafeedContext datafeedContext = DatafeedContext.builder(0L) + .setDatafeedConfig(datafeed.build()) + .setJob(jobBuilder.build()) + .setRestartTimeInfo(new RestartTimeInfo(3_600_000L, 7_200_000L, false)) + .setTimingStats(new DatafeedTimingStats(jobBuilder.getId())) + .build(); TransportStartDatafeedAction.DatafeedTask datafeedTask = newDatafeedTask("datafeed1"); @@ -153,12 +153,12 @@ public void testBuild_GivenScrollDatafeedAndOldJobWithLatestBucketAfterLatestRec }, e -> fail() ); - DatafeedContext datafeedContext = new DatafeedContext( - datafeed.build(), - jobBuilder.build(), - new RestartTimeInfo(3_800_000L, 3_600_000L, false), - new DatafeedTimingStats(jobBuilder.getId()) - ); + DatafeedContext datafeedContext = DatafeedContext.builder(0L) + .setDatafeedConfig(datafeed.build()) + .setJob(jobBuilder.build()) + .setRestartTimeInfo(new RestartTimeInfo(3_800_000L, 3_600_000L, false)) + .setTimingStats(new DatafeedTimingStats(jobBuilder.getId())) + .build(); TransportStartDatafeedAction.DatafeedTask datafeedTask = newDatafeedTask("datafeed1"); @@ -198,12 +198,12 @@ public void testBuildGivenRemoteIndicesButNoRemoteSearching() throws Exception { } ); - DatafeedContext datafeedContext = new DatafeedContext( - datafeed.build(), - jobBuilder.build(), - new RestartTimeInfo(null, null, false), - new DatafeedTimingStats(jobBuilder.getId()) - ); + DatafeedContext datafeedContext = DatafeedContext.builder(0L) + .setDatafeedConfig(datafeed.build()) + .setJob(jobBuilder.build()) + .setRestartTimeInfo(new RestartTimeInfo(null, null, false)) + .setTimingStats(new DatafeedTimingStats(jobBuilder.getId())) + .build(); TransportStartDatafeedAction.DatafeedTask datafeedTask = newDatafeedTask("datafeed1"); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java index 2d7175e2be8a8..501138e1e873f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManagerTests.java @@ -37,9 +37,6 @@ import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction.DatafeedTask; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedActionTests; -import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; -import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider; -import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider; import org.elasticsearch.xpack.ml.job.persistence.RestartTimeInfo; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor; @@ -78,9 +75,7 @@ public class DatafeedManagerTests extends ESTestCase { private ThreadPool threadPool; private DatafeedJob datafeedJob; private DatafeedManager datafeedManager; - private DatafeedConfigProvider datafeedConfigProvider; - private JobConfigProvider jobConfigProvider; - private JobResultsProvider jobResultsProvider; + private DatafeedContextProvider datafeedContextProvider; private long currentTime = 120000; private AnomalyDetectionAuditor auditor; private ArgumentCaptor capturedClusterStateListener = ArgumentCaptor.forClass(ClusterStateListener.class); @@ -138,29 +133,13 @@ public void setUpTests() { AutodetectProcessManager autodetectProcessManager = mock(AutodetectProcessManager.class); doAnswer(invocation -> hasOpenAutodetectCommunicator.get()).when(autodetectProcessManager).hasOpenAutodetectCommunicator(anyLong()); - jobConfigProvider = mock(JobConfigProvider.class); - final Job.Builder datafeedJob = createDatafeedJob(); - doAnswer(invocationOnMock -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(datafeedJob); - return null; - }).when(jobConfigProvider).getJob(eq(JOB_ID), any()); + datafeedContextProvider = mock(DatafeedContextProvider.class); - datafeedConfigProvider = mock(DatafeedConfigProvider.class); - final DatafeedConfig.Builder datafeedConfig = createDatafeedConfig(DATAFEED_ID, JOB_ID); - doAnswer(invocationOnMock -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(datafeedConfig); - return null; - }).when(datafeedConfigProvider).getDatafeedConfig(eq(DATAFEED_ID), any()); - - jobResultsProvider = mock(JobResultsProvider.class); - givenDatafeedHasNeverRunBefore(); + DatafeedConfig.Builder datafeedConfig = createDatafeedConfig(DATAFEED_ID, job.getId()); + givenDatafeedHasNeverRunBefore(job.build(), datafeedConfig.build()); datafeedManager = new DatafeedManager(threadPool, mock(Client.class), clusterService, datafeedJobBuilder, - () -> currentTime, auditor, autodetectProcessManager, jobConfigProvider, datafeedConfigProvider, jobResultsProvider); + () -> currentTime, auditor, autodetectProcessManager, datafeedContextProvider); verify(clusterService).addListener(capturedClusterStateListener.capture()); } @@ -492,19 +471,19 @@ private DatafeedTask spyDatafeedTask(DatafeedTask task) { return task; } - private void givenDatafeedHasNeverRunBefore() { - doAnswer(invocationOnMock -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(new RestartTimeInfo(null, null, false)); - return null; - }).when(jobResultsProvider).getRestartTimeInfo(eq(JOB_ID), any()); - + private void givenDatafeedHasNeverRunBefore(Job job, DatafeedConfig datafeed) { doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") - Consumer consumer = (Consumer) invocationOnMock.getArguments()[1]; - consumer.accept(new DatafeedTimingStats(JOB_ID)); + ActionListener datafeedContextListener = (ActionListener) invocationOnMock.getArguments()[2]; + DatafeedContext datafeedContext = DatafeedContext.builder(0L) + .setJob(job) + .setDatafeedConfig(datafeed) + .setModelSnapshot(null) + .setRestartTimeInfo(new RestartTimeInfo(null, null, false)) + .setTimingStats(new DatafeedTimingStats(job.getId())) + .build(); + datafeedContextListener.onResponse(datafeedContext); return null; - }).when(jobResultsProvider).datafeedTimingStats(eq(JOB_ID), any(), any()); + }).when(datafeedContextProvider).buildDatafeedContext(eq(DATAFEED_ID), anyLong(), any()); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java index 7fae2bdee980a..a7f421201a6b5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java @@ -49,9 +49,11 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.datafeed.DatafeedContextProvider; import org.elasticsearch.xpack.ml.job.JobNodeSelector; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; import org.elasticsearch.xpack.ml.process.MlMemoryTracker; +import org.junit.Before; import java.util.ArrayList; import java.util.Collections; @@ -66,6 +68,21 @@ public class OpenJobPersistentTasksExecutorTests extends ESTestCase { + private ClusterService clusterService; + private AutodetectProcessManager autodetectProcessManager; + private DatafeedContextProvider datafeedContextProvider; + private Client client; + private MlMemoryTracker mlMemoryTracker; + + @Before + public void setUpMocks() { + clusterService = mock(ClusterService.class); + autodetectProcessManager = mock(AutodetectProcessManager.class); + datafeedContextProvider = mock(DatafeedContextProvider.class); + client = mock(Client.class); + mlMemoryTracker = mock(MlMemoryTracker.class); + } + public void testValidate_jobMissing() { expectThrows(ResourceNotFoundException.class, () -> validateJobAndId("job_id2", null)); } @@ -92,16 +109,13 @@ public void testValidate_givenValidJob() { } public void testGetAssignment_GivenJobThatRequiresMigration() { - ClusterService clusterService = mock(ClusterService.class); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - OpenJobPersistentTasksExecutor executor = new OpenJobPersistentTasksExecutor( - Settings.EMPTY, clusterService, mock(AutodetectProcessManager.class), mock(MlMemoryTracker.class), mock(Client.class), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); + OpenJobPersistentTasksExecutor executor = createExecutor(Settings.EMPTY); OpenJobAction.JobParams params = new OpenJobAction.JobParams("missing_job_field"); assertEquals(AWAITING_MIGRATION, executor.getAssignment(params, mock(ClusterState.class))); @@ -110,7 +124,6 @@ Settings.EMPTY, clusterService, mock(AutodetectProcessManager.class), mock(MlMem // An index being unavailable should take precedence over waiting for a lazy node public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 1).build(); - ClusterService clusterService = mock(ClusterService.class); ClusterSettings clusterSettings = new ClusterSettings(settings, Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) @@ -125,9 +138,7 @@ public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { csBuilder.metadata(metadata); csBuilder.routingTable(routingTable.build()); - OpenJobPersistentTasksExecutor executor = new OpenJobPersistentTasksExecutor( - settings, clusterService, mock(AutodetectProcessManager.class), mock(MlMemoryTracker.class), mock(Client.class), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); + OpenJobPersistentTasksExecutor executor = createExecutor(settings); OpenJobAction.JobParams params = new OpenJobAction.JobParams("unavailable_index_with_lazy_node"); params.setJob(mock(Job.class)); @@ -138,7 +149,6 @@ settings, clusterService, mock(AutodetectProcessManager.class), mock(MlMemoryTra public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 0).build(); - ClusterService clusterService = mock(ClusterService.class); ClusterSettings clusterSettings = new ClusterSettings(settings, Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, MachineLearning.MAX_LAZY_ML_NODES, MachineLearning.MAX_OPEN_JOBS_PER_NODE, MachineLearning.USE_AUTO_MACHINE_MEMORY_PERCENT) @@ -152,9 +162,7 @@ public void testGetAssignment_GivenLazyJobAndNoGlobalLazyNodes() { csBuilder.metadata(metadata); csBuilder.routingTable(routingTable.build()); - OpenJobPersistentTasksExecutor executor = new OpenJobPersistentTasksExecutor( - settings, clusterService, mock(AutodetectProcessManager.class), mock(MlMemoryTracker.class), mock(Client.class), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); + OpenJobPersistentTasksExecutor executor = createExecutor(settings); Job job = mock(Job.class); when(job.allowLazyOpen()).thenReturn(true); @@ -224,4 +232,9 @@ public static Job jobWithRules(String jobId) { return job.build(new Date()); } + private OpenJobPersistentTasksExecutor createExecutor(Settings settings) { + return new OpenJobPersistentTasksExecutor( + settings, clusterService, autodetectProcessManager, datafeedContextProvider, mlMemoryTracker, client, + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); + } } From 83488ffb5e2e7be74165da25f3334e4e6bd34900 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 1 Dec 2020 07:49:16 -0500 Subject: [PATCH 05/27] [ML] update ChunkedTrainedModelPersisterTests to signal latch on persistence Minor test bug ChunkedTrainedModelPersisterTests was not mocking the inference index refresh call. Consequently the latch was not being signaled. --- .../process/ChunkedTrainedModelPersisterTests.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java index 2ae767b33983c..3b3dc142349b8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.license.License; @@ -98,6 +99,12 @@ public void testPersistAllDocs() { return null; }).when(trainedModelProvider).storeTrainedModelMetadata(any(TrainedModelMetadata.class), any(ActionListener.class)); + doAnswer(invocationOnMock -> { + ActionListener storeListener = (ActionListener) invocationOnMock.getArguments()[0]; + storeListener.onResponse(null); + return null; + }).when(trainedModelProvider).refreshInferenceIndex(any(ActionListener.class)); + ChunkedTrainedModelPersister resultProcessor = createChunkedTrainedModelPersister(extractedFieldList, analyticsConfig); ModelSizeInfo modelSizeInfo = ModelSizeInfoTests.createRandom(); TrainedModelDefinitionChunk chunk1 = new TrainedModelDefinitionChunk(randomAlphaOfLength(10), 0, false); From 6b0a1df126be69dd1653e041a5286d8adf96ca84 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 1 Dec 2020 07:51:36 -0500 Subject: [PATCH 06/27] [ML] fixes minor snapshot upgrader task failure bug and rest client test failure (#65607) This commit fixes two issues: - On task failure and when the task has completed, there is no reason to utilize the single queued executor. - The high level rest client tests should "await_completion" to make sure the task has completed before allowing other tests to continue. closes https://github.com/elastic/elasticsearch/issues/65364 --- .../MlClientDocumentationIT.java | 5 +-- .../autodetect/JobModelSnapshotUpgrader.java | 31 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index b3d7d7168da71..9df6be7717d3a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -2366,13 +2366,14 @@ public void testUpgradeJobSnapshot() throws IOException, InterruptedException { jobId, // <1> snapshotId, // <2> TimeValue.timeValueMinutes(30), // <3> - false); // <4> + true); // <4> // end::upgrade-job-model-snapshot-request try { // tag::upgrade-job-model-snapshot-execute UpgradeJobModelSnapshotResponse response = client.machineLearning().upgradeJobSnapshot(request, RequestOptions.DEFAULT); // end::upgrade-job-model-snapshot-execute + fail("upgrade model snapshot should not have succeeded."); } catch (ElasticsearchException ex) { assertThat(ex.getMessage(), containsString("Expected persisted state but no state exists")); } @@ -2384,7 +2385,7 @@ public void testUpgradeJobSnapshot() throws IOException, InterruptedException { // end::upgrade-job-model-snapshot-response } { - UpgradeJobModelSnapshotRequest request = new UpgradeJobModelSnapshotRequest(jobId, snapshotId, null, false); + UpgradeJobModelSnapshotRequest request = new UpgradeJobModelSnapshotRequest(jobId, snapshotId, null, true); // tag::upgrade-job-model-snapshot-execute-listener ActionListener listener = diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java index bafa9fe7f54a8..90c13925add7e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java @@ -217,17 +217,19 @@ void restoreState() { shutdown(null); return; } - submitOperation(() -> { - writeHeader(); - process.persistState( - params.modelSnapshot().getTimestamp().getTime(), - params.modelSnapshot().getSnapshotId(), - params.modelSnapshot().getDescription()); - return null; + submitOperation( + () -> { + writeHeader(); + process.persistState( + params.modelSnapshot().getTimestamp().getTime(), + params.modelSnapshot().getSnapshotId(), + params.modelSnapshot().getDescription()); + return null; + }, // Execute callback in the UTILITY thread pool, as the current thread in the callback will be one in the // autodetectWorkerExecutor. Trying to run the callback in that executor will cause a dead lock as that // executor has a single processing queue. - }, (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> shutdown(e))); + (aVoid, e) -> threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> shutdown(e))); logger.info("asked for state to be persisted"); }, f -> { @@ -274,18 +276,21 @@ protected void doRun() throws Exception { } private void checkProcessIsAlive() { - if (!process.isProcessAlive()) { + if (process.isProcessAlive() == false) { // Don't log here - it just causes double logging when the exception gets logged throw new ElasticsearchException("[{}] Unexpected death of autodetect: {}", job.getId(), process.readError()); } } void shutdown(Exception e) { + // No point in sending an action to the executor if the process has died + if (process.isProcessAlive() == false) { + onFinish.accept(e); + autodetectWorkerExecutor.shutdown(); + stateStreamer.cancel(); + return; + } autodetectWorkerExecutor.execute(() -> { - if (process.isProcessAlive() == false) { - onFinish.accept(e); - return; - } try { if (process.isReady()) { process.close(); From 581e5c82b4fd74ae2332d815e007623e1141dc6e Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 1 Dec 2020 08:57:07 -0500 Subject: [PATCH 07/27] [DOCS] Update rollup glossary item (#65519) Co-authored-by: Lisa Cawley --- docs/reference/glossary.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc index 7212fdc0f5192..e16cd070f5836 100644 --- a/docs/reference/glossary.asciidoc +++ b/docs/reference/glossary.asciidoc @@ -447,6 +447,17 @@ See the {ref}/indices-rollover-index.html[rollover index API]. // end::rollover-def[] -- +ifdef::permanently-unreleased-branch[] + +[[glossary-rollup]] rollup :: +// tag::rollup-def[] +Aggregates an index's time series data and stores the results in another index. +For example, you can roll up hourly data into daily or weekly summaries. +// end::rollup-def[] + +endif::[] +ifndef::permanently-unreleased-branch[] + [[glossary-rollup]] rollup :: // tag::rollup-def[] Summarize high-granularity data into a more compressed format to @@ -466,6 +477,8 @@ index the summaries into a separate rollup index. The job configuration controls what information is rolled up and how often. // end::rollup-job-def[] +endif::[] + [[glossary-routing]] routing :: + -- From eac210436b4475b776eebe1e48d71c6a4a5ad428 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 1 Dec 2020 09:00:45 -0500 Subject: [PATCH 08/27] [DOCS] Label legacy rollup APIs (#65518) --- .../reference/rollup/apis/delete-job.asciidoc | 20 ++++++++++++++++ docs/reference/rollup/apis/get-job.asciidoc | 19 +++++++++++++++ docs/reference/rollup/apis/put-job.asciidoc | 24 +++++++++++++++++++ .../rollup/apis/rollup-caps.asciidoc | 19 +++++++++++++++ .../rollup/apis/rollup-index-caps.asciidoc | 19 +++++++++++++++ .../rollup/apis/rollup-search.asciidoc | 19 +++++++++++++++ docs/reference/rollup/apis/start-job.asciidoc | 20 ++++++++++++++++ docs/reference/rollup/apis/stop-job.asciidoc | 20 ++++++++++++++++ docs/reference/rollup/rollup-apis.asciidoc | 12 +++++----- 9 files changed, 166 insertions(+), 6 deletions(-) diff --git a/docs/reference/rollup/apis/delete-job.asciidoc b/docs/reference/rollup/apis/delete-job.asciidoc index 12ad52eceda3d..9f425f8f869fa 100644 --- a/docs/reference/rollup/apis/delete-job.asciidoc +++ b/docs/reference/rollup/apis/delete-job.asciidoc @@ -1,3 +1,21 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-delete-job]] +=== Delete legacy {rollup-jobs} API +[subs="attributes"] +++++ +Delete legacy {rollup-jobs} +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Deletes a legacy {rollup-job}. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-delete-job]] @@ -11,6 +29,8 @@ Deletes an existing {rollup-job}. experimental[] +endif::[] + [[rollup-delete-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc index f29938d054f77..df0c41ad17b4a 100644 --- a/docs/reference/rollup/apis/get-job.asciidoc +++ b/docs/reference/rollup/apis/get-job.asciidoc @@ -1,3 +1,20 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-get-job]] +=== Get legacy {rollup-jobs} API +++++ +Get legacy job +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Gets the configuration, stats, and status of legacy {rollup-jobs}. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-get-job]] @@ -10,6 +27,8 @@ Retrieves the configuration, stats, and status of {rollup-jobs}. experimental[] +endif::[] + [[rollup-get-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc index 48a71f4ced155..e728f790f0f0a 100644 --- a/docs/reference/rollup/apis/put-job.asciidoc +++ b/docs/reference/rollup/apis/put-job.asciidoc @@ -1,3 +1,25 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-put-job]] +=== Create legacy {rollup-jobs} API +[subs="attributes"] +++++ +Create legacy {rollup-jobs} +++++ + +// tag::legacy-rollup-admon[] +WARNING: This documentation is about legacy rollups. Legacy rollups are +deprecated and will be replaced by new rollup functionality introduced in {es} +7.x. +// end::legacy-rollup-admon[] + +Creates a legacy {rollup-job}. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-put-job]] @@ -11,6 +33,8 @@ Creates a {rollup-job}. experimental[] +endif::[] + [[rollup-put-job-api-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/rollup-caps.asciidoc b/docs/reference/rollup/apis/rollup-caps.asciidoc index 1d0e620a94f55..e4b5d2b60229b 100644 --- a/docs/reference/rollup/apis/rollup-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-caps.asciidoc @@ -1,3 +1,20 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-get-rollup-caps]] +=== Get legacy {rollup-job} capabilities API +++++ +Get legacy rollup caps +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Returns the capabilities of legacy {rollup-jobs} for an index pattern. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-get-rollup-caps]] @@ -11,6 +28,8 @@ specific index or index pattern. experimental[] +endif::[] + [[rollup-get-rollup-caps-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/rollup-index-caps.asciidoc b/docs/reference/rollup/apis/rollup-index-caps.asciidoc index b8cce32db5d15..c8a93a3e0bec4 100644 --- a/docs/reference/rollup/apis/rollup-index-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-index-caps.asciidoc @@ -1,3 +1,20 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-get-rollup-index-caps]] +=== Get legacy rollup index capabilities API +++++ +Get legacy rollup index caps +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Returns the capabilities of rollup jobs for a legacy rollup index. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-get-rollup-index-caps]] @@ -11,6 +28,8 @@ index where rollup data is stored). experimental[] +endif::[] + [[rollup-get-rollup-index-caps-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc index 19bfb125467cf..d28ddc30d7905 100644 --- a/docs/reference/rollup/apis/rollup-search.asciidoc +++ b/docs/reference/rollup/apis/rollup-search.asciidoc @@ -1,3 +1,20 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-search]] +=== Legacy rollup search +++++ +Legacy rollup search +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Searches legacy rollup data using <>. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-search]] @@ -10,6 +27,8 @@ Enables searching rolled-up data using the standard query DSL. experimental[] +endif::[] + [[rollup-search-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/start-job.asciidoc b/docs/reference/rollup/apis/start-job.asciidoc index 28b09375dab85..e12ea8d998b44 100644 --- a/docs/reference/rollup/apis/start-job.asciidoc +++ b/docs/reference/rollup/apis/start-job.asciidoc @@ -1,3 +1,21 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-start-job]] +=== Start legacy {rollup-jobs} API +[subs="attributes"] +++++ +Start legacy {rollup-jobs} +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Starts a stopped legacy {rollup-job}. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-start-job]] @@ -11,6 +29,8 @@ Starts an existing, stopped {rollup-job}. experimental[] +endif::[] + [[rollup-start-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/stop-job.asciidoc b/docs/reference/rollup/apis/stop-job.asciidoc index 727981265cb10..4f5b07d2ad307 100644 --- a/docs/reference/rollup/apis/stop-job.asciidoc +++ b/docs/reference/rollup/apis/stop-job.asciidoc @@ -1,3 +1,21 @@ +ifdef::permanently-unreleased-branch[] + +[role="xpack"] +[testenv="basic"] +[[rollup-stop-job]] +=== Stop legacy {rollup-jobs} API +[subs="attributes"] +++++ +Stop legacy {rollup-jobs} +++++ + +include::put-job.asciidoc[tag=legacy-rollup-admon] + +Stops an ongoing legacy {rollup-job}. + +endif::[] +ifndef::permanently-unreleased-branch[] + [role="xpack"] [testenv="basic"] [[rollup-stop-job]] @@ -11,6 +29,8 @@ Stops an existing, started {rollup-job}. experimental[] +endif::[] + [[rollup-stop-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/rollup-apis.asciidoc b/docs/reference/rollup/rollup-apis.asciidoc index 00ede5729eb55..a8791b40d3af9 100644 --- a/docs/reference/rollup/rollup-apis.asciidoc +++ b/docs/reference/rollup/rollup-apis.asciidoc @@ -24,22 +24,22 @@ release. [[rollup-jobs-endpoint]] ==== Jobs -* <> or <> -* <> or <> -* <> +* <> or <> +* <> or <> +* <> [discrete] [[rollup-data-endpoint]] ==== Data -* <> -* <> +* <> +* <> [discrete] [[rollup-search-endpoint]] ==== Search -* <> +* <> include::apis/rollup-api.asciidoc[] include::apis/put-job.asciidoc[] From 318340a16b74533642c280db727fabe9ae6b850b Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 1 Dec 2020 16:16:30 +0200 Subject: [PATCH 09/27] [ML] Reduce log noise in BaseMlIntegTestCase (#65675) Tests inheriting `BaseMlIntegTestCase` have noisy logs with the error `No processor type exists with name [...]`. Those are caused because of loaded templates that need the `IngestCommonPlugin` loaded. This commit adds a test dependency to the `ingest-common` module and loads `IngestCommonPlugin` in `BaseMlIntegTestCase` to reduce such noise. --- x-pack/plugin/ml/build.gradle | 1 + .../org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index 10537d3ba8344..5d073e90ed40b 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -52,6 +52,7 @@ dependencies { testImplementation project(path: xpackModule('ilm'), configuration: 'default') compileOnly project(path: xpackModule('autoscaling'), configuration: 'default') testImplementation project(path: xpackModule('data-streams'), configuration: 'default') + testImplementation project(':modules:ingest-common') // This should not be here testImplementation project(path: xpackModule('security'), configuration: 'testArtifacts') diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 81fba17277e63..fdc3dacd2cd59 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.license.LicenseService; import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.plugins.Plugin; @@ -111,6 +112,7 @@ protected Collection> nodePlugins() { return Arrays.asList( LocalStateMachineLearning.class, CommonAnalysisPlugin.class, + IngestCommonPlugin.class, ReindexPlugin.class, // ILM is required for .ml-state template index settings IndexLifecycle.class, From 9ff95ebddd8d5dccce30d7537a37a4b9a3a807bb Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 1 Dec 2020 16:16:41 +0200 Subject: [PATCH 10/27] [ML] Add unit tests for DatafeedContext and RestartTimeInfo (#65674) These were meant to go with #65630 but were left out. --- .../ml/datafeed/DatafeedContextTests.java | 87 +++++++++++++++++++ .../job/persistence/RestartTimeInfoTests.java | 87 +++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextTests.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfoTests.java diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextTests.java new file mode 100644 index 0000000000000..f99348c712488 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedContextTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.persistence.RestartTimeInfo; + +import java.util.Date; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DatafeedContextTests extends ESTestCase { + + public void testShouldRecoverFromCurrentSnapshot_GivenDatafeedStartTimeIsAfterJobCheckpoint() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 20L, true); + ModelSnapshot modelSnapshot = newSnapshot(10L, 10L); + + DatafeedContext context = createContext(20L, restartTimeInfo, modelSnapshot); + + assertThat(context.shouldRecoverFromCurrentSnapshot(), is(false)); + } + + public void testShouldRecoverFromCurrentSnapshot_GivenRestartTimeInfoIsAfterNonNullSnapshot() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 20L, true); + ModelSnapshot modelSnapshot = newSnapshot(10L, 10L); + + DatafeedContext context = createContext(10L, restartTimeInfo, modelSnapshot); + + assertThat(context.shouldRecoverFromCurrentSnapshot(), is(true)); + } + + public void testShouldRecoverFromCurrentSnapshot_GivenHaveSeenDataBeforeAndNullSnapshot() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 20L, true); + + DatafeedContext context = createContext(10L, restartTimeInfo, null); + + assertThat(context.shouldRecoverFromCurrentSnapshot(), is(true)); + } + + public void testShouldRecoverFromCurrentSnapshot_GivenHaveNotSeenDataBeforeAndNullSnapshot() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(null, null, false); + + DatafeedContext context = createContext(10L, restartTimeInfo, null); + + assertThat(context.shouldRecoverFromCurrentSnapshot(), is(false)); + } + + public void testShouldRecoverFromCurrentSnapshot_GivenRestartTimeInfoMatchesSnapshot() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 20L, true); + ModelSnapshot modelSnapshot = newSnapshot(20L, 20L); + + DatafeedContext context = createContext(10L, restartTimeInfo, modelSnapshot); + + assertThat(context.shouldRecoverFromCurrentSnapshot(), is(false)); + } + + private static DatafeedContext createContext(long datafeedStartTime, RestartTimeInfo restartTimeInfo, + @Nullable ModelSnapshot modelSnapshot) { + Job job = JobTests.createRandomizedJob(); + return DatafeedContext.builder(datafeedStartTime) + .setJob(job) + .setDatafeedConfig(DatafeedConfigTests.createRandomizedDatafeedConfig(job.getId())) + .setTimingStats(new DatafeedTimingStats(job.getId())) + .setRestartTimeInfo(restartTimeInfo) + .setModelSnapshot(modelSnapshot) + .build(); + } + + private static ModelSnapshot newSnapshot(@Nullable Long latestResultTime, @Nullable Long latestRecordTime) { + ModelSnapshot snapshot = mock(ModelSnapshot.class); + when(snapshot.getLatestResultTimeStamp()).thenReturn(latestResultTime == null ? null : new Date(latestResultTime)); + when(snapshot.getLatestRecordTimeStamp()).thenReturn(latestRecordTime == null ? null : new Date(latestRecordTime)); + return snapshot; + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfoTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfoTests.java new file mode 100644 index 0000000000000..f281003519d17 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/RestartTimeInfoTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; + +import java.util.Date; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RestartTimeInfoTests extends ESTestCase { + + public void testIsAfter_GivenNullsAndTimestampIsZero() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(null, null, false); + assertThat(restartTimeInfo.isAfter(0), is(false)); + } + + public void testIsAfter_GivenNullsAndTimestampIsNonZero() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(null, null, false); + assertThat(restartTimeInfo.isAfter(1L), is(false)); + } + + public void testIsAfter_GivenTimestampIsBeforeFinalBucketButAfterLatestRecord() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(10L, 20L, true); + assertThat(restartTimeInfo.isAfter(15L), is(true)); + } + + public void testIsAfter_GivenTimestampIsAfterFinalBucketButBeforeLatestRecord() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 10L, true); + assertThat(restartTimeInfo.isAfter(15L), is(true)); + } + + public void testIsAfter_GivenTimestampIsAfterFinalBucketAndAfterLatestRecord() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 10L, true); + assertThat(restartTimeInfo.isAfter(30L), is(false)); + } + + public void testIsAfter_GivenTimestampIsBeforeFinalBucketAndBeforeLatestRecord() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 10L, true); + assertThat(restartTimeInfo.isAfter(5L), is(true)); + } + + public void testIsAfterModelSnapshot_GivenNulls() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(null, null, false); + ModelSnapshot snapshot = newSnapshot(null, null); + assertThat(restartTimeInfo.isAfterModelSnapshot(snapshot), is(false)); + } + + public void testIsAfterModelSnapshot_GivenModelSnapshotLatestRecordTimeIsBefore() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 30L, true); + ModelSnapshot snapshot = newSnapshot(40L, 25L); + assertThat(restartTimeInfo.isAfterModelSnapshot(snapshot), is(true)); + } + + public void testIsAfterModelSnapshot_GivenModelSnapshotLatestResultTimeIsBefore() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 30L, true); + ModelSnapshot snapshot = newSnapshot(15L, 35L); + assertThat(restartTimeInfo.isAfterModelSnapshot(snapshot), is(true)); + } + + public void testIsAfterModelSnapshot_GivenModelSnapshotIsAfter() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 30L, true); + ModelSnapshot snapshot = newSnapshot(30L, 35L); + assertThat(restartTimeInfo.isAfterModelSnapshot(snapshot), is(false)); + } + + public void testIsAfterModelSnapshot_GivenModelSnapshotMatches() { + RestartTimeInfo restartTimeInfo = new RestartTimeInfo(20L, 30L, true); + ModelSnapshot snapshot = newSnapshot(20L, 30L); + assertThat(restartTimeInfo.isAfterModelSnapshot(snapshot), is(false)); + } + + private static ModelSnapshot newSnapshot(@Nullable Long latestResultTime, @Nullable Long latestRecordTime) { + ModelSnapshot snapshot = mock(ModelSnapshot.class); + when(snapshot.getLatestResultTimeStamp()).thenReturn(latestResultTime == null ? null : new Date(latestResultTime)); + when(snapshot.getLatestRecordTimeStamp()).thenReturn(latestRecordTime == null ? null : new Date(latestRecordTime)); + return snapshot; + } +} From b902d807039d3adc3ae16f14af931d9748b0e92f Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 1 Dec 2020 15:41:48 +0000 Subject: [PATCH 11/27] Record timestamp field range in index metadata (#65564) Queries including a filter by timestamp range are common in time-series data. Moreover older time-series indices are typically made read-only so that the timestamp range becomes immutable. By recording in the index metadata the range of timestamps covered by each index we can very efficiently skip shards on the coordinating node, even if those shards are not assigned. This commit computes the timestamp range of immutable indices and records it in the index metadata as the shards start for the first time. Note that the only indices it considers immutable today are ones using the `ReadOnlyEngine`, which includes frozen indices and searchable snapshots but not regular indices with a write block. --- .../action/shard/ShardStateAction.java | 69 +++- .../cluster/metadata/IndexMetadata.java | 39 +- .../metadata/MetadataIndexStateService.java | 3 + .../allocation/IndexMetadataUpdater.java | 2 + .../elasticsearch/index/engine/Engine.java | 9 + .../index/engine/InternalEngine.java | 6 + .../index/engine/ReadOnlyEngine.java | 26 ++ .../index/mapper/DateFieldMapper.java | 37 +- .../index/shard/IndexLongFieldRange.java | 370 ++++++++++++++++++ .../elasticsearch/index/shard/IndexShard.java | 43 +- .../index/shard/ShardLongFieldRange.java | 141 +++++++ .../cluster/IndicesClusterStateService.java | 28 +- .../recovery/PeerRecoveryTargetService.java | 3 +- .../indices/recovery/RecoveryTarget.java | 2 +- .../snapshots/RestoreService.java | 5 +- .../reroute/ClusterRerouteResponseTests.java | 10 +- .../cluster/ClusterStateTests.java | 20 +- ...dStartedClusterStateTaskExecutorTests.java | 92 ++++- .../action/shard/ShardStateActionTests.java | 17 +- .../metadata/ToAndFromJsonMetadataTests.java | 15 +- .../index/mapper/DateFieldMapperTests.java | 34 ++ .../shard/IndexLongFieldRangeTestUtils.java | 77 ++++ .../index/shard/IndexLongFieldRangeTests.java | 133 +++++++ .../shard/IndexLongFieldRangeWireTests.java | 70 ++++ .../IndexLongFieldRangeXContentTests.java | 54 +++ .../shard/ShardLongFieldRangeWireTests.java | 94 +++++ ...actIndicesClusterStateServiceTestCase.java | 7 + .../indices/cluster/ClusterStateChanges.java | 8 +- .../indices/recovery/RecoveryTests.java | 3 +- .../recovery/RecoveriesCollectionTests.java | 5 +- .../ClusterStateCreationUtils.java | 4 + .../index/shard/IndexShardTestCase.java | 2 +- .../index/engine/FrozenIndexIT.java | 100 +++++ .../index/engine/FrozenIndexTests.java | 80 +++- .../SearchableSnapshotsIntegTests.java | 117 ++++++ 35 files changed, 1675 insertions(+), 50 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java create mode 100644 server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTestUtils.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeXContentTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java create mode 100644 x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexIT.java diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 98b325b35419c..c5bea146dc0ad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.action.shard; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -35,7 +36,9 @@ import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; @@ -48,6 +51,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; @@ -65,9 +71,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; @@ -473,16 +481,23 @@ public int hashCode() { public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, + final ShardLongFieldRange timestampMillisRange, final ActionListener listener) { - shardStarted(shardRouting, primaryTerm, message, listener, clusterService.state()); + shardStarted(shardRouting, primaryTerm, message, timestampMillisRange, listener, clusterService.state()); } public void shardStarted(final ShardRouting shardRouting, final long primaryTerm, final String message, + final ShardLongFieldRange timestampMillisRange, final ActionListener listener, final ClusterState currentState) { - StartedShardEntry entry = new StartedShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), primaryTerm, message); + final StartedShardEntry entry = new StartedShardEntry( + shardRouting.shardId(), + shardRouting.allocationId().getId(), + primaryTerm, + message, + timestampMillisRange); sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, entry, listener); } @@ -529,6 +544,7 @@ public ClusterTasksResult execute(ClusterState currentState, List tasksToBeApplied = new ArrayList<>(); List shardRoutingsToBeApplied = new ArrayList<>(tasks.size()); Set seenShardRoutings = new HashSet<>(); // to prevent duplicates + final Map updatedTimestampRanges = new HashMap<>(); for (StartedShardEntry task : tasks) { final ShardRouting matched = currentState.getRoutingTable().getByAllocationId(task.shardId, task.allocationId); if (matched == null) { @@ -569,6 +585,22 @@ public ClusterTasksResult execute(ClusterState currentState, tasksToBeApplied.add(task); shardRoutingsToBeApplied.add(matched); seenShardRoutings.add(matched); + + // expand the timestamp range recorded in the index metadata if needed + final Index index = task.shardId.getIndex(); + IndexLongFieldRange currentTimestampMillisRange = updatedTimestampRanges.get(index); + final IndexMetadata indexMetadata = currentState.metadata().index(index); + if (currentTimestampMillisRange == null) { + currentTimestampMillisRange = indexMetadata.getTimestampMillisRange(); + } + final IndexLongFieldRange newTimestampMillisRange; + newTimestampMillisRange = currentTimestampMillisRange.extendWithShardRange( + task.shardId.id(), + indexMetadata.getNumberOfShards(), + task.timestampMillisRange); + if (newTimestampMillisRange != currentTimestampMillisRange) { + updatedTimestampRanges.put(index, newTimestampMillisRange); + } } } } @@ -578,6 +610,19 @@ public ClusterTasksResult execute(ClusterState currentState, ClusterState maybeUpdatedState = currentState; try { maybeUpdatedState = allocationService.applyStartedShards(currentState, shardRoutingsToBeApplied); + + if (updatedTimestampRanges.isEmpty() == false) { + final Metadata.Builder metadataBuilder = Metadata.builder(maybeUpdatedState.metadata()); + for (Map.Entry updatedTimestampRangeEntry : updatedTimestampRanges.entrySet()) { + metadataBuilder.put(IndexMetadata + .builder(metadataBuilder.getSafe(updatedTimestampRangeEntry.getKey())) + .timestampMillisRange(updatedTimestampRangeEntry.getValue())); + } + maybeUpdatedState = ClusterState.builder(maybeUpdatedState).metadata(metadataBuilder).build(); + } + + assert assertStartedIndicesHaveCompleteTimestampRanges(maybeUpdatedState); + builder.successes(tasksToBeApplied); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("failed to apply started shards {}", shardRoutingsToBeApplied), e); @@ -587,6 +632,16 @@ public ClusterTasksResult execute(ClusterState currentState, return builder.build(maybeUpdatedState); } + private static boolean assertStartedIndicesHaveCompleteTimestampRanges(ClusterState clusterState) { + for (ObjectObjectCursor cursor : clusterState.getRoutingTable().getIndicesRouting()) { + assert cursor.value.allPrimaryShardsActive() == false + || clusterState.metadata().index(cursor.key).getTimestampMillisRange().isComplete() + : "index [" + cursor.key + "] should have complete timestamp range, but got " + + clusterState.metadata().index(cursor.key).getTimestampMillisRange() + " for " + cursor.value.prettyPrint(); + } + return true; + } + @Override public void onFailure(String source, Exception e) { if (e instanceof FailedToCommitClusterStateException || e instanceof NotMasterException) { @@ -609,6 +664,7 @@ public static class StartedShardEntry extends TransportRequest { final String allocationId; final long primaryTerm; final String message; + final ShardLongFieldRange timestampMillisRange; StartedShardEntry(StreamInput in) throws IOException { super(in); @@ -616,13 +672,19 @@ public static class StartedShardEntry extends TransportRequest { allocationId = in.readString(); primaryTerm = in.readVLong(); this.message = in.readString(); + this.timestampMillisRange = ShardLongFieldRange.readFrom(in); } - public StartedShardEntry(final ShardId shardId, final String allocationId, final long primaryTerm, final String message) { + public StartedShardEntry(final ShardId shardId, + final String allocationId, + final long primaryTerm, + final String message, + final ShardLongFieldRange timestampMillisRange) { this.shardId = shardId; this.allocationId = allocationId; this.primaryTerm = primaryTerm; this.message = message; + this.timestampMillisRange = timestampMillisRange; } @Override @@ -632,6 +694,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(allocationId); out.writeVLong(primaryTerm); out.writeString(message); + timestampMillisRange.writeTo(out); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 1cfd0c7db4975..85f4aca4f4a3d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -55,6 +55,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; @@ -341,6 +342,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException { static final String KEY_ALIASES = "aliases"; static final String KEY_ROLLOVER_INFOS = "rollover_info"; static final String KEY_SYSTEM = "system"; + static final String KEY_TIMESTAMP_RANGE = "timestamp_range"; public static final String KEY_PRIMARY_TERMS = "primary_terms"; public static final String INDEX_STATE_FILE_PREFIX = "state-"; @@ -391,6 +393,8 @@ public static APIBlock readFrom(StreamInput input) throws IOException { private final ImmutableOpenMap rolloverInfos; private final boolean isSystem; + private final IndexLongFieldRange timestampMillisRange; + private IndexMetadata( final Index index, final long version, @@ -416,7 +420,8 @@ private IndexMetadata( final int routingPartitionSize, final ActiveShardCount waitForActiveShards, final ImmutableOpenMap rolloverInfos, - final boolean isSystem) { + final boolean isSystem, + final IndexLongFieldRange timestampMillisRange) { this.index = index; this.version = version; @@ -449,6 +454,7 @@ private IndexMetadata( this.waitForActiveShards = waitForActiveShards; this.rolloverInfos = rolloverInfos; this.isSystem = isSystem; + this.timestampMillisRange = timestampMillisRange; assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards; } @@ -621,6 +627,10 @@ public DiscoveryNodeFilters excludeFilters() { return excludeFilters; } + public IndexLongFieldRange getTimestampMillisRange() { + return timestampMillisRange; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -730,6 +740,7 @@ private static class IndexMetadataDiff implements Diff { private final Diff>> inSyncAllocationIds; private final Diff> rolloverInfos; private final boolean isSystem; + private final IndexLongFieldRange timestampMillisRange; IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); @@ -748,6 +759,7 @@ private static class IndexMetadataDiff implements Diff { DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer()); isSystem = after.isSystem; + timestampMillisRange = after.timestampMillisRange; } private static final DiffableUtils.DiffableValueReader ALIAS_METADATA_DIFF_VALUE_READER = @@ -785,6 +797,7 @@ private static class IndexMetadataDiff implements Diff { } else { isSystem = false; } + timestampMillisRange = IndexLongFieldRange.readFrom(in); } @Override @@ -808,6 +821,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { out.writeBoolean(isSystem); } + timestampMillisRange.writeTo(out); } @Override @@ -827,6 +841,7 @@ public IndexMetadata apply(IndexMetadata part) { builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds)); builder.rolloverInfos.putAll(rolloverInfos.apply(part.rolloverInfos)); builder.system(part.isSystem); + builder.timestampMillisRange(timestampMillisRange); return builder.build(); } } @@ -872,6 +887,7 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { builder.system(in.readBoolean()); } + builder.timestampMillisRange(IndexLongFieldRange.readFrom(in)); return builder.build(); } @@ -913,6 +929,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) { out.writeBoolean(isSystem); } + timestampMillisRange.writeTo(out); } public boolean isSystem() { @@ -944,6 +961,7 @@ public static class Builder { private final ImmutableOpenMap.Builder rolloverInfos; private Integer routingNumShards; private boolean isSystem; + private IndexLongFieldRange timestampMillisRange = IndexLongFieldRange.NO_SHARDS; public Builder(String index) { this.index = index; @@ -971,6 +989,7 @@ public Builder(IndexMetadata indexMetadata) { this.inSyncAllocationIds = ImmutableOpenIntMap.builder(indexMetadata.inSyncAllocationIds); this.rolloverInfos = ImmutableOpenMap.builder(indexMetadata.rolloverInfos); this.isSystem = indexMetadata.isSystem; + this.timestampMillisRange = indexMetadata.timestampMillisRange; } public Builder index(String index) { @@ -1183,6 +1202,15 @@ public boolean isSystem() { return isSystem; } + public Builder timestampMillisRange(IndexLongFieldRange timestampMillisRange) { + this.timestampMillisRange = timestampMillisRange; + return this; + } + + public IndexLongFieldRange getTimestampMillisRange() { + return timestampMillisRange; + } + public IndexMetadata build() { ImmutableOpenMap.Builder tmpAliases = aliases; Settings tmpSettings = settings; @@ -1288,7 +1316,8 @@ public IndexMetadata build() { routingPartitionSize, waitForActiveShards, rolloverInfos.build(), - isSystem); + isSystem, + timestampMillisRange); } public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -1390,6 +1419,10 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build builder.endObject(); builder.field(KEY_SYSTEM, indexMetadata.isSystem); + builder.startObject(KEY_TIMESTAMP_RANGE); + indexMetadata.timestampMillisRange.toXContent(builder, params); + builder.endObject(); + builder.endObject(); } @@ -1470,6 +1503,8 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti // simply ignored when upgrading from 2.x assert Version.CURRENT.major <= 5; parser.skipChildren(); + } else if (KEY_TIMESTAMP_RANGE.equals(currentFieldName)) { + builder.timestampMillisRange(IndexLongFieldRange.fromXContent(parser)); } else { // assume it's custom index metadata builder.putCustom(currentFieldName, parser.mapStrings()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 8c3f1ad09190a..16bdb9f48c235 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -69,6 +69,7 @@ import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.ShardLimitValidator; @@ -759,6 +760,7 @@ static Tuple> closeRoutingTable(final Clus blocks.addIndexBlock(index.getName(), INDEX_CLOSED_BLOCK); final IndexMetadata.Builder updatedMetadata = IndexMetadata.builder(indexMetadata).state(IndexMetadata.State.CLOSE); metadata.put(updatedMetadata + .timestampMillisRange(IndexLongFieldRange.NO_SHARDS) .settingsVersion(indexMetadata.getSettingsVersion() + 1) .settings(Settings.builder() .put(indexMetadata.getSettings()) @@ -847,6 +849,7 @@ ClusterState openIndices(final Index[] indices, final ClusterState currentState) .state(IndexMetadata.State.OPEN) .settingsVersion(indexMetadata.getSettingsVersion() + 1) .settings(updatedSettings) + .timestampMillisRange(IndexLongFieldRange.NO_SHARDS) .build(); // The index might be closed because we couldn't import it due to old incompatible version diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java index 9b07766ab09c1..d9d2caf738217 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java @@ -170,6 +170,8 @@ private IndexMetadata.Builder updateInSyncAllocations(RoutingTable newRoutingTab final String allocationId; if (recoverySource == RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE) { allocationId = RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID; + indexMetadataBuilder.timestampMillisRange(indexMetadataBuilder.getTimestampMillisRange() + .removeShard(shardId.id(), oldIndexMetadata.getNumberOfShards())); } else { assert recoverySource instanceof RecoverySource.SnapshotRecoverySource : recoverySource; allocationId = updates.initializedPrimary.allocationId().getId(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 4042203a4fa05..5259c1da80a48 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -70,6 +70,7 @@ import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; @@ -1861,4 +1862,12 @@ public interface TranslogRecoveryRunner { * to advance this marker to at least the given sequence number. */ public abstract void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary); + + /** + * @return a {@link ShardLongFieldRange} containing the min and max raw values of the given field for this shard if the engine + * guarantees these values never to change, or {@link ShardLongFieldRange#EMPTY} if this field is empty, or + * {@link ShardLongFieldRange#UNKNOWN} if this field's value range may change in future. + */ + public abstract ShardLongFieldRange getRawFieldRange(String field) throws IOException; + } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index d25a069f826fb..02981f4794637 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -87,6 +87,7 @@ import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ElasticsearchMergePolicy; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; @@ -2765,4 +2766,9 @@ SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE), Boole refresh("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL, true); } + @Override + public ShardLongFieldRange getRawFieldRange(String field) { + return ShardLongFieldRange.UNKNOWN; + } + } diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java index c9d9baaef0d26..899863ee7c478 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java @@ -18,9 +18,11 @@ */ package org.elasticsearch.index.engine; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper; import org.apache.lucene.search.ReferenceManager; @@ -35,6 +37,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; @@ -497,4 +500,27 @@ protected static DirectoryReader openDirectory(Directory directory) throws IOExc public CompletionStats completionStats(String... fieldNamePatterns) { return completionStatsCache.get(fieldNamePatterns); } + + /** + * @return a {@link ShardLongFieldRange} containing the min and max raw values of the given field for this shard, or {@link + * ShardLongFieldRange#EMPTY} if this field is not found or empty. + */ + @Override + public ShardLongFieldRange getRawFieldRange(String field) throws IOException { + try (Searcher searcher = acquireSearcher("field_range")) { + final DirectoryReader directoryReader = searcher.getDirectoryReader(); + + final byte[] minPackedValue = PointValues.getMinPackedValue(directoryReader, field); + final byte[] maxPackedValue = PointValues.getMaxPackedValue(directoryReader, field); + + if (minPackedValue == null || maxPackedValue == null) { + assert minPackedValue == null && maxPackedValue == null + : Arrays.toString(minPackedValue) + "-" + Arrays.toString(maxPackedValue); + return ShardLongFieldRange.EMPTY; + } + + return ShardLongFieldRange.of(LongPoint.decodeDimension(minPackedValue, 0), LongPoint.decodeDimension(maxPackedValue, 0)); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d0fa0d9ccffea..478cad469d357 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -96,6 +96,16 @@ public long parsePointAsMillis(byte[] value) { return LongPoint.decodeDimension(value, 0); } + @Override + public long roundDownToMillis(long value) { + return value; + } + + @Override + public long roundUpToMillis(long value) { + return value; + } + @Override protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis()); @@ -119,7 +129,22 @@ public Instant clampToValidRange(Instant instant) { @Override public long parsePointAsMillis(byte[] value) { - return DateUtils.toMilliSeconds(LongPoint.decodeDimension(value, 0)); + return roundDownToMillis(LongPoint.decodeDimension(value, 0)); + } + + @Override + public long roundDownToMillis(long value) { + return DateUtils.toMilliSeconds(value); + } + + @Override + public long roundUpToMillis(long value) { + if (value <= 0L) { + // if negative then throws an IAE; if zero then return zero + return DateUtils.toMilliSeconds(value); + } else { + return DateUtils.toMilliSeconds(value - 1L) + 1L; + } } @Override @@ -165,6 +190,16 @@ NumericType numericType() { */ public abstract long parsePointAsMillis(byte[] value); + /** + * Round the given raw value down to a number of milliseconds since the epoch. + */ + public abstract long roundDownToMillis(long value); + + /** + * Round the given raw value up to a number of milliseconds since the epoch. + */ + public abstract long roundUpToMillis(long value); + public static Resolution ofOrdinal(int ord) { for (Resolution resolution : values()) { if (ord == resolution.ordinal()) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java b/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java new file mode 100644 index 0000000000000..3f6a6b0be058f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java @@ -0,0 +1,370 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.IntStream; + +import static org.elasticsearch.index.shard.ShardLongFieldRange.LONG_FIELD_RANGE_VERSION_INTRODUCED; + +/** + * Class representing an (inclusive) range of {@code long} values in a field in an index which may comprise multiple shards. This + * information is accumulated shard-by-shard, and we keep track of which shards are represented in this value. Only once all shards are + * represented should this information be considered accurate for the index. + */ +public class IndexLongFieldRange implements Writeable, ToXContentFragment { + + /** + * Sentinel value indicating that no information is currently available, for instance because the index has just been created. + */ + public static final IndexLongFieldRange NO_SHARDS = new IndexLongFieldRange(new int[0], Long.MAX_VALUE, Long.MIN_VALUE); + + /** + * Sentinel value indicating an empty range, for instance because the field is missing or has no values in any shard. + */ + public static final IndexLongFieldRange EMPTY = new IndexLongFieldRange(null, Long.MAX_VALUE, Long.MIN_VALUE); + + /** + * Sentinel value indicating the actual range is unknown, for instance because more docs may be added in future. + */ + public static final IndexLongFieldRange UNKNOWN = new IndexLongFieldRange(null, Long.MIN_VALUE, Long.MAX_VALUE); + + @Nullable // if this range includes all shards + private final int[] shards; + private final long min, max; + + private IndexLongFieldRange(int[] shards, long min, long max) { + assert (min == Long.MAX_VALUE && max == Long.MIN_VALUE) || min <= max : min + " vs " + max; + assert shards == null || shards.length > 0 || (min == Long.MAX_VALUE && max == Long.MIN_VALUE) : Arrays.toString(shards); + assert shards == null || Arrays.equals(shards, Arrays.stream(shards).sorted().distinct().toArray()) : Arrays.toString(shards); + this.shards = shards; + this.min = min; + this.max = max; + } + + /** + * @return whether this range includes information from all shards yet. + */ + public boolean isComplete() { + return shards == null; + } + + // exposed for testing + int[] getShards() { + return shards; + } + + // exposed for testing + long getMinUnsafe() { + return min; + } + + // exposed for testing + long getMaxUnsafe() { + return max; + } + + /** + * @return the (inclusive) minimum of this range. + */ + public long getMin() { + assert shards == null : "min is meaningless if we don't have data from all shards yet"; + assert this != EMPTY : "min is meaningless if range is empty"; + assert this != UNKNOWN : "min is meaningless if range is unknown"; + return min; + } + + /** + * @return the (inclusive) maximum of this range. + */ + public long getMax() { + assert shards == null : "max is meaningless if we don't have data from all shards yet"; + assert this != EMPTY : "max is meaningless if range is empty"; + assert this != UNKNOWN : "max is meaningless if range is unknown"; + return max; + } + + private static final byte WIRE_TYPE_OTHER = (byte)0; + private static final byte WIRE_TYPE_NO_SHARDS = (byte)1; + private static final byte WIRE_TYPE_UNKNOWN = (byte)2; + private static final byte WIRE_TYPE_EMPTY = (byte)3; + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { + if (this == NO_SHARDS) { + out.writeByte(WIRE_TYPE_NO_SHARDS); + } else if (this == UNKNOWN) { + out.writeByte(WIRE_TYPE_UNKNOWN); + } else if (this == EMPTY) { + out.writeByte(WIRE_TYPE_EMPTY); + } else { + out.writeByte(WIRE_TYPE_OTHER); + if (shards == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeVIntArray(shards); + } + out.writeZLong(min); + out.writeZLong(max); + } + } + } + + public static IndexLongFieldRange readFrom(StreamInput in) throws IOException { + if (in.getVersion().before(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { + // conservative treatment for BWC + return UNKNOWN; + } + + final byte type = in.readByte(); + switch (type) { + case WIRE_TYPE_NO_SHARDS: + return NO_SHARDS; + case WIRE_TYPE_UNKNOWN: + return UNKNOWN; + case WIRE_TYPE_EMPTY: + return EMPTY; + case WIRE_TYPE_OTHER: + return new IndexLongFieldRange(in.readBoolean() ? in.readVIntArray() : null, in.readZLong(), in.readZLong()); + default: + throw new IllegalStateException("type [" + type + "] not known"); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (this == UNKNOWN) { + builder.field("unknown", true); + } else if (this == EMPTY) { + builder.field("empty", true); + } else if (this == NO_SHARDS) { + builder.startArray("shards"); + builder.endArray(); + } else { + builder.field("min", min); + builder.field("max", max); + if (shards != null) { + builder.startArray("shards"); + for (int shard : shards) { + builder.value(shard); + } + builder.endArray(); + } + } + return builder; + } + + public static IndexLongFieldRange fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token; + String currentFieldName = null; + Boolean isUnknown = null; + Boolean isEmpty = null; + Long min = null; + Long max = null; + List shardsList = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("unknown".equals(currentFieldName)) { + if (Boolean.FALSE.equals(isUnknown)) { + throw new IllegalArgumentException("unexpected field 'unknown'"); + } else { + isUnknown = Boolean.TRUE; + isEmpty = Boolean.FALSE; + } + } else if ("empty".equals(currentFieldName)) { + if (Boolean.FALSE.equals(isEmpty)) { + throw new IllegalArgumentException("unexpected field 'empty'"); + } else { + isUnknown = Boolean.FALSE; + isEmpty = Boolean.TRUE; + } + } else if ("min".equals(currentFieldName)) { + if (Boolean.TRUE.equals(isUnknown) || Boolean.TRUE.equals(isEmpty)) { + throw new IllegalArgumentException("unexpected field 'min'"); + } else { + isUnknown = Boolean.FALSE; + isEmpty = Boolean.FALSE; + min = parser.longValue(); + } + } else if ("max".equals(currentFieldName)) { + if (Boolean.TRUE.equals(isUnknown) || Boolean.TRUE.equals(isEmpty)) { + throw new IllegalArgumentException("unexpected field 'max'"); + } else { + isUnknown = Boolean.FALSE; + isEmpty = Boolean.FALSE; + max = parser.longValue(); + } + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("shards".equals(currentFieldName)) { + if (Boolean.TRUE.equals(isUnknown) || Boolean.TRUE.equals(isEmpty) || shardsList != null) { + throw new IllegalArgumentException("unexpected array 'shards'"); + } else { + isUnknown = Boolean.FALSE; + isEmpty = Boolean.FALSE; + shardsList = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token.isValue()) { + shardsList.add(parser.intValue()); + } + } + } + } else { + throw new IllegalArgumentException("Unexpected array: " + currentFieldName); + } + } else { + throw new IllegalArgumentException("Unexpected token: " + token); + } + } + + if (Boolean.TRUE.equals(isUnknown)) { + //noinspection ConstantConditions this assertion is always true but left here for the benefit of readers + assert min == null && max == null && shardsList == null && Boolean.FALSE.equals(isEmpty); + return UNKNOWN; + } else if (Boolean.TRUE.equals(isEmpty)) { + //noinspection ConstantConditions this assertion is always true but left here for the benefit of readers + assert min == null && max == null && shardsList == null && Boolean.FALSE.equals(isUnknown); + return EMPTY; + } else if (shardsList != null && shardsList.isEmpty()) { + //noinspection ConstantConditions this assertion is always true but left here for the benefit of readers + assert min == null && max == null && Boolean.FALSE.equals(isEmpty) && Boolean.FALSE.equals(isUnknown); + return NO_SHARDS; + } else if (min != null) { + //noinspection ConstantConditions this assertion is always true but left here for the benefit of readers + assert Boolean.FALSE.equals(isUnknown) && Boolean.FALSE.equals(isEmpty); + if (max == null) { + throw new IllegalArgumentException("field 'max' unexpectedly missing"); + } + final int[] shards; + if (shardsList != null) { + shards = shardsList.stream().mapToInt(i -> i).toArray(); + assert shards.length > 0; + } else { + shards = null; + } + return new IndexLongFieldRange(shards, min, max); + } else { + throw new IllegalArgumentException("field range contents unexpectedly missing"); + } + } + + public IndexLongFieldRange extendWithShardRange(int shardId, int shardCount, ShardLongFieldRange shardFieldRange) { + if (shardFieldRange == ShardLongFieldRange.UNKNOWN) { + assert shards == null + ? this == UNKNOWN + : Arrays.stream(shards).noneMatch(i -> i == shardId); + return UNKNOWN; + } + if (shards == null || Arrays.stream(shards).anyMatch(i -> i == shardId)) { + assert shardFieldRange == ShardLongFieldRange.EMPTY || min <= shardFieldRange.getMin() && shardFieldRange.getMax() <= max; + return this; + } + final int[] newShards; + if (shards.length == shardCount - 1) { + assert Arrays.equals(shards, IntStream.range(0, shardCount).filter(i -> i != shardId).toArray()) + : Arrays.toString(shards) + " + " + shardId; + if (shardFieldRange == ShardLongFieldRange.EMPTY && min == EMPTY.min && max == EMPTY.max) { + return EMPTY; + } + newShards = null; + } else { + newShards = IntStream.concat(Arrays.stream(this.shards), IntStream.of(shardId)).sorted().toArray(); + } + if (shardFieldRange == ShardLongFieldRange.EMPTY) { + return new IndexLongFieldRange(newShards, min, max); + } else { + return new IndexLongFieldRange(newShards, Math.min(shardFieldRange.getMin(), min), Math.max(shardFieldRange.getMax(), max)); + } + } + + @Override + public String toString() { + if (this == NO_SHARDS) { + return "NO_SHARDS"; + } else if (this == UNKNOWN) { + return "UNKNOWN"; + } else if (this == EMPTY) { + return "EMPTY"; + } else if (shards == null) { + return "[" + min + "-" + max + "]"; + } else { + return "[" + min + "-" + max + ", shards=" + Arrays.toString(shards) + "]"; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (this == EMPTY || this == UNKNOWN || this == NO_SHARDS || o == EMPTY || o == UNKNOWN || o == NO_SHARDS) return false; + IndexLongFieldRange that = (IndexLongFieldRange) o; + return min == that.min && + max == that.max && + Arrays.equals(shards, that.shards); + } + + @Override + public int hashCode() { + int result = Objects.hash(min, max); + result = 31 * result + Arrays.hashCode(shards); + return result; + } + + /** + * Remove the given shard from the set of known shards, possibly without adjusting the min and max. Used when allocating a stale primary + * which may have a different range from the original, so we must allow the range to grow. Note that this doesn't usually allow the + * range to shrink, so we may in theory hit this shard more than needed after allocating a stale primary. + */ + public IndexLongFieldRange removeShard(int shardId, int numberOfShards) { + assert 0 <= shardId && shardId < numberOfShards : shardId + " vs " + numberOfShards; + + if (shards != null && Arrays.stream(shards).noneMatch(i -> i == shardId)) { + return this; + } + if (shards == null && numberOfShards == 1) { + return NO_SHARDS; + } + if (this == UNKNOWN) { + return this; + } + if (shards != null && shards.length == 1 && shards[0] == shardId) { + return NO_SHARDS; + } + + final IntStream currentShards = shards == null ? IntStream.range(0, numberOfShards) : Arrays.stream(shards); + return new IndexLongFieldRange(currentShards.filter(i -> i != shardId).toArray(), min, max); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index f5109b7ca9ab1..8cbd7b6e54362 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -49,6 +49,7 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.support.replication.PendingReplicationActions; import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -108,9 +109,11 @@ import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.get.ShardGetService; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperForType; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.ParsedDocument; @@ -1715,6 +1718,44 @@ public RecoveryState recoveryState() { return this.recoveryState; } + @Override + public ShardLongFieldRange getTimestampMillisRange() { + assert isReadAllowed(); + + if (mapperService() == null) { + return ShardLongFieldRange.UNKNOWN; // no mapper service, no idea if the field even exists + } + final MappedFieldType mappedFieldType = mapperService().fieldType(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + if (mappedFieldType instanceof DateFieldMapper.DateFieldType == false) { + return ShardLongFieldRange.UNKNOWN; // field missing or not a date + } + final DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) mappedFieldType; + + final Engine engine = getEngine(); + final ShardLongFieldRange rawTimestampFieldRange; + try { + rawTimestampFieldRange = engine.getRawFieldRange(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + } catch (IOException e) { + logger.debug("exception obtaining range for timestamp field", e); + return ShardLongFieldRange.UNKNOWN; + } + if (rawTimestampFieldRange == ShardLongFieldRange.UNKNOWN) { + return ShardLongFieldRange.UNKNOWN; + } + if (rawTimestampFieldRange == ShardLongFieldRange.EMPTY) { + return ShardLongFieldRange.EMPTY; + } + + try { + return ShardLongFieldRange.of( + dateFieldType.resolution().roundDownToMillis(rawTimestampFieldRange.getMin()), + dateFieldType.resolution().roundUpToMillis(rawTimestampFieldRange.getMax())); + } catch (IllegalArgumentException e) { + logger.debug(new ParameterizedMessage("could not convert {} to a millisecond time range", rawTimestampFieldRange), e); + return ShardLongFieldRange.UNKNOWN; // any search might match this shard + } + } + /** * perform the last stages of recovery once all translog operations are done. * note that you should still call {@link #postRecovery(String)}. @@ -2630,7 +2671,7 @@ private void executeRecovery(String reason, RecoveryState recoveryState, PeerRec markAsRecovering(reason, recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(ActionRunnable.wrap(ActionListener.wrap(r -> { if (r) { - recoveryListener.onRecoveryDone(recoveryState); + recoveryListener.onRecoveryDone(recoveryState, getTimestampMillisRange()); } }, e -> recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true)), action)); diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java b/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java new file mode 100644 index 0000000000000..b76cf2d89bc9c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * Class representing an (inclusive) range of {@code long} values in a field in a single shard. + */ +public class ShardLongFieldRange implements Writeable { + + public static final Version LONG_FIELD_RANGE_VERSION_INTRODUCED = Version.V_8_0_0; + + /** + * Sentinel value indicating an empty range, for instance because the field is missing or has no values. + */ + public static final ShardLongFieldRange EMPTY = new ShardLongFieldRange(Long.MAX_VALUE, Long.MIN_VALUE); + + /** + * Sentinel value indicating the actual range is unknown, for instance because more docs may be added in future. + */ + public static final ShardLongFieldRange UNKNOWN = new ShardLongFieldRange(Long.MIN_VALUE, Long.MAX_VALUE); + + /** + * Construct a new {@link ShardLongFieldRange} with the given (inclusive) minimum and maximum. + */ + public static ShardLongFieldRange of(long min, long max) { + assert min <= max : min + " vs " + max; + return new ShardLongFieldRange(min, max); + } + + private final long min, max; + + private ShardLongFieldRange(long min, long max) { + this.min = min; + this.max = max; + } + + /** + * @return the (inclusive) minimum of this range. + */ + public long getMin() { + assert this != EMPTY && this != UNKNOWN && min <= max: "must not use actual min of sentinel values"; + return min; + } + + /** + * @return the (inclusive) maximum of this range. + */ + public long getMax() { + assert this != EMPTY && this != UNKNOWN && min <= max : "must not use actual max of sentinel values"; + return max; + } + + @Override + public String toString() { + if (this == UNKNOWN) { + return "UNKNOWN"; + } else if (this == EMPTY) { + return "EMPTY"; + } else { + return "[" + min + "-" + max + "]"; + } + } + + private static final byte WIRE_TYPE_OTHER = (byte)0; + private static final byte WIRE_TYPE_UNKNOWN = (byte)1; + private static final byte WIRE_TYPE_EMPTY = (byte)2; + + public static ShardLongFieldRange readFrom(StreamInput in) throws IOException { + if (in.getVersion().before(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { + // conservative treatment for BWC + return UNKNOWN; + } + + final byte type = in.readByte(); + switch (type) { + case WIRE_TYPE_UNKNOWN: + return UNKNOWN; + case WIRE_TYPE_EMPTY: + return EMPTY; + case WIRE_TYPE_OTHER: + return ShardLongFieldRange.of(in.readZLong(), in.readZLong()); + default: + throw new IllegalStateException("type [" + type + "] not known"); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { + if (this == UNKNOWN) { + out.writeByte(WIRE_TYPE_UNKNOWN); + } else if (this == EMPTY) { + out.writeByte(WIRE_TYPE_EMPTY); + } else { + out.writeByte(WIRE_TYPE_OTHER); + out.writeZLong(min); + out.writeZLong(max); + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (this == EMPTY || this == UNKNOWN || o == EMPTY || o == UNKNOWN) return false; + final ShardLongFieldRange that = (ShardLongFieldRange) o; + return min == that.min && max == that.max; + } + + @Override + public int hashCode() { + return Objects.hash(min, max); + } +} + diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 49c9379395e38..38693076e45f0 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -63,6 +63,7 @@ import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.IndexShardRelocatedException; import org.elasticsearch.index.shard.IndexShardState; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.PrimaryReplicaSyncer; import org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.elasticsearch.index.shard.ShardId; @@ -649,9 +650,14 @@ private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shardRouting.shardId(), state, nodes.getMasterNode()); } if (nodes.getMasterNode() != null) { - shardStateAction.shardStarted(shardRouting, primaryTerm, "master " + nodes.getMasterNode() + - " marked shard as initializing, but shard state is [" + state + "], mark shard as started", - SHARD_STATE_ACTION_LISTENER, clusterState); + shardStateAction.shardStarted( + shardRouting, + primaryTerm, + "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + state + + "], mark shard as started", + shard.getTimestampMillisRange(), + SHARD_STATE_ACTION_LISTENER, + clusterState); } } } @@ -705,8 +711,13 @@ private RecoveryListener(final ShardRouting shardRouting, final long primaryTerm } @Override - public void onRecoveryDone(final RecoveryState state) { - shardStateAction.shardStarted(shardRouting, primaryTerm, "after " + state.getRecoverySource(), SHARD_STATE_ACTION_LISTENER); + public void onRecoveryDone(final RecoveryState state, ShardLongFieldRange timestampMillisFieldRange) { + shardStateAction.shardStarted( + shardRouting, + primaryTerm, + "after " + state.getRecoverySource(), + timestampMillisFieldRange, + SHARD_STATE_ACTION_LISTENER); } @Override @@ -798,6 +809,13 @@ public interface Shard { */ RecoveryState recoveryState(); + /** + * @return the range of the {@code @timestamp} field for this shard, in milliseconds since the epoch, or {@link + * ShardLongFieldRange#EMPTY} if this field is not found, or {@link ShardLongFieldRange#UNKNOWN} if its range is not fixed. + */ + @Nullable + ShardLongFieldRange getTimestampMillisRange(); + /** * Updates the shard state based on an incoming cluster state: * - Updates and persists the new routing value. diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 08e111f163f83..013595ef762ad 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -50,6 +50,7 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.index.store.Store; @@ -271,7 +272,7 @@ public static StartRecoveryRequest getStartRecoveryRequest(Logger logger, Discov } public interface RecoveryListener { - void onRecoveryDone(RecoveryState state); + void onRecoveryDone(RecoveryState state, ShardLongFieldRange timestampMillisFieldRange); void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure); } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index ae762d85979ba..18f589148abd1 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -256,7 +256,7 @@ public void markAsDone() { // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now decRef(); } - listener.onRecoveryDone(state()); + listener.onRecoveryDone(state(), indexShard.getTimestampMillisRange()); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 65923dee57498..3f12d4a085170 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -69,6 +69,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.ShardLimitValidator; @@ -339,7 +340,8 @@ public ClusterState execute(ClusterState currentState) { .index(renamedIndexName); indexMdBuilder.settings(Settings.builder() .put(snapshotIndexMetadata.getSettings()) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())); + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())) + .timestampMillisRange(IndexLongFieldRange.NO_SHARDS); shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState); if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty()) { // Remove all aliases - they shouldn't be restored @@ -372,6 +374,7 @@ public ClusterState execute(ClusterState currentState) { 1 + currentIndexMetadata.getSettingsVersion())); indexMdBuilder.aliasesVersion( Math.max(snapshotIndexMetadata.getAliasesVersion(), 1 + currentIndexMetadata.getAliasesVersion())); + indexMdBuilder.timestampMillisRange(IndexLongFieldRange.NO_SHARDS); for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) { indexMdBuilder.primaryTerm(shard, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 7109e2fa282b3..704f1c0416490 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -122,7 +122,10 @@ public void testToXContent() throws IOException { " \"0\" : [ ]\n" + " },\n" + " \"rollover_info\" : { },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -220,7 +223,10 @@ public void testToXContent() throws IOException { " \"0\" : [ ]\n" + " },\n" + " \"rollover_info\" : { },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 593d52b5e3d75..a61a878921f2f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -243,7 +243,10 @@ public void testToXContent() throws IOException { " \"time\" : 1\n" + " }\n" + " },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -425,7 +428,10 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti " \"time\" : 1\n" + " }\n" + " },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -614,7 +620,10 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti " \"time\" : 1\n" + " }\n" + " },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -741,7 +750,10 @@ public void testToXContentSameTypeName() throws IOException { " \"0\" : [ ]\n" + " },\n" + " \"rollover_info\" : { },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java index 81c4f45a6ae2e..f07d941921cb5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java @@ -32,7 +32,9 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardLongFieldRange; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +52,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; public class ShardStartedClusterStateTaskExecutorTests extends ESAllocationTestCase { @@ -78,7 +81,12 @@ public void testEmptyTaskListProducesSameClusterState() throws Exception { public void testNonExistentIndexMarkedAsSuccessful() throws Exception { final ClusterState clusterState = stateWithNoShard(); - final StartedShardEntry entry = new StartedShardEntry(new ShardId("test", "_na", 0), "aId", randomNonNegativeLong(), "test"); + final StartedShardEntry entry = new StartedShardEntry( + new ShardId("test", "_na", 0), + "aId", + randomNonNegativeLong(), + "test", + ShardLongFieldRange.UNKNOWN); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(entry)); assertSame(clusterState, result.resultingState); @@ -95,10 +103,20 @@ public void testNonExistentShardsAreMarkedAsSuccessful() throws Exception { final List tasks = Stream.concat( // Existent shard id but different allocation id IntStream.range(0, randomIntBetween(1, 5)) - .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetadata.getIndex(), 0), String.valueOf(i), 0L, "allocation id")), + .mapToObj(i -> new StartedShardEntry( + new ShardId(indexMetadata.getIndex(), 0), + String.valueOf(i), + 0L, + "allocation id", + ShardLongFieldRange.UNKNOWN)), // Non existent shard id IntStream.range(1, randomIntBetween(2, 5)) - .mapToObj(i -> new StartedShardEntry(new ShardId(indexMetadata.getIndex(), i), String.valueOf(i), 0L, "shard id")) + .mapToObj(i -> new StartedShardEntry( + new ShardId(indexMetadata.getIndex(), i), + String.valueOf(i), + 0L, + "shard id", + ShardLongFieldRange.UNKNOWN)) ).collect(Collectors.toList()); @@ -127,7 +145,7 @@ public void testNonInitializingShardAreMarkedAsSuccessful() throws Exception { allocationId = shardRoutingTable.replicaShards().iterator().next().allocationId().getId(); } final long primaryTerm = indexMetadata.primaryTerm(shardId.id()); - return new StartedShardEntry(shardId, allocationId, primaryTerm, "test"); + return new StartedShardEntry(shardId, allocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN); }).collect(Collectors.toList()); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); @@ -150,11 +168,11 @@ public void testStartedShards() throws Exception { final String primaryAllocationId = primaryShard.allocationId().getId(); final List tasks = new ArrayList<>(); - tasks.add(new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test")); + tasks.add(new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN)); if (randomBoolean()) { final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next(); final String replicaAllocationId = replicaShard.allocationId().getId(); - tasks.add(new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test")); + tasks.add(new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN)); } final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); assertNotSame(clusterState, result.resultingState); @@ -179,7 +197,7 @@ public void testDuplicateStartsAreOkay() throws Exception { final long primaryTerm = indexMetadata.primaryTerm(shardId.id()); final List tasks = IntStream.range(0, randomIntBetween(2, 10)) - .mapToObj(i -> new StartedShardEntry(shardId, allocationId, primaryTerm, "test")) + .mapToObj(i -> new StartedShardEntry(shardId, allocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN)) .collect(Collectors.toList()); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); @@ -210,8 +228,12 @@ public void testPrimaryTermsMismatch() throws Exception { final ShardId shardId = new ShardId(clusterState.metadata().index(indexName).getIndex(), shard); final String primaryAllocationId = clusterState.routingTable().shardRoutingTable(shardId).primaryShard().allocationId().getId(); { - final StartedShardEntry task = - new StartedShardEntry(shardId, primaryAllocationId, primaryTerm - 1, "primary terms does not match on primary"); + final StartedShardEntry task = new StartedShardEntry( + shardId, + primaryAllocationId, + primaryTerm - 1, + "primary terms does not match on primary", + ShardLongFieldRange.UNKNOWN); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); assertSame(clusterState, result.resultingState); @@ -223,8 +245,8 @@ public void testPrimaryTermsMismatch() throws Exception { assertSame(clusterState, result.resultingState); } { - final StartedShardEntry task = - new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "primary terms match on primary"); + final StartedShardEntry task = new StartedShardEntry( + shardId, primaryAllocationId, primaryTerm, "primary terms match on primary", ShardLongFieldRange.UNKNOWN); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); assertNotSame(clusterState, result.resultingState); @@ -241,7 +263,8 @@ public void testPrimaryTermsMismatch() throws Exception { final String replicaAllocationId = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next() .allocationId().getId(); - final StartedShardEntry task = new StartedShardEntry(shardId, replicaAllocationId, replicaPrimaryTerm, "test on replica"); + final StartedShardEntry task = new StartedShardEntry( + shardId, replicaAllocationId, replicaPrimaryTerm, "test on replica", ShardLongFieldRange.UNKNOWN); final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, singletonList(task)); assertNotSame(clusterState, result.resultingState); @@ -254,6 +277,51 @@ public void testPrimaryTermsMismatch() throws Exception { } } + public void testExpandsTimestampRange() throws Exception { + final String indexName = "test"; + final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING, ShardRoutingState.INITIALIZING); + + final IndexMetadata indexMetadata = clusterState.metadata().index(indexName); + final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0); + final long primaryTerm = indexMetadata.primaryTerm(shardId.id()); + final ShardRouting primaryShard = clusterState.routingTable().shardRoutingTable(shardId).primaryShard(); + final String primaryAllocationId = primaryShard.allocationId().getId(); + + assertThat(indexMetadata.getTimestampMillisRange(), sameInstance(IndexLongFieldRange.NO_SHARDS)); + + final ShardLongFieldRange shardTimestampMillisRange = randomBoolean() ? ShardLongFieldRange.UNKNOWN : + randomBoolean() ? ShardLongFieldRange.EMPTY : ShardLongFieldRange.of(1606407943000L, 1606407944000L); + + final List tasks = new ArrayList<>(); + tasks.add(new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", shardTimestampMillisRange)); + if (randomBoolean()) { + final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next(); + final String replicaAllocationId = replicaShard.allocationId().getId(); + tasks.add(new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", shardTimestampMillisRange)); + } + final ClusterStateTaskExecutor.ClusterTasksResult result = executeTasks(clusterState, tasks); + assertNotSame(clusterState, result.resultingState); + assertThat(result.executionResults.size(), equalTo(tasks.size())); + tasks.forEach(task -> { + assertThat(result.executionResults.containsKey(task), is(true)); + assertThat(((ClusterStateTaskExecutor.TaskResult) result.executionResults.get(task)).isSuccess(), is(true)); + + final IndexShardRoutingTable shardRoutingTable = result.resultingState.routingTable().shardRoutingTable(task.shardId); + assertThat(shardRoutingTable.getByAllocationId(task.allocationId).state(), is(ShardRoutingState.STARTED)); + + final IndexLongFieldRange timestampMillisRange = result.resultingState.metadata().index(indexName).getTimestampMillisRange(); + if (shardTimestampMillisRange == ShardLongFieldRange.UNKNOWN) { + assertThat(timestampMillisRange, sameInstance(IndexLongFieldRange.UNKNOWN)); + } else if (shardTimestampMillisRange == ShardLongFieldRange.EMPTY) { + assertThat(timestampMillisRange, sameInstance(IndexLongFieldRange.EMPTY)); + } else { + assertTrue(timestampMillisRange.isComplete()); + assertThat(timestampMillisRange.getMin(), equalTo(shardTimestampMillisRange.getMin())); + assertThat(timestampMillisRange.getMax(), equalTo(shardTimestampMillisRange.getMax())); + } + }); + } + private ClusterStateTaskExecutor.ClusterTasksResult executeTasks(final ClusterState state, final List tasks) throws Exception { final ClusterStateTaskExecutor.ClusterTasksResult result = executor.execute(state, tasks); diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index da36c6e9da64c..b5c30f54b54f7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -42,7 +42,9 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardLongFieldRangeWireTests; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.TestThreadPool; @@ -75,6 +77,7 @@ import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; @@ -423,7 +426,7 @@ public void testShardStarted() throws InterruptedException { final ShardRouting shardRouting = getRandomShardRouting(index); final long primaryTerm = clusterService.state().metadata().index(shardRouting.index()).primaryTerm(shardRouting.id()); final TestListener listener = new TestListener(); - shardStateAction.shardStarted(shardRouting, primaryTerm, "testShardStarted", listener); + shardStateAction.shardStarted(shardRouting, primaryTerm, "testShardStarted", ShardLongFieldRange.UNKNOWN, listener); final CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear(); assertThat(capturedRequests[0].request, instanceOf(ShardStateAction.StartedShardEntry.class)); @@ -432,6 +435,7 @@ public void testShardStarted() throws InterruptedException { assertThat(entry.shardId, equalTo(shardRouting.shardId())); assertThat(entry.allocationId, equalTo(shardRouting.allocationId().getId())); assertThat(entry.primaryTerm, equalTo(primaryTerm)); + assertThat(entry.timestampMillisRange, sameInstance(ShardLongFieldRange.UNKNOWN)); transport.handleResponse(capturedRequests[0].requestId, TransportResponse.Empty.INSTANCE); listener.await(); @@ -514,13 +518,22 @@ public void testStartedShardEntrySerialization() throws Exception { final String message = randomRealisticUnicodeOfCodepointLengthBetween(10, 100); final Version version = randomFrom(randomCompatibleVersion(random(), Version.CURRENT)); - try (StreamInput in = serialize(new StartedShardEntry(shardId, allocationId, primaryTerm, message), version).streamInput()) { + final ShardLongFieldRange timestampMillisRange = ShardLongFieldRangeWireTests.randomRange(); + final StartedShardEntry startedShardEntry = new StartedShardEntry( + shardId, + allocationId, + primaryTerm, + message, + timestampMillisRange); + try (StreamInput in = serialize(startedShardEntry, version).streamInput()) { in.setVersion(version); final StartedShardEntry deserialized = new StartedShardEntry(in); assertThat(deserialized.shardId, equalTo(shardId)); assertThat(deserialized.allocationId, equalTo(allocationId)); assertThat(deserialized.primaryTerm, equalTo(primaryTerm)); assertThat(deserialized.message, equalTo(message)); + assertThat(deserialized.timestampMillisRange, version.onOrAfter(ShardLongFieldRange.LONG_FIELD_RANGE_VERSION_INTRODUCED) ? + equalTo(timestampMillisRange) : sameInstance(ShardLongFieldRange.UNKNOWN)); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index e20b342d147d0..817d00e8815d8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -297,7 +297,10 @@ public void testToXContentAPI_SameTypeName() throws IOException { " \"0\" : [ ]\n" + " },\n" + " \"rollover_info\" : { },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -454,7 +457,10 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce " \"time\" : 1\n" + " }\n" + " },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + @@ -556,7 +562,10 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce " \"time\" : 1\n" + " }\n" + " },\n" + - " \"system\" : false\n" + + " \"system\" : false,\n" + + " \"timestamp_range\" : {\n" + + " \"shards\" : [ ]\n" + + " }\n" + " }\n" + " },\n" + " \"index-graveyard\" : {\n" + diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index d41a9f04cef97..5871a16b9e450 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.search.DocValueFormat; @@ -36,6 +37,10 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; public class DateFieldMapperTests extends MapperTestCase { @@ -333,4 +338,33 @@ public void testFetchDocValuesNanos() throws IOException { assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); } + + public void testResolutionRounding() { + final long millis = randomLong(); + assertThat(DateFieldMapper.Resolution.MILLISECONDS.roundDownToMillis(millis), equalTo(millis)); + assertThat(DateFieldMapper.Resolution.MILLISECONDS.roundUpToMillis(millis), equalTo(millis)); + + final long nanos = randomNonNegativeLong(); + final long down = DateFieldMapper.Resolution.NANOSECONDS.roundDownToMillis(nanos); + assertThat(DateUtils.toNanoSeconds(down), lessThanOrEqualTo(nanos)); + try { + assertThat(DateUtils.toNanoSeconds(down + 1), greaterThan(nanos)); + } catch (IllegalArgumentException e) { + // ok, down+1 was out of range + } + + final long up = DateFieldMapper.Resolution.NANOSECONDS.roundUpToMillis(nanos); + try { + assertThat(DateUtils.toNanoSeconds(up), greaterThanOrEqualTo(nanos)); + } catch (IllegalArgumentException e) { + // ok, up may be out of range by 1; we check that up-1 is in range below (as long as it's >0) + assertThat(up, greaterThan(0L)); + } + + if (up > 0) { + assertThat(DateUtils.toNanoSeconds(up - 1), lessThan(nanos)); + } else { + assertThat(up, equalTo(0L)); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTestUtils.java b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTestUtils.java new file mode 100644 index 0000000000000..14597cfa8625d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTestUtils.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.junit.Assert.assertSame; + +public class IndexLongFieldRangeTestUtils { + + static IndexLongFieldRange randomRange() { + switch (ESTestCase.between(1, 3)) { + case 1: + return IndexLongFieldRange.UNKNOWN; + case 2: + return IndexLongFieldRange.EMPTY; + case 3: + return randomSpecificRange(); + default: + throw new AssertionError("impossible"); + } + } + + static IndexLongFieldRange randomSpecificRange() { + return randomSpecificRange(null); + } + + static IndexLongFieldRange randomSpecificRange(Boolean complete) { + IndexLongFieldRange range = IndexLongFieldRange.NO_SHARDS; + + final int shardCount = ESTestCase.between(1, 5); + for (int i = 0; i < shardCount; i++) { + if (Boolean.FALSE.equals(complete) && range.getShards().length == shardCount - 1) { + // caller requested an incomplete range so we must skip the last shard + break; + } else if (Boolean.TRUE.equals(complete) || randomBoolean()) { + range = range.extendWithShardRange( + i, + shardCount, + randomBoolean() ? ShardLongFieldRange.EMPTY : ShardLongFieldRangeWireTests.randomSpecificRange()); + } + } + + assert range != IndexLongFieldRange.UNKNOWN; + assert complete == null || complete.equals(range.isComplete()); + return range; + } + + static boolean checkForSameInstances(IndexLongFieldRange expected, IndexLongFieldRange actual) { + final boolean expectSame = expected == IndexLongFieldRange.UNKNOWN + || expected == IndexLongFieldRange.EMPTY + || expected == IndexLongFieldRange.NO_SHARDS; + if (expectSame) { + assertSame(expected, actual); + } + return expectSame; + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTests.java new file mode 100644 index 0000000000000..7a41b192424f6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeTests.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.randomSpecificRange; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; + +public class IndexLongFieldRangeTests extends ESTestCase { + + public void testUnknownShardImpliesUnknownIndex() { + final IndexLongFieldRange range = randomSpecificRange(false); + assertThat(range.extendWithShardRange( + IntStream.of(range.getShards()).max().orElse(0) + 1, + between(1, 10), + ShardLongFieldRange.UNKNOWN), + sameInstance(IndexLongFieldRange.UNKNOWN)); + } + + public void testExtendWithKnownShardIsNoOp() { + IndexLongFieldRange range = randomSpecificRange(); + if (range == IndexLongFieldRange.NO_SHARDS) { + // need at least one known shard + range = range.extendWithShardRange(between(0, 5), 5, ShardLongFieldRange.EMPTY); + } + + final ShardLongFieldRange shardRange; + if (range.getMinUnsafe() == IndexLongFieldRange.EMPTY.getMinUnsafe() + && range.getMaxUnsafe() == IndexLongFieldRange.EMPTY.getMaxUnsafe()) { + shardRange = ShardLongFieldRange.EMPTY; + } else { + final long min = randomLongBetween(range.getMinUnsafe(), range.getMaxUnsafe()); + final long max = randomLongBetween(min, range.getMaxUnsafe()); + shardRange = randomBoolean() ? ShardLongFieldRange.EMPTY : ShardLongFieldRange.of(min, max); + } + + assertThat(range.extendWithShardRange( + range.isComplete() ? between(1, 10) : randomFrom(IntStream.of(range.getShards()).boxed().collect(Collectors.toList())), + between(1, 10), + shardRange), + sameInstance(range)); + } + + public void testExtendUnknownRangeIsNoOp() { + assertThat(IndexLongFieldRange.UNKNOWN.extendWithShardRange( + between(0, 10), + between(0, 10), + ShardLongFieldRangeWireTests.randomRange()), + sameInstance(IndexLongFieldRange.UNKNOWN)); + } + + public void testCompleteEmptyRangeIsEmptyInstance() { + final int shardCount = between(1, 5); + IndexLongFieldRange range = IndexLongFieldRange.NO_SHARDS; + for (int i = 0; i < shardCount; i++) { + assertFalse(range.isComplete()); + range = range.extendWithShardRange(i, shardCount, ShardLongFieldRange.EMPTY); + } + assertThat(range, sameInstance(IndexLongFieldRange.EMPTY)); + assertTrue(range.isComplete()); + } + + public void testIsCompleteWhenAllShardRangesIncluded() { + final int shardCount = between(1, 5); + IndexLongFieldRange range = IndexLongFieldRange.NO_SHARDS; + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + for (int i = 0; i < shardCount; i++) { + assertFalse(range.isComplete()); + final ShardLongFieldRange shardFieldRange; + if (randomBoolean()) { + shardFieldRange = ShardLongFieldRange.EMPTY; + } else { + shardFieldRange = ShardLongFieldRangeWireTests.randomSpecificRange(); + min = Math.min(min, shardFieldRange.getMin()); + max = Math.max(max, shardFieldRange.getMax()); + } + range = range.extendWithShardRange( + i, + shardCount, + shardFieldRange); + } + assertTrue(range.isComplete()); + if (range != IndexLongFieldRange.EMPTY) { + assertThat(range.getMin(), equalTo(min)); + assertThat(range.getMax(), equalTo(max)); + } else { + assertThat(min, equalTo(Long.MAX_VALUE)); + assertThat(max, equalTo(Long.MIN_VALUE)); + } + } + + public void testCanRemoveShardRange() { + assertThat(IndexLongFieldRange.UNKNOWN.removeShard(between(0, 4), 5), sameInstance(IndexLongFieldRange.UNKNOWN)); + assertThat(IndexLongFieldRange.UNKNOWN.removeShard(0, 1), sameInstance(IndexLongFieldRange.NO_SHARDS)); + + final IndexLongFieldRange initialRange = randomSpecificRange(); + final int shardCount = initialRange.isComplete() + ? between(1, 5) : Arrays.stream(initialRange.getShards()).max().orElse(0) + between(1, 3); + + final int shard = between(0, shardCount - 1); + final IndexLongFieldRange rangeWithoutShard = initialRange.removeShard(shard, shardCount); + assertFalse(rangeWithoutShard.isComplete()); + assertTrue(Arrays.stream(rangeWithoutShard.getShards()).noneMatch(i -> i == shard)); + if (rangeWithoutShard != IndexLongFieldRange.NO_SHARDS) { + assertThat(rangeWithoutShard.getMinUnsafe(), equalTo(initialRange.getMinUnsafe())); + assertThat(rangeWithoutShard.getMaxUnsafe(), equalTo(initialRange.getMaxUnsafe())); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java new file mode 100644 index 0000000000000..81f5940fcafd2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Arrays; + +import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.checkForSameInstances; +import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.randomRange; + +public class IndexLongFieldRangeWireTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return IndexLongFieldRange::readFrom; + } + + @Override + protected IndexLongFieldRange createTestInstance() { + return randomRange(); + } + + @Override + protected IndexLongFieldRange mutateInstance(IndexLongFieldRange instance) throws IOException { + if (instance == IndexLongFieldRange.UNKNOWN) { + return IndexLongFieldRangeTestUtils.randomSpecificRange(); + } + + if (randomBoolean()) { + return IndexLongFieldRange.UNKNOWN; + } + + while (true) { + final IndexLongFieldRange newInstance = IndexLongFieldRangeTestUtils.randomSpecificRange(); + if (newInstance.getMinUnsafe() != instance.getMinUnsafe() + || newInstance.getMaxUnsafe() != instance.getMaxUnsafe() + || Arrays.equals(newInstance.getShards(), instance.getShards()) == false) { + return newInstance; + } + } + } + + + @Override + protected void assertEqualInstances(IndexLongFieldRange expectedInstance, IndexLongFieldRange newInstance) { + if (checkForSameInstances(expectedInstance, newInstance) == false) { + super.assertEqualInstances(expectedInstance, newInstance); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeXContentTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeXContentTests.java new file mode 100644 index 0000000000000..f48346950e6bc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeXContentTests.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.checkForSameInstances; +import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.randomRange; +import static org.hamcrest.Matchers.sameInstance; + +public class IndexLongFieldRangeXContentTests extends AbstractXContentTestCase { + @Override + protected IndexLongFieldRange createTestInstance() { + return randomRange(); + } + + @Override + protected IndexLongFieldRange doParseInstance(XContentParser parser) throws IOException { + assertThat(parser.nextToken(), sameInstance(XContentParser.Token.START_OBJECT)); + return IndexLongFieldRange.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected void assertEqualInstances(IndexLongFieldRange expectedInstance, IndexLongFieldRange newInstance) { + if (checkForSameInstances(expectedInstance, newInstance) == false) { + super.assertEqualInstances(expectedInstance, newInstance); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java new file mode 100644 index 0000000000000..ffd523c4c2377 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class ShardLongFieldRangeWireTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return ShardLongFieldRange::readFrom; + } + + @Override + protected ShardLongFieldRange createTestInstance() { + return randomRange(); + } + + public static ShardLongFieldRange randomRange() { + switch (between(1, 3)) { + case 1: + return ShardLongFieldRange.UNKNOWN; + case 2: + return ShardLongFieldRange.EMPTY; + case 3: + return randomSpecificRange(); + default: + throw new AssertionError("impossible"); + } + } + + static ShardLongFieldRange randomSpecificRange() { + final long min = randomLong(); + return ShardLongFieldRange.of(min, randomLongBetween(min, Long.MAX_VALUE)); + } + + @Override + protected ShardLongFieldRange mutateInstance(ShardLongFieldRange instance) throws IOException { + if (instance == ShardLongFieldRange.UNKNOWN) { + return randomBoolean() ? ShardLongFieldRange.EMPTY : randomSpecificRange(); + } + if (instance == ShardLongFieldRange.EMPTY) { + return randomBoolean() ? ShardLongFieldRange.UNKNOWN : randomSpecificRange(); + } + + switch (between(1, 4)) { + case 1: + return ShardLongFieldRange.UNKNOWN; + case 2: + return ShardLongFieldRange.EMPTY; + case 3: + return instance.getMin() == Long.MAX_VALUE + ? randomSpecificRange() + : ShardLongFieldRange.of(randomValueOtherThan(instance.getMin(), + () -> randomLongBetween(Long.MIN_VALUE, instance.getMax())), instance.getMax()); + case 4: + return instance.getMax() == Long.MIN_VALUE + ? randomSpecificRange() + : ShardLongFieldRange.of(instance.getMin(), randomValueOtherThan(instance.getMax(), + () -> randomLongBetween(instance.getMin(), Long.MAX_VALUE))); + default: + throw new AssertionError("impossible"); + } + } + + @Override + protected void assertEqualInstances(ShardLongFieldRange expectedInstance, ShardLongFieldRange newInstance) { + if (expectedInstance == ShardLongFieldRange.UNKNOWN || expectedInstance == ShardLongFieldRange.EMPTY) { + assertSame(expectedInstance, newInstance); + } else { + super.assertEqualInstances(expectedInstance, newInstance); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index 5d08c0f787621..eba2de1eb2f25 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; @@ -407,5 +408,11 @@ public void updateTerm(long newTerm) { } this.term = newTerm; } + + @Override + public ShardLongFieldRange getTimestampMillisRange() { + return ShardLongFieldRange.EMPTY; + } + } } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 7468718025f28..a6b30646cf701 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -91,6 +91,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.ShardLimitValidator; import org.elasticsearch.indices.SystemIndices; @@ -313,7 +314,12 @@ public ClusterState applyStartedShards(ClusterState clusterState, List startedShards) { return runTasks(shardStartedClusterStateTaskExecutor, clusterState, startedShards.entrySet().stream() - .map(e -> new StartedShardEntry(e.getKey().shardId(), e.getKey().allocationId().getId(), e.getValue(), "shard started")) + .map(e -> new StartedShardEntry( + e.getKey().shardId(), + e.getKey().allocationId().getId(), + e.getValue(), + "shard started", + ShardLongFieldRange.UNKNOWN)) .collect(Collectors.toList())); } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index b7e7b6557279d..1c61c74067cca 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.replication.RecoveryDuringReplicationTests; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.SnapshotMatchers; import org.elasticsearch.index.translog.Translog; @@ -377,7 +378,7 @@ public long addDocument(Iterable doc) throws IOExcepti expectThrows(Exception.class, () -> group.recoverReplica(replica, (shard, sourceNode) -> new RecoveryTarget(shard, sourceNode, new PeerRecoveryTargetService.RecoveryListener() { @Override - public void onRecoveryDone(RecoveryState state) { + public void onRecoveryDone(RecoveryState state, ShardLongFieldRange timestampMillisFieldRange) { throw new AssertionError("recovery must fail"); } diff --git a/server/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/server/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java index a3b908fae5c9d..b58bd3fbf6396 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java +++ b/server/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.replication.ESIndexLevelReplicationTestCase; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.recovery.RecoveriesCollection; @@ -41,7 +42,7 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase { static final PeerRecoveryTargetService.RecoveryListener listener = new PeerRecoveryTargetService.RecoveryListener() { @Override - public void onRecoveryDone(RecoveryState state) { + public void onRecoveryDone(RecoveryState state, ShardLongFieldRange timestampMillisFieldRange) { } @@ -76,7 +77,7 @@ public void testRecoveryTimeout() throws Exception { final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica(), new PeerRecoveryTargetService.RecoveryListener() { @Override - public void onRecoveryDone(RecoveryState state) { + public void onRecoveryDone(RecoveryState state, ShardLongFieldRange timestampMillisFieldRange) { latch.countDown(); } diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 02f0018a50756..f0e3725876298 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -97,6 +98,8 @@ public static ClusterState state(String index, boolean activePrimaryLocal, Shard .put(SETTING_VERSION_CREATED, Version.CURRENT) .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm) + .timestampMillisRange(primaryState == ShardRoutingState.STARTED || primaryState == ShardRoutingState.RELOCATING + ? IndexLongFieldRange.UNKNOWN : IndexLongFieldRange.NO_SHARDS) .build(); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); @@ -300,6 +303,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice IndexMetadata indexMetadata = IndexMetadata.builder(index) .settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).put(SETTING_NUMBER_OF_SHARDS, numberOfShards) .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis())) + .timestampMillisRange(IndexLongFieldRange.UNKNOWN) .build(); metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded(); IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index d677891743cfa..d23c58d93e12a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -123,7 +123,7 @@ public abstract class IndexShardTestCase extends ESTestCase { protected static final PeerRecoveryTargetService.RecoveryListener recoveryListener = new PeerRecoveryTargetService.RecoveryListener() { @Override - public void onRecoveryDone(RecoveryState state) { + public void onRecoveryDone(RecoveryState state, ShardLongFieldRange timestampMillisFieldRange) { } diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexIT.java new file mode 100644 index 0000000000000..1e3192f64b747 --- /dev/null +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexIT.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.index.engine; + +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand; +import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; +import org.elasticsearch.xpack.frozen.FrozenIndices; +import org.joda.time.Instant; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0) +public class FrozenIndexIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(FrozenIndices.class, LocalStateCompositeXPackPlugin.class); + } + + @Override + protected boolean addMockInternalEngine() { + return false; + } + + public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOException { + final List nodeNames = internalCluster().startNodes(2); + + createIndex("index", Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build()); + + final IndexResponse indexResponse = client().prepareIndex("index") + .setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567Z").get(); + + ensureGreen("index"); + + assertThat(client().admin().indices().prepareFlush("index").get().getSuccessfulShards(), equalTo(2)); + assertThat(client().admin().indices().prepareRefresh("index").get().getSuccessfulShards(), equalTo(2)); + + final String excludeSetting = INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(); + assertAcked(client().admin().indices().prepareUpdateSettings("index").setSettings( + Settings.builder().put(excludeSetting, nodeNames.get(0)))); + assertAcked(client().admin().cluster().prepareReroute().add(new CancelAllocationCommand("index", 0, nodeNames.get(0), true))); + assertThat(client().admin().cluster().prepareHealth("index").get().getUnassignedShards(), equalTo(1)); + + assertThat(client().prepareDelete("index", indexResponse.getId()).get().status(), equalTo(RestStatus.OK)); + + assertAcked(client().execute(FreezeIndexAction.INSTANCE, + new FreezeRequest("index").waitForActiveShards(ActiveShardCount.ONE)).actionGet()); + + assertThat(client().admin().cluster().prepareState().get().getState().metadata().index("index").getTimestampMillisRange(), + sameInstance(IndexLongFieldRange.EMPTY)); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeNames.get(1))); + assertThat(client().admin().cluster().prepareHealth("index").get().getUnassignedShards(), equalTo(2)); + assertAcked(client().admin().indices().prepareUpdateSettings("index") + .setSettings(Settings.builder().putNull(excludeSetting))); + assertThat(client().admin().cluster().prepareHealth("index").get().getUnassignedShards(), equalTo(2)); + + assertAcked(client().admin().cluster().prepareReroute().add( + new AllocateStalePrimaryAllocationCommand("index", 0, nodeNames.get(0), true))); + + ensureYellowAndNoInitializingShards("index"); + + final IndexLongFieldRange timestampFieldRange + = client().admin().cluster().prepareState().get().getState().metadata().index("index").getTimestampMillisRange(); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + assertTrue(timestampFieldRange.isComplete()); + assertThat(timestampFieldRange.getMin(), equalTo(Instant.parse("2010-01-06T02:03:04.567Z").getMillis())); + assertThat(timestampFieldRange.getMax(), equalTo(Instant.parse("2010-01-06T02:03:04.567Z").getMillis())); + } + +} diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexTests.java index 6192e4ca55a18..c52747e4be3d1 100644 --- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -9,17 +9,15 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; -import org.elasticsearch.xpack.core.search.action.ClosePointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.common.Strings; @@ -32,6 +30,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.indices.IndicesService; @@ -40,16 +39,21 @@ import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; +import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; +import org.elasticsearch.xpack.core.search.action.ClosePointInTimeRequest; import org.elasticsearch.xpack.core.search.action.OpenPointInTimeAction; import org.elasticsearch.xpack.core.search.action.OpenPointInTimeRequest; import org.elasticsearch.xpack.core.search.action.OpenPointInTimeResponse; import org.elasticsearch.xpack.frozen.FrozenIndices; import org.hamcrest.Matchers; +import org.joda.time.Instant; import java.io.IOException; import java.util.Arrays; @@ -63,13 +67,15 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; public class FrozenIndexTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(FrozenIndices.class, XPackPlugin.class); + return pluginList(FrozenIndices.class, LocalStateCompositeXPackPlugin.class); } String openReaders(TimeValue keepAlive, String... indices) { @@ -190,10 +196,13 @@ public void testSearchAndGetAPIsAreThrottled() throws IOException { } public void testFreezeAndUnfreeze() { - createIndex("index", Settings.builder().put("index.number_of_shards", 2).build()); + final IndexService originalIndexService = createIndex("index", Settings.builder().put("index.number_of_shards", 2).build()); + assertThat(originalIndexService.getMetadata().getTimestampMillisRange(), sameInstance(IndexLongFieldRange.UNKNOWN)); + client().prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); client().prepareIndex("index").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); client().prepareIndex("index").setId("3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); + if (randomBoolean()) { // sometimes close it assertAcked(client().admin().indices().prepareClose("index").get()); @@ -206,6 +215,7 @@ public void testFreezeAndUnfreeze() { assertTrue(indexService.getIndexSettings().isSearchThrottled()); IndexShard shard = indexService.getShard(0); assertEquals(0, shard.refreshStats().getTotal()); + assertThat(indexService.getMetadata().getTimestampMillisRange(), sameInstance(IndexLongFieldRange.UNKNOWN)); } assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index").setFreeze(false)).actionGet()); @@ -217,6 +227,7 @@ public void testFreezeAndUnfreeze() { IndexShard shard = indexService.getShard(0); Engine engine = IndexShardTestCase.getEngine(shard); assertThat(engine, Matchers.instanceOf(InternalEngine.class)); + assertThat(indexService.getMetadata().getTimestampMillisRange(), sameInstance(IndexLongFieldRange.UNKNOWN)); } client().prepareIndex("index").setId("4").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get(); } @@ -329,7 +340,7 @@ public void testCanMatch() throws IOException { new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); IndicesStatsResponse response = client().admin().indices().prepareStats("index").clear().setRefresh(true).get(); - assertEquals(0, response.getTotal().refresh.getTotal()); // never opened a reader + assertEquals(0, response.getTotal().refresh.getTotal()); } } @@ -477,4 +488,59 @@ public void testTranslogStats() { equalTo(indexService.getIndexSettings().isSoftDeleteEnabled() ? 0 : nbDocs)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } + + public void testComputesTimestampRangeFromMilliseconds() { + final int shardCount = between(1, 3); + createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build()); + client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-05T01:02:03.456Z").get(); + client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567Z").get(); + + assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); + + final IndexLongFieldRange timestampFieldRange + = client().admin().cluster().prepareState().get().getState().metadata().index("index").getTimestampMillisRange(); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + assertTrue(timestampFieldRange.isComplete()); + assertThat(timestampFieldRange.getMin(), equalTo(Instant.parse("2010-01-05T01:02:03.456Z").getMillis())); + assertThat(timestampFieldRange.getMax(), equalTo(Instant.parse("2010-01-06T02:03:04.567Z").getMillis())); + + for (ShardStats shardStats : client().admin().indices().prepareStats("index").clear().setRefresh(true).get().getShards()) { + assertThat("shard " + shardStats.getShardRouting() + " refreshed to get the timestamp range", + shardStats.getStats().refresh.getTotal(), greaterThanOrEqualTo(1L)); + } + } + + public void testComputesTimestampRangeFromNanoseconds() throws IOException { + + final XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .field("type", "date_nanos") + .field("format", "strict_date_optional_time_nanos") + .endObject() + .endObject() + .endObject(); + + final int shardCount = between(1, 3); + createIndex("index", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount).build(), mapping); + client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-05T01:02:03.456789012Z").get(); + client().prepareIndex("index").setSource(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, "2010-01-06T02:03:04.567890123Z").get(); + + assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet()); + + final IndexLongFieldRange timestampFieldRange + = client().admin().cluster().prepareState().get().getState().metadata().index("index").getTimestampMillisRange(); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + assertTrue(timestampFieldRange.isComplete()); + assertThat(timestampFieldRange.getMin(), equalTo(Instant.parse("2010-01-05T01:02:03.456Z").getMillis())); + assertThat(timestampFieldRange.getMax(), equalTo(Instant.parse("2010-01-06T02:03:04.568Z").getMillis())); + + for (ShardStats shardStats : client().admin().indices().prepareStats("index").clear().setRefresh(true).get().getShards()) { + assertThat("shard " + shardStats.getShardRouting() + " refreshed to get the timestamp range", + shardStats.getStats().refresh.getTotal(), greaterThanOrEqualTo(1L)); + } + } + } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 44626e951ed06..f12a7d87df671 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -28,11 +29,13 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.indices.IndicesService; @@ -49,6 +52,7 @@ import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; import org.hamcrest.Matchers; +import org.joda.time.Instant; import java.io.IOException; import java.nio.file.Files; @@ -83,6 +87,8 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegTestCase { @@ -132,6 +138,21 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { assertShardFolders(indexName, false); + assertThat( + client().admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .setIndices(indexName) + .get() + .getState() + .metadata() + .index(indexName) + .getTimestampMillisRange(), + sameInstance(IndexLongFieldRange.UNKNOWN) + ); + final boolean deletedBeforeMount = randomBoolean(); if (deletedBeforeMount) { assertAcked(client().admin().indices().prepareDelete(indexName)); @@ -214,6 +235,21 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { ensureGreen(restoredIndexName); assertShardFolders(restoredIndexName, true); + assertThat( + client().admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .setIndices(restoredIndexName) + .get() + .getState() + .metadata() + .index(restoredIndexName) + .getTimestampMillisRange(), + sameInstance(IndexLongFieldRange.UNKNOWN) + ); + if (deletedBeforeMount) { assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName)); @@ -668,6 +704,87 @@ public void testSnapshotMountedIndexLeavesBlobsUntouched() throws Exception { assertThat(snapshotTwoStatus.getStats().getProcessedFileCount(), equalTo(numShards)); // one segment_N per shard } + public void testSnapshotMountedIndexWithTimestampsRecordsTimestampRangeInIndexMetadata() throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final int numShards = between(1, 3); + + assertAcked( + client().admin() + .indices() + .prepareCreate(indexName) + .setMapping( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD) + .field("type", "date_nanos") + .field("format", "strict_date_optional_time_nanos") + .endObject() + .endObject() + .endObject() + ) + .setSettings(indexSettingsNoReplicas(numShards).put(INDEX_SOFT_DELETES_SETTING.getKey(), true)) + ); + ensureGreen(indexName); + + final List indexRequestBuilders = new ArrayList<>(); + final int docCount = between(0, 1000); + for (int i = 0; i < docCount; i++) { + indexRequestBuilders.add( + client().prepareIndex(indexName) + .setSource( + DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, + String.format( + Locale.ROOT, + "2020-11-26T%02d:%02d:%02d.%09dZ", + between(0, 23), + between(0, 59), + between(0, 59), + randomLongBetween(0, 999999999L) + ) + ) + ); + } + indexRandom(true, false, indexRequestBuilders); + assertThat( + client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), + equalTo(0) + ); + refresh(indexName); + forceMerge(); + + final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + createRepository(repositoryName, "fs"); + + final SnapshotId snapshotOne = createSnapshot(repositoryName, "snapshot-1", List.of(indexName)).snapshotId(); + assertAcked(client().admin().indices().prepareDelete(indexName)); + + mountSnapshot(repositoryName, snapshotOne.getName(), indexName, indexName, Settings.EMPTY); + ensureGreen(indexName); + + final IndexLongFieldRange timestampMillisRange = client().admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .setIndices(indexName) + .get() + .getState() + .metadata() + .index(indexName) + .getTimestampMillisRange(); + + assertTrue(timestampMillisRange.isComplete()); + assertThat(timestampMillisRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + if (docCount == 0) { + assertThat(timestampMillisRange, sameInstance(IndexLongFieldRange.EMPTY)); + } else { + assertThat(timestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + assertThat(timestampMillisRange.getMin(), greaterThanOrEqualTo(Instant.parse("2020-11-26T00:00:00Z").getMillis())); + assertThat(timestampMillisRange.getMin(), lessThanOrEqualTo(Instant.parse("2020-11-27T00:00:00Z").getMillis())); + } + } + private void assertTotalHits(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { final Thread[] threads = new Thread[between(1, 5)]; final AtomicArray allHits = new AtomicArray<>(threads.length); From 44a60d00a64fe73136ef8ab878e4f067dcaef7f2 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 1 Dec 2020 17:23:53 +0100 Subject: [PATCH 12/27] Add 6.8 branch to packer cache script (#65510) * Add 6.8 branch to packer cache script * use --reference on cloning 6.8 brach to speed cloning up a bit * Remove 6.8 branch checkout after resolving deps to keep image small --- .ci/packer_cache.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 3cd4abcc21090..ab93c839b9cc9 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -16,12 +16,21 @@ while [ -h "$SCRIPT" ] ; do done source $(dirname "${SCRIPT}")/java-versions.properties -export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} -# We are caching BWC versions too, need these so we can build those +## We are caching BWC versions too, need these so we can build those export JAVA8_HOME="${HOME}"/.java/java8 export JAVA11_HOME="${HOME}"/.java/java11 export JAVA12_HOME="${HOME}"/.java/openjdk12 export JAVA13_HOME="${HOME}"/.java/openjdk13 export JAVA14_HOME="${HOME}"/.java/openjdk14 -./gradlew --parallel clean -s resolveAllDependencies +## 6.8 branch is not referenced from any bwc project in master so we need to +## resolve its dependencies explicitly +rm -rf checkout/6.8 +git clone --reference $(dirname "${SCRIPT}")/../.git https://github.com/elastic/elasticsearch.git --branch 6.8 --single-branch checkout/6.8 +export JAVA_HOME="${JAVA11_HOME}" +./checkout/6.8/gradlew --project-dir ./checkout/6.8 --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true --stacktrace resolveAllDependencies +rm -rf ./checkout/6.8 +## Gradle is able to resolve dependencies resolved with earlier gradle versions +## therefore we run master _AFTER_ we run 6.8 which uses an earlier gradle version +export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} +./gradlew --parallel clean -s resolveAllDependencies From 08df709c814584a94652b7dcf93b467fff6b3057 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 1 Dec 2020 18:25:03 +0200 Subject: [PATCH 13/27] [ML] Update serialization versions for revert snapshot force param (#65680) --- .../xpack/core/ml/action/RevertModelSnapshotAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java index 8f60540659b9e..611bab81bc8ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java @@ -74,7 +74,7 @@ public Request(StreamInput in) throws IOException { jobId = in.readString(); snapshotId = in.readString(); deleteInterveningResults = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_11_0)) { force = in.readBoolean(); } else { force = false; @@ -121,7 +121,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); out.writeString(snapshotId); out.writeBoolean(deleteInterveningResults); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_11_0)) { out.writeBoolean(force); } } From ac1dbb7ffd3ab61ee03757d6aa95c52f5ac61134 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 1 Dec 2020 11:30:17 -0500 Subject: [PATCH 14/27] [DOCS] EQL: Remove outdated wildcard ref (#65684) --- docs/reference/eql/functions.asciidoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/reference/eql/functions.asciidoc b/docs/reference/eql/functions.asciidoc index a3e707c6891c5..8d0f1a6da3644 100644 --- a/docs/reference/eql/functions.asciidoc +++ b/docs/reference/eql/functions.asciidoc @@ -1015,10 +1015,6 @@ expressions. Matching is case-sensitive. *Example* [source,eql] ---- -// The two following expressions are equivalent. -process.name == "*regsvr32*" or process.name == "*explorer*" -wildcard(process.name, "*regsvr32*", "*explorer*") - // process.name = "regsvr32.exe" wildcard(process.name, "*regsvr32*") // returns true wildcard(process.name, "*regsvr32*", "*explorer*") // returns true From dc64498f1027935302f1f6a19a7732c4569a88f3 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 1 Dec 2020 11:38:40 -0500 Subject: [PATCH 15/27] [DOCS] Fix URL in rollup API JSON spec (#65683) --- .../src/test/resources/rest-api-spec/api/rollup.rollup.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json index 04c794c822bd6..0e53cb0820ace 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup.json @@ -1,7 +1,7 @@ { "rollup.rollup":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-rollup.html", + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/rollup-api.html", "description":"Rollup an index" }, "stability":"stable", From c327794ae8ce93a10642e844ddb93aaca266e287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Dec 2020 18:49:50 +0100 Subject: [PATCH 16/27] Fix range query on date fields for number inputs (#63692) Currently, if you write a date range query with numeric 'to' or 'from' bounds, they can be interpreted as years if no format is provided. We use "strict_date_optional_time||epoch_millis" in this case that can interpret inputs like 1000 as the year 1000 for example. This PR change this to always interpret and parse numbers with the "epoch_millis" parser if no other formatter was provided. Closes #63680 --- .../migration/migrate_8_0/search.asciidoc | 18 ++++++++++++ docs/reference/query-dsl/range-query.asciidoc | 8 ++++++ .../release-notes/8.0.0-alpha1.asciidoc | 3 +- .../test/multi_cluster/30_field_caps.yml | 4 +-- .../test/field_caps/30_filter.yml | 6 ++-- .../search/simple/SimpleSearchIT.java | 20 +++++++++++++ .../index/mapper/DateFieldMapper.java | 22 ++++++++++++--- .../mapper/DateScriptFieldType.java | 28 ++++++------------- 8 files changed, 79 insertions(+), 30 deletions(-) diff --git a/docs/reference/migration/migrate_8_0/search.asciidoc b/docs/reference/migration/migrate_8_0/search.asciidoc index 51a244367b2b7..79d29678a6b33 100644 --- a/docs/reference/migration/migrate_8_0/search.asciidoc +++ b/docs/reference/migration/migrate_8_0/search.asciidoc @@ -157,3 +157,21 @@ Change any use of `-1` as `from` parameter in request body or url parameters by setting it to `0` or omitting it entirely. Requests containing negative values will return an error. ==== + +.Range queries on date fields treat numeric values alwas as milliseconds-since-epoch. +[%collapsible] +==== +*Details* + +Range queries on date fields used to misinterpret small numbers (e.g. four digits like 1000) +as a year when no additional format was set, but would interpret other numeric values as +milliseconds since epoch. We now treat all numeric values in absence of a specific `format` +parameter as milliseconds since epoch. If you want to query for years instead, with a missing +`format` you now need to quote the input value (e.g. "1984"). + +*Impact* + +If you query date fields without a specified `format`, check if the values in your queries are +actually meant to be milliseconds-since-epoch and use a numeric value in this case. If not, use +a string value which gets parsed by either the date format set on the field in the mappings or +by `strict_date_optional_time` by default. + +==== diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc index 32184875aa0b6..7c078fc4067ca 100644 --- a/docs/reference/query-dsl/range-query.asciidoc +++ b/docs/reference/query-dsl/range-query.asciidoc @@ -183,6 +183,14 @@ to `2099-12-01T23:59:59.999_999_999Z`. This date uses the provided year (`2099`) and month (`12`) but uses the default day (`01`), hour (`23`), minute (`59`), second (`59`), and nanosecond (`999_999_999`). +[[numeric-date]] +====== Numeric date range value + +When no date format is specified and the range query is targeting a date field, numeric +values are interpreted representing milliseconds-since-the-epoch. If you want the value +to represent a year, e.g. 2020, you need to pass it as a String value (e.g. "2020") that +will be parsed according to the default format or the set format. + [[range-query-date-math-rounding]] ====== Date math and rounding {es} rounds <> values in parameters as follows: diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc index 9851c32c709e2..3a2823b222132 100644 --- a/docs/reference/release-notes/8.0.0-alpha1.asciidoc +++ b/docs/reference/release-notes/8.0.0-alpha1.asciidoc @@ -29,5 +29,6 @@ by using appropriate thresholds. If for instance we want to simulate `index.inde all we need to do is to set `index.indexing.slowlog.threshold.index.debug` and `index.indexing.slowlog.threshold.index.trace` to `-1` {es-pull}57591[#57591] - +Search:: +* Consistent treatment of numeric values for range query on date fields without `format` {es-pull}[#63692] diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml index cfecd009bd1ef..5d7182cf9be9a 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml @@ -106,7 +106,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { lt: 2018 } } } } + body: { index_filter: { range: { created_at: { lt: "2018" } } } } - match: {indices: ["field_caps_index_4","my_remote_cluster:field_caps_index_1"]} - length: {fields.number: 1} @@ -117,7 +117,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { gt: 2019 } } } } + body: { index_filter: { range: { created_at: { gt: "2019" } } } } - match: {indices: ["my_remote_cluster:field_caps_index_3"]} - length: {fields.number: 1} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml index da692dbe8e850..31e8661c9e472 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml @@ -81,7 +81,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2010 }}}} + body: { index_filter: { range: { timestamp: { gte: "2010" }}}} - match: {indices: ["test-1", "test-2", "test-3"]} - length: {fields.field1: 3} @@ -90,7 +90,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2019 } } } } + body: { index_filter: { range: { timestamp: { gte: "2019" } } } } - match: {indices: ["test-2", "test-3"]} - length: {fields.field1: 2} @@ -106,7 +106,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { lt: 2019 } } } } + body: { index_filter: { range: { timestamp: { lt: "2019" } } } } - match: {indices: ["test-1"]} - length: {fields.field1: 1} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 2ef7a961fde68..b8f2e884e6042 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -54,6 +54,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class SimpleSearchIT extends ESIntegTestCase { @@ -198,6 +200,7 @@ public void testSimpleDateRange() throws Exception { createIndex("test"); client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); + client().prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); ensureGreen(); refresh(); SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") @@ -223,6 +226,23 @@ public void testSimpleDateRange() throws Exception { searchResponse = client().prepareSearch("test").setQuery( QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); assertHitCount(searchResponse, 2L); + + // a string value of "1000" should be parsed as the year 1000 and return all three docs + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); + + // a numeric value of 1000 should be parsed as 1000 millis since epoch and return only docs after 1970 + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt(1000)) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + String[] expectedIds = new String[] {"1", "2"}; + assertThat(searchResponse.getHits().getHits()[0].getId(), is(oneOf(expectedIds))); + assertThat(searchResponse.getHits().getHits()[1].getId(), is(oneOf(expectedIds))); } public void testRangeQueryKeyword() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 478cad469d357..9243b5e77d6bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -73,6 +73,7 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @@ -384,9 +385,17 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser = forcedDateParser == null - ? dateMathParser - : forcedDateParser; + DateMathParser parser; + if (forcedDateParser == null) { + if (lowerTerm instanceof Number || upperTerm instanceof Number) { + // force epoch_millis + parser = EPOCH_MILLIS_PARSER; + } else { + parser = dateMathParser; + } + } else { + parser = forcedDateParser; + } return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { Query query = LongPoint.newRangeQuery(name(), l, u); if (hasDocValues()) { @@ -479,7 +488,12 @@ public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { if (dateParser == null) { - dateParser = this.dateMathParser; + if (from instanceof Number || to instanceof Number) { + // force epoch_millis + dateParser = EPOCH_MILLIS_PARSER; + } else { + dateParser = this.dateMathParser; + } } long fromInclusive = Long.MIN_VALUE; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index 79184ac7bb727..c54ad030cb9da 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; + import org.apache.lucene.search.Query; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; @@ -91,10 +92,12 @@ protected AbstractScriptFieldType buildFieldType() { }); private final DateFormatter dateTimeFormatter; + private final DateMathParser dateMathParser; private DateScriptFieldType(String name, DateFieldScript.Factory scriptFactory, DateFormatter dateTimeFormatter, Builder builder) { super(name, (n, params, ctx) -> scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), builder); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } DateScriptFieldType( @@ -107,6 +110,7 @@ private DateScriptFieldType(String name, DateFieldScript.Factory scriptFactory, ) { super(name, (n, params, ctx) -> scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), script, meta, toXContent); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } @Override @@ -148,7 +152,7 @@ public Query distanceFeatureQuery(Object origin, String pivot, QueryShardContext origin, true, null, - dateTimeFormatter.toDateMathParser(), + this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS ); @@ -179,7 +183,7 @@ public Query rangeQuery( @Nullable DateMathParser parser, QueryShardContext context ) { - parser = parser == null ? dateTimeFormatter.toDateMathParser() : parser; + parser = parser == null ? this.dateMathParser : parser; checkAllowExpensiveQueries(context); return DateFieldType.dateRangeQuery( lowerTerm, @@ -197,14 +201,7 @@ public Query rangeQuery( @Override public Query termQuery(Object value, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { - long l = DateFieldType.parseToLong( - value, - false, - null, - dateTimeFormatter.toDateMathParser(), - now, - DateFieldMapper.Resolution.MILLISECONDS - ); + long l = DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); checkAllowExpensiveQueries(context); return new LongScriptFieldTermQuery(script, leafFactory(context)::newInstance, name(), l); }); @@ -218,16 +215,7 @@ public Query termsQuery(List values, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { LongSet terms = new LongHashSet(values.size()); for (Object value : values) { - terms.add( - DateFieldType.parseToLong( - value, - false, - null, - dateTimeFormatter.toDateMathParser(), - now, - DateFieldMapper.Resolution.MILLISECONDS - ) - ); + terms.add(DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS)); } checkAllowExpensiveQueries(context); return new LongScriptFieldTermsQuery(script, leafFactory(context)::newInstance, name(), terms); From 1c3ddf8ff1199b11d498fe2d9ef70eb65b505855 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Tue, 1 Dec 2020 12:56:12 -0500 Subject: [PATCH 17/27] [DOCS] EQL: Flatten EQL syntax headings (#65693) --- docs/reference/eql/syntax.asciidoc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index b5ce89e3ae23d..9c793b4409ae5 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -38,7 +38,7 @@ field using the `event_category_field` parameter of the EQL search API. [discrete] [[eql-syntax-match-any-event-category]] -==== Match any event category +=== Match any event category To match events of any category, use the `any` keyword. You can also use the `any` keyword to search for documents without a event category field. @@ -53,7 +53,7 @@ any where network.protocol == "http" [discrete] [[eql-syntax-escape-an-event-category]] -==== Escape an event category +=== Escape an event category Use enclosing double quotes (`"`) or three enclosing double quotes (`"""`) to escape event categories that: @@ -77,7 +77,7 @@ escape event categories that: [discrete] [[eql-syntax-escape-a-field-name]] -==== Escape a field name +=== Escape a field name Use enclosing enclosing backticks (+++`+++) to escape field names that: @@ -110,7 +110,7 @@ EQL operators are case-sensitive by default. [discrete] [[eql-syntax-comparison-operators]] -==== Comparison operators +=== Comparison operators [source,eql] ---- @@ -197,7 +197,7 @@ process where process.parent.name == "foo" and process.name == "foo" [discrete] [[eql-syntax-logical-operators]] -==== Logical operators +=== Logical operators [source,eql] ---- @@ -217,7 +217,7 @@ Returns `true` if the condition to the right is `false`. [discrete] [[eql-syntax-lookup-operators]] -==== Lookup operators +=== Lookup operators [source,eql] ---- @@ -240,7 +240,7 @@ to compare strings. [discrete] [[eql-syntax-math-operators]] -==== Math operators +=== Math operators [source,eql] ---- @@ -334,7 +334,7 @@ Strings enclosed in single quotes (`'`) are not supported. [discrete] [[eql-syntax-escape-characters]] -==== Escape characters in a string +=== Escape characters in a string When used within a string, special characters, such as a carriage return or double quote (`"`), must be escaped with a preceding backslash (`\`). @@ -360,7 +360,7 @@ double quote (`\"`) instead. [discrete] [[eql-syntax-raw-strings]] -==== Raw strings +=== Raw strings Raw strings treat special characters, such as backslashes (`\`), as literal characters. Raw strings are enclosed in three double quotes (`"""`). @@ -380,7 +380,7 @@ use a regular string with the `\"` escape sequence. [discrete] [[eql-syntax-wildcards]] -==== Wildcards +=== Wildcards For string comparisons using the `:` operator, you can use wildcards (`*`) to match specific patterns: From 8d5f101efcebd411ae26dba51fd1c31627e6b79c Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 1 Dec 2020 13:32:48 -0500 Subject: [PATCH 18/27] [ML] test mute for testUpgradeJobSnapshot (#65700) relates to https://github.com/elastic/elasticsearch/issues/65699 --- .../test/java/org/elasticsearch/client/MachineLearningIT.java | 1 + .../client/documentation/MlClientDocumentationIT.java | 1 + 2 files changed, 2 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 0375cedba50be..96fa9491fa52e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -2828,6 +2828,7 @@ public void testUpdateModelSnapshot() throws Exception { getModelSnapshotsResponse2.snapshots().get(0).getDescription()); } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/65699") public void testUpgradeJobSnapshot() throws Exception { String jobId = "test-upgrade-model-snapshot"; String snapshotId = "1541587919"; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 9df6be7717d3a..730d9bae8f8ed 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -2336,6 +2336,7 @@ public void onFailure(Exception e) { } } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/65699") public void testUpgradeJobSnapshot() throws IOException, InterruptedException { RestHighLevelClient client = highLevelClient(); From b4233f0cd438009c4f13fa394325a03bf642aa40 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 1 Dec 2020 13:53:33 -0500 Subject: [PATCH 19/27] [DOCS] Fix _doc_count example typo (#65686) --- docs/reference/mapping/fields/doc-count-field.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/fields/doc-count-field.asciidoc b/docs/reference/mapping/fields/doc-count-field.asciidoc index 52eb8ea52a71f..d8898a690bf62 100644 --- a/docs/reference/mapping/fields/doc-count-field.asciidoc +++ b/docs/reference/mapping/fields/doc-count-field.asciidoc @@ -72,7 +72,7 @@ PUT my_index/_doc/2 "values" : [0.1, 0.25, 0.35, 0.4, 0.45, 0.5], "counts" : [8, 17, 8, 7, 6, 2] }, - "_doc_count_": 62 <1> + "_doc_count": 62 <1> } -------------------------------------------------- <1> Field `_doc_count` must be a positive integer storing the number of documents aggregated to produce each histogram. From 3c3a43249f993f9c602e0875d491e990c752efe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Dec 2020 21:40:27 +0100 Subject: [PATCH 20/27] Support unmapped fields in search 'fields' option (#65386) Currently, the 'fields' option only supports fetching mapped fields. Since 'fields' is meant to be the central place to retrieve document content, it should allow for loading unmapped values. This change adds implementation and tests for this feature. Closes #63690 --- .../retrieve-selected-fields.asciidoc | 85 ++++++ .../test/search/330_fetch_fields.yml | 107 +++++++ .../action/search/ExpandSearchPhase.java | 2 +- .../action/search/SearchRequestBuilder.java | 11 +- .../index/query/InnerHitBuilder.java | 12 +- .../metrics/TopHitsAggregationBuilder.java | 10 +- .../search/builder/SearchSourceBuilder.java | 9 +- .../fetch/subphase/FetchDocValuesContext.java | 2 +- .../fetch/subphase/FetchFieldsPhase.java | 1 + .../search/fetch/subphase/FieldAndFormat.java | 28 +- .../search/fetch/subphase/FieldFetcher.java | 111 ++++++- .../fetch/subphase/FieldFetcherTests.java | 287 +++++++++++++++++- .../rest-api-spec/test/flattened/10_basic.yml | 37 ++- 13 files changed, 668 insertions(+), 34 deletions(-) diff --git a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc index eae7a541bd942..81f2406fd5d9f 100644 --- a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc @@ -167,7 +167,92 @@ no dedicated array type, and any field could contain multiple values. The a specific order. See the mapping documentation on <> for more background. +[discrete] +[[retrieve-unmapped-fields]] +==== Retrieving unmapped fields + +By default, the `fields` parameter returns only values of mapped fields. However, +Elasticsearch allows storing fields in `_source` that are unmapped, for example by +setting <> to `false` or by using an +object field with `enabled: false`, thereby disabling parsing and indexing of its content. + +Fields in such an object can be retrieved from `_source` using the `include_unmapped` option +in the `fields` section: + +[source,console] +---- +PUT my-index-000001 +{ + "mappings": { + "enabled": false <1> + } +} + +PUT my-index-000001/_doc/1?refresh=true +{ + "user_id": "kimchy", + "session_data": { + "object": { + "some_field": "some_value" + } + } +} + +POST my-index-000001/_search +{ + "fields": [ + "user_id", + { + "field": "session_data.object.*", + "include_unmapped" : true <2> + } + ], + "_source": false +} +---- + +<1> Disable all mappings. +<2> Include unmapped fields matching this field pattern. +The response will contain fields results under the `session_data.object.*` path even if the +fields are unmapped, but will not contain `user_id` since it is unmapped but the `include_unmapped` +flag hasn't been set to `true` for that field pattern. + +[source,console-result] +---- +{ + "took" : 2, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index-000001", + "_id" : "1", + "_score" : 1.0, + "fields" : { + "session_data.object.some_field": [ + "some_value" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took" : 2/"took": $body.took/] +// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] +// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] [discrete] [[docvalue-fields]] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index b43a63398d138..b9843ed2424b3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -295,3 +295,110 @@ setup: - is_true: hits.hits.0._id - match: { hits.hits.0.fields.count: [2] } - is_false: hits.hits.0.fields.count_without_dv +--- +Test unmapped field: + - skip: + version: ' - 7.99.99' + reason: support isn't yet backported + - do: + indices.create: + index: test + body: + mappings: + dynamic: false + properties: + f1: + type: keyword + f2: + type: object + enabled: false + f3: + type: object + - do: + index: + index: test + id: 1 + refresh: true + body: + f1: some text + f2: + a: foo + b: bar + f3: + c: baz + f4: some other text + - do: + search: + index: test + body: + fields: + - f1 + - { "field" : "f4", "include_unmapped" : true } + - match: + hits.hits.0.fields.f1: + - some text + - match: + hits.hits.0.fields.f4: + - some other text + - do: + search: + index: test + body: + fields: + - { "field" : "f*", "include_unmapped" : true } + - match: + hits.hits.0.fields.f1: + - some text + - match: + hits.hits.0.fields.f2\.a: + - foo + - match: + hits.hits.0.fields.f2\.b: + - bar + - match: + hits.hits.0.fields.f3\.c: + - baz + - match: + hits.hits.0.fields.f4: + - some other text +--- +Test unmapped fields inside disabled objects: + - skip: + version: ' - 7.99.99' + reason: support isn't yet backported + - do: + indices.create: + index: test + body: + mappings: + properties: + f1: + type: object + enabled: false + - do: + index: + index: test + id: 1 + refresh: true + body: + f1: + - some text + - a: b + - + - 1 + - 2 + - 3 + - do: + search: + index: test + body: + fields: [ { "field" : "*", "include_unmapped" : true } ] + - match: + hits.hits.0.fields.f1: + - 1 + - 2 + - 3 + - some text + - match: + hits.hits.0.fields.f1\.a: + - b diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 9ee9bf947a46d..79340fb2abeaa 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -138,7 +138,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio } } if (options.getFetchFields() != null) { - options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format)); + options.getFetchFields().forEach(ff -> groupSource.fetchField(ff)); } if (options.getDocValueFields() != null) { options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format)); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 3a02214abb6f0..9bb4973015bdb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.rescore.RescorerBuilder; import org.elasticsearch.search.slice.SliceBuilder; @@ -310,18 +311,18 @@ public SearchRequestBuilder addDocValueField(String name) { * @param name The field to load */ public SearchRequestBuilder addFetchField(String name) { - sourceBuilder().fetchField(name, null); + sourceBuilder().fetchField(new FieldAndFormat(name, null, null)); return this; } /** * Adds a field to load and return. The field must be present in the document _source. * - * @param name The field to load - * @param format an optional format string used when formatting values, for example a date format. + * @param fetchField a {@link FieldAndFormat} specifying the field pattern, optional format (for example a date format) and + * whether this field pattern sould also include unmapped fields */ - public SearchRequestBuilder addFetchField(String name, String format) { - sourceBuilder().fetchField(name, format); + public SearchRequestBuilder addFetchField(FieldAndFormat fetchField) { + sourceBuilder().fetchField(fetchField); return this; } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 5416395cb7627..8aa80b8a27dfd 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -395,10 +395,20 @@ public InnerHitBuilder addFetchField(String name) { * @param format an optional format string used when formatting values, for example a date format. */ public InnerHitBuilder addFetchField(String name, @Nullable String format) { + return addFetchField(name, format, null); + } + + /** + * Adds a field to load and return as part of the search request. + * @param name the field name. + * @param format an optional format string used when formatting values, for example a date format. + * @param includeUnmapped whether unmapped fields should be returned as well + */ + public InnerHitBuilder addFetchField(String name, @Nullable String format, Boolean includeUnmapped) { if (fetchFields == null || fetchFields.isEmpty()) { fetchFields = new ArrayList<>(); } - fetchFields.add(new FieldAndFormat(name, format)); + fetchFields.add(new FieldAndFormat(name, format, includeUnmapped)); return this; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index c1174ac0d1d81..f9dba1e7141f4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -446,14 +446,14 @@ public List docValueFields() { /** * Adds a field to load and return as part of the search request. */ - public TopHitsAggregationBuilder fetchField(String field, String format) { - if (field == null) { + public TopHitsAggregationBuilder fetchField(FieldAndFormat fieldAndFormat) { + if (fieldAndFormat == null) { throw new IllegalArgumentException("[fields] must not be null: [" + name + "]"); } if (fetchFields == null) { fetchFields = new ArrayList<>(); } - fetchFields.add(new FieldAndFormat(field, format)); + fetchFields.add(fieldAndFormat); return this; } @@ -461,7 +461,7 @@ public TopHitsAggregationBuilder fetchField(String field, String format) { * Adds a field to load and return as part of the search request. */ public TopHitsAggregationBuilder fetchField(String field) { - return fetchField(field, null); + return fetchField(new FieldAndFormat(field, null, null)); } /** @@ -796,7 +796,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa } else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { FieldAndFormat ff = FieldAndFormat.fromXContent(parser); - factory.fetchField(ff.field, ff.format); + factory.fetchField(ff); } } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { List> sorts = SortBuilder.fromXContent(parser); diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 41329c6af212b..b2e66eb97e844 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -868,19 +868,18 @@ public List fetchFields() { * Adds a field to load and return as part of the search request. */ public SearchSourceBuilder fetchField(String name) { - return fetchField(name, null); + return fetchField(new FieldAndFormat(name, null, null)); } /** * Adds a field to load and return as part of the search request. - * @param name the field name. - * @param format an optional format string used when formatting values, for example a date format. + * @param fetchField defining the field name, optional format and optional inclusion of unmapped fields */ - public SearchSourceBuilder fetchField(String name, @Nullable String format) { + public SearchSourceBuilder fetchField(FieldAndFormat fetchField) { if (fetchFields == null) { fetchFields = new ArrayList<>(); } - fetchFields.add(new FieldAndFormat(name, format)); + fetchFields.add(fetchField); return this; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java index b653b9babe7ad..bbe267d31d141 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java @@ -45,7 +45,7 @@ public FetchDocValuesContext(QueryShardContext shardContext, List fieldNames = shardContext.simpleMatchToIndexNames(field.field); for (String fieldName : fieldNames) { if (shardContext.isFieldMapped(fieldName)) { - fields.add(new FieldAndFormat(fieldName, field.format)); + fields.add(new FieldAndFormat(fieldName, field.format, field.includeUnmapped)); } } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index c450e61c33ab7..67c9f614ef913 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -54,6 +54,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { } FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getQueryShardContext(), searchLookup, fetchFieldsContext.fields()); + return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java index cf4edd13f5cd8..b5379204234b4 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; @@ -40,14 +41,16 @@ public final class FieldAndFormat implements Writeable, ToXContentObject { private static final ParseField FIELD_FIELD = new ParseField("field"); private static final ParseField FORMAT_FIELD = new ParseField("format"); + private static final ParseField INCLUDE_UNMAPPED_FIELD = new ParseField("include_unmapped"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("fetch_field_and_format", - a -> new FieldAndFormat((String) a[0], (String) a[1])); + a -> new FieldAndFormat((String) a[0], (String) a[1], (Boolean) a[2])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_FIELD); PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FORMAT_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), INCLUDE_UNMAPPED_FIELD); } /** @@ -69,6 +72,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (format != null) { builder.field(FORMAT_FIELD.getPreferredName(), format); } + if (this.includeUnmapped != null) { + builder.field(INCLUDE_UNMAPPED_FIELD.getPreferredName(), includeUnmapped); + } builder.endObject(); return builder; } @@ -79,28 +85,44 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws /** The format of the field, or {@code null} if defaults should be used. */ public final String format; - /** Sole constructor. */ + /** Whether to include unmapped fields or not. */ + public final Boolean includeUnmapped; + public FieldAndFormat(String field, @Nullable String format) { + this(field, format, null); + } + + public FieldAndFormat(String field, @Nullable String format, @Nullable Boolean includeUnmapped) { this.field = Objects.requireNonNull(field); this.format = format; + this.includeUnmapped = includeUnmapped; } /** Serialization constructor. */ public FieldAndFormat(StreamInput in) throws IOException { this.field = in.readString(); format = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.CURRENT)) { + this.includeUnmapped = in.readOptionalBoolean(); + } else { + this.includeUnmapped = null; + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeOptionalString(format); + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalBoolean(this.includeUnmapped); + } } @Override public int hashCode() { int h = field.hashCode(); h = 31 * h + Objects.hashCode(format); + h = 31 * h + Objects.hashCode(includeUnmapped); return h; } @@ -110,6 +132,6 @@ public boolean equals(Object obj) { return false; } FieldAndFormat other = (FieldAndFormat) obj; - return field.equals(other.field) && Objects.equals(format, other.format); + return field.equals(other.field) && Objects.equals(format, other.format) && Objects.equals(includeUnmapped, other.includeUnmapped); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index eda55d22a6092..b9a68a555375b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -20,7 +20,10 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; @@ -31,6 +34,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,9 +49,16 @@ public static FieldFetcher create(QueryShardContext context, Collection fieldAndFormats) { List fieldContexts = new ArrayList<>(); + List unmappedFetchPattern = new ArrayList<>(); + Set mappedToExclude = new HashSet<>(); + boolean includeUnmapped = false; for (FieldAndFormat fieldAndFormat : fieldAndFormats) { String fieldPattern = fieldAndFormat.field; + if (fieldAndFormat.includeUnmapped != null && fieldAndFormat.includeUnmapped) { + unmappedFetchPattern.add(fieldAndFormat.field); + includeUnmapped = true; + } String format = fieldAndFormat.format; Collection concreteFields = context.simpleMatchToIndexNames(fieldPattern); @@ -57,17 +68,34 @@ public static FieldFetcher create(QueryShardContext context, continue; } ValueFetcher valueFetcher = ft.valueFetcher(context, format); + mappedToExclude.add(field); fieldContexts.add(new FieldContext(field, valueFetcher)); } } - - return new FieldFetcher(fieldContexts); + CharacterRunAutomaton unmappedFetchAutomaton = new CharacterRunAutomaton(Automata.makeEmpty()); + if (unmappedFetchPattern.isEmpty() == false) { + unmappedFetchAutomaton = new CharacterRunAutomaton( + Regex.simpleMatchToAutomaton(unmappedFetchPattern.toArray(new String[unmappedFetchPattern.size()])) + ); + } + return new FieldFetcher(fieldContexts, unmappedFetchAutomaton, mappedToExclude, includeUnmapped); } private final List fieldContexts; - - private FieldFetcher(List fieldContexts) { + private final CharacterRunAutomaton unmappedFetchAutomaton; + private final Set mappedToExclude; + private final boolean includeUnmapped; + + private FieldFetcher( + List fieldContexts, + CharacterRunAutomaton unmappedFetchAutomaton, + Set mappedToExclude, + boolean includeUnmapped + ) { this.fieldContexts = fieldContexts; + this.unmappedFetchAutomaton = unmappedFetchAutomaton; + this.mappedToExclude = mappedToExclude; + this.includeUnmapped = includeUnmapped; } public Map fetch(SourceLookup sourceLookup, Set ignoredFields) throws IOException { @@ -85,9 +113,84 @@ public Map fetch(SourceLookup sourceLookup, Set i documentFields.put(field, new DocumentField(field, parsedValues)); } } + if (this.includeUnmapped) { + collectUnmapped(documentFields, sourceLookup.loadSourceIfNeeded(), "", 0); + } return documentFields; } + private void collectUnmapped(Map documentFields, Map source, String parentPath, int lastState) { + for (String key : source.keySet()) { + Object value = source.get(key); + String currentPath = parentPath + key; + int currentState = step(this.unmappedFetchAutomaton, key, lastState); + if (currentState == -1) { + // current path doesn't match any fields pattern + continue; + } + if (value instanceof Map) { + // one step deeper into source tree + collectUnmapped( + documentFields, + (Map) value, + currentPath + ".", + step(this.unmappedFetchAutomaton, ".", currentState) + ); + } else if (value instanceof List) { + // iterate through list values + collectUnmappedList(documentFields, (List) value, currentPath, currentState); + } else { + // we have a leaf value + if (this.unmappedFetchAutomaton.isAccept(currentState) && this.mappedToExclude.contains(currentPath) == false) { + if (value != null) { + DocumentField currentEntry = documentFields.get(currentPath); + if (currentEntry == null) { + List list = new ArrayList<>(); + list.add(value); + documentFields.put(currentPath, new DocumentField(currentPath, list)); + } else { + currentEntry.getValues().add(value); + } + } + } + } + } + } + + private void collectUnmappedList(Map documentFields, Iterable iterable, String parentPath, int lastState) { + List list = new ArrayList<>(); + for (Object value : iterable) { + if (value instanceof Map) { + collectUnmapped( + documentFields, + (Map) value, + parentPath + ".", + step(this.unmappedFetchAutomaton, ".", lastState) + ); + } else if (value instanceof List) { + // weird case, but can happen for objects with "enabled" : "false" + collectUnmappedList(documentFields, (List) value, parentPath, lastState); + } else if (this.unmappedFetchAutomaton.isAccept(lastState) && this.mappedToExclude.contains(parentPath) == false) { + list.add(value); + } + } + if (list.isEmpty() == false) { + DocumentField currentEntry = documentFields.get(parentPath); + if (currentEntry == null) { + documentFields.put(parentPath, new DocumentField(parentPath, list)); + } else { + currentEntry.getValues().addAll(list); + } + } + } + + private static int step(CharacterRunAutomaton automaton, String key, int state) { + for (int i = 0; state != -1 && i < key.length(); ++i) { + state = automaton.step(state, key.charAt(i)); + } + return state; + } + public void setNextReader(LeafReaderContext readerContext) { for (FieldContext field : fieldContexts) { field.valueFetcher.setNextReader(readerContext); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 96dadaa8c71e6..00d17464f4e70 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; @@ -35,6 +36,7 @@ import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -58,7 +60,7 @@ public void testLeafValues() throws IOException { List fieldAndFormats = List.of( new FieldAndFormat("field", null), new FieldAndFormat("object.field", null)); - Map fields = fetchFields(mapperService, source, fieldAndFormats); + Map fields = fetchFields(mapperService, source, fieldAndFormats, null); assertThat(fields.size(), equalTo(2)); DocumentField field = fields.get("field"); @@ -238,7 +240,8 @@ public void testDateFormat() throws IOException { Map fields = fetchFields(mapperService, source, List.of( new FieldAndFormat("field", null), - new FieldAndFormat("date_field", "yyyy/MM/dd"))); + new FieldAndFormat("date_field", "yyyy/MM/dd")), + null); assertThat(fields.size(), equalTo(2)); DocumentField field = fields.get("field"); @@ -393,21 +396,289 @@ public void testTextSubFields() throws IOException { } } - private static Map fetchFields(MapperService mapperService, XContentBuilder source, String fieldPattern) - throws IOException { + public void testSimpleUnmappedFields() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder() + .startObject() + .field("unmapped_f1", "some text") + .field("unmapped_f2", "some text") + .field("unmapped_f3", "some text") + .field("something_else", "some text") + .nullField("null_value") + .startObject("object") + .field("a", "foo") + .endObject() + .field("object.b", "bar") + .endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_f*", null, true), null); + assertThat(fields.size(), equalTo(3)); + assertThat(fields.keySet(), containsInAnyOrder("unmapped_f1", "unmapped_f2", "unmapped_f3")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("un*1", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.keySet(), containsInAnyOrder("unmapped_f1")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("*thing*", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.keySet(), containsInAnyOrder("something_else")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("null*", null, true), null); + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("object.a", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertEquals("foo", fields.get("object.a").getValues().get(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("object.b", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertEquals("bar", fields.get("object.b").getValues().get(0)); + } + + public void testSimpleUnmappedArray() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject().array("unmapped_field", "foo", "bar").endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.keySet(), containsInAnyOrder("unmapped_field")); + DocumentField field = fields.get("unmapped_field"); + + assertThat(field.getValues().size(), equalTo(2)); + assertThat(field.getValues(), hasItems("foo", "bar")); + } + + public void testSimpleUnmappedArrayWithObjects() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .startArray("unmapped_field") + .startObject() + .field("f1", "a") + .endObject() + .startObject() + .field("f2", "b") + .endObject() + .endArray() + .endObject(); - List fields = List.of(new FieldAndFormat(fieldPattern, null)); - return fetchFields(mapperService, source, fields); + Map fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field", null, true), null); + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field.f*", null, true), null); + assertThat(fields.size(), equalTo(2)); + assertThat(fields.get("unmapped_field.f1").getValue(), equalTo("a")); + assertThat(fields.get("unmapped_field.f2").getValue(), equalTo("b")); + + source = XContentFactory.jsonBuilder().startObject() + .startArray("unmapped_field") + .startObject() + .field("f1", "a") + .array("f2", 1, 2) + .array("f3", 1, 2) + .endObject() + .startObject() + .field("f1", "b") // same field name, this should result in a list returned + .array("f2", 3, 4) + .array("f3", "foo") + .endObject() + .endArray() + .endObject(); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field.f1", null, true), null); + assertThat(fields.size(), equalTo(1)); + DocumentField field = fields.get("unmapped_field.f1"); + assertThat(field.getValues().size(), equalTo(2)); + assertThat(field.getValues(), hasItems("a", "b")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field.f2", null, true), null); + assertThat(fields.size(), equalTo(1)); + field = fields.get("unmapped_field.f2"); + assertThat(field.getValues().size(), equalTo(4)); + assertThat(field.getValues(), hasItems(1, 2, 3, 4)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_field.f3", null, true), null); + assertThat(fields.size(), equalTo(1)); + field = fields.get("unmapped_field.f3"); + assertThat(field.getValues().size(), equalTo(3)); + assertThat(field.getValues(), hasItems(1, 2, "foo")); } - private static Map fetchFields(MapperService mapperService, XContentBuilder source, List fields) + public void testUnmappedFieldsInsideObject() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("obj") + .field("type", "object") + .field("dynamic", "false") + .startObject("properties") + .startObject("f1").field("type", "keyword").endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createMapperService(mapping); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("obj.f1", "value1") + .field("obj.f2", "unmapped_value_f2") + .field("obj.innerObj.f3", "unmapped_value_f3") + .field("obj.innerObj.f4", "unmapped_value_f4") + .endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, false), null); + + // without unmapped fields this should only return "obj.f1" + assertThat(fields.size(), equalTo(1)); + assertThat(fields.keySet(), containsInAnyOrder("obj.f1")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, true), null); + assertThat(fields.size(), equalTo(4)); + assertThat(fields.keySet(), containsInAnyOrder("obj.f1", "obj.f2", "obj.innerObj.f3", "obj.innerObj.f4")); + } + + public void testUnmappedFieldsInsideDisabledObject() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("obj") + .field("type", "object") + .field("enabled", "false") + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createMapperService(mapping); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .startArray("obj") + .value("string_value") + .startObject() + .field("a", "b") + .endObject() + .startArray() + .value(1).value(2).value(3) + .endArray() + .endArray() + .endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, false), null); + // without unmapped fields this should return nothing + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("*", null, true), null); + assertThat(fields.size(), equalTo(2)); + assertThat(fields.keySet(), containsInAnyOrder("obj", "obj.a")); + + List obj = fields.get("obj").getValues(); + assertEquals(4, obj.size()); + assertThat(obj, hasItems("string_value", 1, 2, 3)); + + List innerObj = fields.get("obj.a").getValues(); + assertEquals(1, innerObj.size()); + assertEquals("b", fields.get("obj.a").getValue()); + } + + /** + * If a mapped field for some reason contains a "_source" value that is not returned by the + * mapped retrieval mechanism (e.g. because its malformed), we don't want to fetch it from _source. + */ + public void testMappedFieldNotOverwritten() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_doc") + .startObject("properties") + .startObject("f1") + .field("type", "integer") + .field("ignore_malformed", "true") + .endObject() + .endObject() + .endObject() + .endObject(); + + MapperService mapperService = createMapperService(mapping); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("f1", "malformed") + .endObject(); + + // this should not return a field bc. f1 is in the ignored fields + Map fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("*", null, true)), Set.of("f1")); + assertThat(fields.size(), equalTo(0)); + + // and this should neither + fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("*", null, true)), Set.of("f1")); + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("f1", null, true)), Set.of("f1")); + assertThat(fields.size(), equalTo(0)); + + // check this also does not overwrite with arrays + source = XContentFactory.jsonBuilder().startObject() + .array("f1", "malformed") + .endObject(); + + fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("f1", null, true)), Set.of("f1")); + assertThat(fields.size(), equalTo(0)); + } + + public void testUnmappedFieldsWildcard() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .startObject("unmapped_object") + .field("a", "foo") + .field("b", "bar") + .endObject() + .endObject(); + + Map fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_object", null, true), null); + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmap*object", null, true), null); + assertThat(fields.size(), equalTo(0)); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_object.*", null, true), null); + assertThat(fields.size(), equalTo(2)); + assertThat(fields.keySet(), containsInAnyOrder("unmapped_object.a", "unmapped_object.b")); + + assertThat(fields.get("unmapped_object.a").getValue(), equalTo("foo")); + assertThat(fields.get("unmapped_object.b").getValue(), equalTo("bar")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_object.a", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.get("unmapped_object.a").getValue(), equalTo("foo")); + + fields = fetchFields(mapperService, source, fieldAndFormatList("unmapped_object.b", null, true), null); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.get("unmapped_object.b").getValue(), equalTo("bar")); + } + + private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { + return Collections.singletonList(new FieldAndFormat(name, format, includeUnmapped)); + } + + private Map fetchFields(MapperService mapperService, XContentBuilder source, String fieldPattern) throws IOException { + return fetchFields(mapperService, source, fieldAndFormatList(fieldPattern, null, false), null); + } + + private static Map fetchFields( + MapperService mapperService, + XContentBuilder source, + List fields, + @Nullable Set ignoreFields + ) throws IOException { SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); FieldFetcher fieldFetcher = FieldFetcher.create(newQueryShardContext(mapperService), null, fields); - return fieldFetcher.fetch(sourceLookup, Set.of()); + return fieldFetcher.fetch(sourceLookup, ignoreFields != null ? ignoreFields : Collections.emptySet()); } public MapperService createMapperService() throws IOException { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml index 69729023158b1..eb84bc70950e0 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml @@ -150,9 +150,44 @@ search: index: test body: - fields: ["flat*"] + fields: [ "flat*" ] - match: { hits.total.value: 1 } - length: { hits.hits: 1 } - length: { hits.hits.0.fields: 1 } - match: { hits.hits.0.fields.flattened: [ { "some_field": "some_value" } ] } + +--- +"Test fields option on flattened object field with include_unmapped": + - skip: + version: ' - 7.99.99' + reason: support isn't yet backported + - do: + indices.create: + index: test + body: + mappings: + properties: + flattened: + type: flattened + + - do: + index: + index: test + id: 1 + body: + flattened: + some_field: some_value + refresh: true + + - do: + search: + index: test + body: + fields: [ { "field" : "flat*", "include_unmapped" : true } ] + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - length: { hits.hits.0.fields: 2 } + - match: { hits.hits.0.fields.flattened: [ { "some_field": "some_value" } ] } + - match: { hits.hits.0.fields.flattened\.some_field: [ "some_value" ] } From 19dfa7be9e163b50a042358246cdbc4e45d2d797 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 1 Dec 2020 22:15:53 +0100 Subject: [PATCH 21/27] Tableau connector: read version to build from src (#65671) This changes the Tableau connector building script to be able to read the version of the connector to be built straight from Version.java file, if that's available (which is the case, now being in tree). A new parameter has been added, to allow specifying a version qualifier. Also, the 'alias' parameter is made optional. --- .../plugin/sql/connectors/tableau/package.sh | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/sql/connectors/tableau/package.sh b/x-pack/plugin/sql/connectors/tableau/package.sh index 5c9a373a5dd35..19fdeb8390f9b 100755 --- a/x-pack/plugin/sql/connectors/tableau/package.sh +++ b/x-pack/plugin/sql/connectors/tableau/package.sh @@ -19,6 +19,9 @@ MY_WORKSPACE=$(realpath ${PACKAGE_WORKSPACE:-build}) MY_TOP_DIR=$(dirname $(realpath $0)) SRC_DIR=connector +ES_VER_FILE="server/src/main/java/org/elasticsearch/Version.java" +ES_VER_REL_PATH="$MY_TOP_DIR/../../../../../$ES_VER_FILE" + TACO_CLASS=$(xmllint \ --xpath '//connector-plugin/@class' $MY_TOP_DIR/$SRC_DIR/manifest.xml \ | awk -F\" '{print $2}') @@ -65,7 +68,8 @@ function package() { git checkout $TAB_SDK_TAG cd .. else - git clone --depth 1 --branch $TAB_SDK_TAG $TAB_SDK_REPO + git -c advice.detachedHead=false clone --depth 1 \ + --branch $TAB_SDK_TAG $TAB_SDK_REPO fi # install environment @@ -86,6 +90,17 @@ function sha() { sha512sum $ES_TACO > $ES_TACO.sha512 } +# Vars: +# set: TACO_VERSION +function read_es_version() { + VER_REGEX="CURRENT = V_[1-9]\{1,2\}_[0-9]\{1,2\}_[0-9]\{1,2\}" + TACO_VERSION=$(grep -o "$VER_REGEX" $ES_VER_REL_PATH | \ + sed -e 's/V_//' -e 's/CURRENT = //' -e 's/_/./g') + if [ -z $TACO_VERSION ]; then + die "failed to read version in source file $ES_VER_REL_PATH" + fi +} + # Vars: # read: CMD_ASM, CMD_SIGN # set: ES_TACO, TACO_VERSION, SIGN_PARAMS @@ -100,6 +115,12 @@ function read_cmd_params() { fi TACO_VERSION=$val ;; + qualifier) + if [ ! -z $VER_QUALIFIER ]; then + die "parameter 'qualifier' already set to: $VER_QUALIFIER" + fi + VER_QUALIFIER=$val + ;; keystore) if [ ! -z $SIGN_KEYSTORE ]; then die "parameter 'keystore' already set to: $SIGN_KEYSTORE" @@ -139,17 +160,21 @@ function read_cmd_params() { if [ $CMD_ASM -gt 0 ]; then if [ -z $TACO_VERSION ]; then - die "parameter 'version' is mandatory for assambling." + if [ ! -f $ES_VER_REL_PATH ]; then + die "parameter 'version' is required for assembling." + else + read_es_version + fi fi - ES_TACO=$TACO_CLASS-$TACO_VERSION.taco + ES_TACO=$TACO_CLASS-$TACO_VERSION$VER_QUALIFIER.taco fi if [ $CMD_SIGN -gt 0 ]; then - if [ -z $SIGN_KEYSTORE ] || [ -z $SIGN_ALIAS ]; then - die "parameters 'keystore' and 'alias' are mandatory for signing." + if [ -z $SIGN_KEYSTORE ]; then + die "parameter 'keystore' is mandatory for signing." fi - SIGN_PARAMS="$SIGN_ALIAS" + SIGN_PARAMS="$SIGN_ALIAS" # note: could be empty SIGN_PARAMS="$SIGN_PARAMS -keystore $SIGN_KEYSTORE" SIGN_PARAMS="$SIGN_PARAMS -tsa $TSA_URL" @@ -203,29 +228,30 @@ function usage() { log "Usage: $MY_FILE " log log "Commands:" - log " asm : assemble the TACO file" + log " asm [version parameters] : assemble the TACO file" log " sign : sign the TACO file" log " pack : assemble and sign" log " clean : remove the workspace" log log "Params take the form key=value with following keys:" - log " version : version of the TACO to produce; mandatory;" + log " version : version of the TACO to produce;" + log " qualifier : qualifier to attach to the version;" log " keystore : path to keystore to use; mandatory;" log " storepassfile : keystore password file; optional;" log " keypassfile : private key password file; optional;" log " onepass : password for both the keystore and the " log " private key; optional;" - log " alias : alias for the keystore entry; mandatory." + log " alias : alias for the keystore entry; optional." log log "All building is done under workspace: $MY_WORKSPACE" log "Can be changed with PACKAGE_WORKSPACE environment variable." log log "Example:" - log " ./$MY_FILE asm version=7.10.1" + log " ./$MY_FILE asm version=7.10.1 qualifier=-SNAPSHOT" log " ./$MY_FILE sign keystore=keystore_file alias=alias_name"\ "storepassfile=password_file" log " ./$MY_FILE pack version=7.10.2 keystore=keystore_file"\ - "alias=alias_name onepass=password" + "onepass=password" log } From 2adac36af88600c55dde91295e51026c329524e1 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 2 Dec 2020 08:54:40 +0100 Subject: [PATCH 22/27] Create async search index if necessary on updates and deletes (#64606) This change ensures that we create the async search index with the right mappings and settings when updating or deleting a document. Users can delete the async search index at any time so we have to re-create it internally if necessary before applying any new operation. --- .../core/async/AsyncTaskIndexService.java | 18 ++++-- .../core/async/AsyncTaskServiceTests.java | 60 ++++++++++++++++--- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index e01e8aca43eea..1ccc8d1e878a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -67,14 +67,16 @@ public final class AsyncTaskIndexService> { public static final String EXPIRATION_TIME_FIELD = "expiration_time"; public static final String RESULT_FIELD = "result"; - private static Settings settings() { + static Settings settings() { return Settings.builder() + .put("index.codec", "best_compression") .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") .build(); } - private static XContentBuilder mappings() throws IOException { + static XContentBuilder mappings() throws IOException { XContentBuilder builder = jsonBuilder() .startObject() .startObject(SINGLE_MAPPING_NAME) @@ -197,7 +199,9 @@ public void updateResponse(String docId, .id(docId) .doc(source, XContentType.JSON) .retryOnConflict(5); - client.update(request, listener); + // updates create the index automatically if it doesn't exist so we force the creation + // preemptively. + createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure)); } catch(Exception e) { listener.onFailure(e); } @@ -215,7 +219,9 @@ public void updateExpirationTime(String docId, .id(docId) .doc(source, XContentType.JSON) .retryOnConflict(5); - client.update(request, listener); + // updates create the index automatically if it doesn't exist so we force the creation + // preemptively. + createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure)); } /** @@ -225,7 +231,9 @@ public void deleteResponse(AsyncExecutionId asyncExecutionId, ActionListener listener) { try { DeleteRequest request = new DeleteRequest(index).id(asyncExecutionId.getDocId()); - client.delete(request, listener); + // deletes create the index automatically if it doesn't exist so we force the creation + // preemptively. + createIndexIfNecessary(ActionListener.wrap(v -> client.delete(request, listener), listener::onFailure)); } catch(Exception e) { listener.onFailure(e); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java index 28e9a9c806a2e..99b538da7dd99 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java @@ -7,11 +7,15 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; @@ -21,7 +25,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.concurrent.ExecutionException; // TODO: test CRUD operations public class AsyncTaskServiceTests extends ESSingleNodeTestCase { @@ -95,14 +98,55 @@ public void testEnsuredAuthenticatedUserIsSame() throws IOException { assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType)); } - public void testSettings() throws ExecutionException, InterruptedException { - PlainActionFuture future = PlainActionFuture.newFuture(); - indexService.createIndexIfNecessary(future); - future.get(); + public void testAutoCreateIndex() throws Exception { + { + PlainActionFuture future = PlainActionFuture.newFuture(); + indexService.createIndexIfNecessary(future); + future.get(); + assertSettings(); + } + AcknowledgedResponse ack = client().admin().indices().prepareDelete(index).get(); + assertTrue(ack.isAcknowledged()); + + AsyncExecutionId id = new AsyncExecutionId("0", new TaskId("N/A", 0)); + AsyncSearchResponse resp = new AsyncSearchResponse(id.getEncoded(), true, true, 0L, 0L); + { + PlainActionFuture future = PlainActionFuture.newFuture(); + indexService.createResponse(id.getDocId(), Collections.emptyMap(), resp, future); + future.get(); + assertSettings(); + } + ack = client().admin().indices().prepareDelete(index).get(); + assertTrue(ack.isAcknowledged()); + { + PlainActionFuture future = PlainActionFuture.newFuture(); + indexService.deleteResponse(id, future); + future.get(); + assertSettings(); + } + ack = client().admin().indices().prepareDelete(index).get(); + assertTrue(ack.isAcknowledged()); + { + PlainActionFuture future = PlainActionFuture.newFuture(); + indexService.updateResponse(id.getDocId(), Collections.emptyMap(), resp, future); + expectThrows(Exception.class, () -> future.get()); + assertSettings(); + } + ack = client().admin().indices().prepareDelete(index).get(); + assertTrue(ack.isAcknowledged()); + { + PlainActionFuture future = PlainActionFuture.newFuture(); + indexService.updateExpirationTime("0", 10L, future); + expectThrows(Exception.class, () -> future.get()); + assertSettings(); + } + } + + private void assertSettings() throws IOException { GetIndexResponse getIndexResponse = client().admin().indices().getIndex( new GetIndexRequest().indices(index)).actionGet(); Settings settings = getIndexResponse.getSettings().get(index); - assertEquals("1", settings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)); - assertEquals("0-1", settings.get(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS)); + Settings expected = AsyncTaskIndexService.settings(); + assertEquals(expected, settings.filter(key -> expected.hasValue(key))); } } From f0fc1b3dadba4459e1485360fd7cbd9ce594e283 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 2 Dec 2020 10:06:40 +0000 Subject: [PATCH 23/27] Use scriptless fields in CoreTestTranslator (#65599) CoreTestTranslator re-writes some core search yaml tests into a tests for runtime queries, and mimics source-only fields by building ad-hoc painless scripts for each runtime type. We now have source-only fields built in via scriptless mappings, so we can cut over to using these instead. --- .../rest-api-spec/test/search/30_limits.yml | 4 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 14 ++--- .../test/CoreTestTranslater.java | 60 ++++--------------- 3 files changed, 23 insertions(+), 55 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 93d2472255809..5720dfd174a17 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -116,7 +116,9 @@ setup: --- "Regexp length limit": - + - skip: + version: all + reason: Long regex breaks HTTP query length when request body is sent as a query param - https://github.com/elastic/elasticsearch/issues/65718 - do: catch: /The length of regex \[1110\] used in the Regexp Query request has exceeded the allowed maximum of \[1000\]\. This maximum can be set by changing the \[index.max_regex_length\] index level setting\./ search: diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index c80c643530fe6..17fe2c7cfda44 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -173,15 +173,15 @@ protected boolean handleIndex(IndexRequest index) { continue; } if (value instanceof Boolean) { - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "boolean")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("boolean")); continue; } if (value instanceof Long || value instanceof Integer) { - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("long")); continue; } if (value instanceof Double) { - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("double")); continue; } if (false == value instanceof String) { @@ -189,27 +189,27 @@ protected boolean handleIndex(IndexRequest index) { } try { Long.parseLong(value.toString()); - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("long")); continue; } catch (IllegalArgumentException iae) { // Try the next one } try { Double.parseDouble(value.toString()); - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("double")); continue; } catch (IllegalArgumentException iae) { // Try the next one } try { DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(value.toString()); - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "date")); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource("date")); continue; } catch (IllegalArgumentException iae) { // Try the next one } // Strings are funny, the regular dynamic mapping puts them in "name.keyword" so we follow along. - indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource(name, "keyword")); + indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource("keyword")); } return true; } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index bfe5f7628d42c..df5e2c4106fc9 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; @@ -72,42 +73,13 @@ public Iterable parameters() throws Exception { protected abstract Suite suite(ClientYamlTestCandidate candidate); - private static String painlessToLoadFromSource(String name, String type) { - String emit = PAINLESS_TO_EMIT.get(type); - if (emit == null) { - return null; - } - StringBuilder b = new StringBuilder(); - b.append("def v = params._source['").append(name).append("'];\n"); - b.append("if (v instanceof Iterable) {\n"); - b.append(" for (def vv : ((Iterable) v)) {\n"); - b.append(" if (vv != null) {\n"); - b.append(" def value = vv;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append(" }\n"); - b.append("} else {\n"); - b.append(" if (v != null) {\n"); - b.append(" def value = v;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append("}\n"); - return b.toString(); - } - - private static final Map PAINLESS_TO_EMIT = Map.ofEntries( - Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), - Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), - Map.entry( - NumberType.DOUBLE.typeName(), - "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" - ), - Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry( - NumberType.LONG.typeName(), - "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" - ) + private static final Set RUNTIME_TYPES = Set.of( + BooleanFieldMapper.CONTENT_TYPE, + DateFieldMapper.CONTENT_TYPE, + NumberType.DOUBLE.typeName(), + KeywordFieldMapper.CONTENT_TYPE, + IpFieldMapper.CONTENT_TYPE, + NumberType.LONG.typeName() ); protected abstract Map dynamicTemplateFor(String type); @@ -118,15 +90,11 @@ protected static Map dynamicTemplateToDisableRuntimeCompatibleFi // TODO there isn't yet a way to create fields in the runtime section from a dynamic template protected static Map dynamicTemplateToAddRuntimeFields(String type) { - return Map.ofEntries( - Map.entry("type", "runtime"), - Map.entry("runtime_type", type), - Map.entry("script", painlessToLoadFromSource("{name}", type)) - ); + return Map.ofEntries(Map.entry("type", "runtime"), Map.entry("runtime_type", type)); } - protected static Map runtimeFieldLoadingFromSource(String name, String type) { - return Map.of("type", type, "script", painlessToLoadFromSource(name, type)); + protected static Map runtimeFieldLoadingFromSource(String type) { + return Map.of("type", type); } private ExecutableSection addIndexTemplate() { @@ -140,7 +108,7 @@ public XContentLocation getLocation() { public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { Map params = Map.of("name", "hack_dynamic_mappings", "create", "true"); List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { + for (String type : RUNTIME_TYPES) { if (type.equals("ip")) { // There isn't a dynamic template to pick up ips. They'll just look like strings. continue; @@ -322,13 +290,11 @@ protected final boolean runtimeifyMappingProperties(Map properti // Our source reading script doesn't emulate ignore_malformed continue; } - String toLoad = painlessToLoadFromSource(name, type); - if (toLoad == null) { + if (RUNTIME_TYPES.contains(type) == false) { continue; } Map runtimeConfig = new HashMap<>(propertyMap); runtimeConfig.put("type", type); - runtimeConfig.put("script", toLoad); runtimeConfig.remove("store"); runtimeConfig.remove("index"); runtimeConfig.remove("doc_values"); From b2b400b67e8f22fc2ff6c897c51b4addbfcf5505 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Dec 2020 10:47:52 +0000 Subject: [PATCH 24/27] Nix bogus assertion in getTimestampMillisRange() (#65720) This method is called without any guarantee that the shard hasn't been closed, so asserting that the shard is active is bogus. Instead we can proceed no matter what state the shard is in, and let the engine throw an `AlreadyClosedException` if needed. Closes #65713 --- .../java/org/elasticsearch/index/shard/IndexShard.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 8cbd7b6e54362..286788749b5f7 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1720,8 +1720,6 @@ public RecoveryState recoveryState() { @Override public ShardLongFieldRange getTimestampMillisRange() { - assert isReadAllowed(); - if (mapperService() == null) { return ShardLongFieldRange.UNKNOWN; // no mapper service, no idea if the field even exists } @@ -1731,11 +1729,10 @@ public ShardLongFieldRange getTimestampMillisRange() { } final DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) mappedFieldType; - final Engine engine = getEngine(); final ShardLongFieldRange rawTimestampFieldRange; try { - rawTimestampFieldRange = engine.getRawFieldRange(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); - } catch (IOException e) { + rawTimestampFieldRange = getEngine().getRawFieldRange(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD); + } catch (IOException | AlreadyClosedException e) { logger.debug("exception obtaining range for timestamp field", e); return ShardLongFieldRange.UNKNOWN; } From 2ebab21672bf372713830f28b2270df47a5658e2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Dec 2020 10:59:15 +0000 Subject: [PATCH 25/27] Disable BWC tests for backport of #65689 --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 4a7952374e44d..2ed2c894d0597 100644 --- a/build.gradle +++ b/build.gradle @@ -175,8 +175,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/65689" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") From e066c0cfd2a0be6da84904edd77d51cbe7ae712c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 2 Dec 2020 11:28:24 +0000 Subject: [PATCH 26/27] Try to make DeprecationHttpIT more robust (#65665) Closes #65589 hopefully. There is a test case that checks whether deprecation logs have been indexed. It uses `assertBusy` so that it keeps checking for a period, since it may take some time for the data to become available. However, the test nonetheless occasionally still fails, with no shards being available to search. In an attempt to address this, add a `_refresh` call on the deprecation data stream, and increase the `assertBusy` timeout to 30s. --- .../elasticsearch/xpack/deprecation/DeprecationHttpIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java index 8b3df929dccc7..a5d95656c7dcc 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java +++ b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; @@ -239,7 +240,8 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { assertBusy(() -> { Response response; try { - response = client().performRequest(new Request("GET", ".logs-deprecation-elasticsearch/_search")); + client().performRequest(new Request("POST", "/.logs-deprecation-elasticsearch/_refresh?ignore_unavailable=true")); + response = client().performRequest(new Request("GET", "/.logs-deprecation-elasticsearch/_search")); } catch (Exception e) { // It can take a moment for the index to be created. If it doesn't exist then the client // throws an exception. Translate it into an assertion error so that assertBusy() will @@ -305,7 +307,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception { ) ) ); - }); + }, 30, TimeUnit.SECONDS); } finally { configureWriteDeprecationLogsToIndex(null); client().performRequest(new Request("DELETE", "_data_stream/.logs-deprecation-elasticsearch")); From 01c343f3e277a488030e8476bde69a99a9edfd08 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Dec 2020 11:59:01 +0000 Subject: [PATCH 27/27] Fix BWC after backport of #65564 (#65727) Relates #65564 Relates #65689 --- build.gradle | 4 +- .../index/shard/IndexLongFieldRange.java | 37 +++++++------------ .../index/shard/ShardLongFieldRange.java | 26 ++++--------- .../action/shard/ShardStateActionTests.java | 3 +- 4 files changed, 25 insertions(+), 45 deletions(-) diff --git a/build.gradle b/build.gradle index 2ed2c894d0597..4a7952374e44d 100644 --- a/build.gradle +++ b/build.gradle @@ -175,8 +175,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/65689" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java b/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java index 3f6a6b0be058f..eb4b3ced2a14a 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexLongFieldRange.java @@ -34,8 +34,6 @@ import java.util.Objects; import java.util.stream.IntStream; -import static org.elasticsearch.index.shard.ShardLongFieldRange.LONG_FIELD_RANGE_VERSION_INTRODUCED; - /** * Class representing an (inclusive) range of {@code long} values in a field in an index which may comprise multiple shards. This * information is accumulated shard-by-shard, and we keep track of which shards are represented in this value. Only once all shards are @@ -120,33 +118,26 @@ public long getMax() { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { - if (this == NO_SHARDS) { - out.writeByte(WIRE_TYPE_NO_SHARDS); - } else if (this == UNKNOWN) { - out.writeByte(WIRE_TYPE_UNKNOWN); - } else if (this == EMPTY) { - out.writeByte(WIRE_TYPE_EMPTY); + if (this == NO_SHARDS) { + out.writeByte(WIRE_TYPE_NO_SHARDS); + } else if (this == UNKNOWN) { + out.writeByte(WIRE_TYPE_UNKNOWN); + } else if (this == EMPTY) { + out.writeByte(WIRE_TYPE_EMPTY); + } else { + out.writeByte(WIRE_TYPE_OTHER); + if (shards == null) { + out.writeBoolean(false); } else { - out.writeByte(WIRE_TYPE_OTHER); - if (shards == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeVIntArray(shards); - } - out.writeZLong(min); - out.writeZLong(max); + out.writeBoolean(true); + out.writeVIntArray(shards); } + out.writeZLong(min); + out.writeZLong(max); } } public static IndexLongFieldRange readFrom(StreamInput in) throws IOException { - if (in.getVersion().before(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { - // conservative treatment for BWC - return UNKNOWN; - } - final byte type = in.readByte(); switch (type) { case WIRE_TYPE_NO_SHARDS: diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java b/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java index b76cf2d89bc9c..f05c35331404f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardLongFieldRange.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.shard; -import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -32,8 +31,6 @@ */ public class ShardLongFieldRange implements Writeable { - public static final Version LONG_FIELD_RANGE_VERSION_INTRODUCED = Version.V_8_0_0; - /** * Sentinel value indicating an empty range, for instance because the field is missing or has no values. */ @@ -91,11 +88,6 @@ public String toString() { private static final byte WIRE_TYPE_EMPTY = (byte)2; public static ShardLongFieldRange readFrom(StreamInput in) throws IOException { - if (in.getVersion().before(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { - // conservative treatment for BWC - return UNKNOWN; - } - final byte type = in.readByte(); switch (type) { case WIRE_TYPE_UNKNOWN: @@ -111,16 +103,14 @@ public static ShardLongFieldRange readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(LONG_FIELD_RANGE_VERSION_INTRODUCED)) { - if (this == UNKNOWN) { - out.writeByte(WIRE_TYPE_UNKNOWN); - } else if (this == EMPTY) { - out.writeByte(WIRE_TYPE_EMPTY); - } else { - out.writeByte(WIRE_TYPE_OTHER); - out.writeZLong(min); - out.writeZLong(max); - } + if (this == UNKNOWN) { + out.writeByte(WIRE_TYPE_UNKNOWN); + } else if (this == EMPTY) { + out.writeByte(WIRE_TYPE_EMPTY); + } else { + out.writeByte(WIRE_TYPE_OTHER); + out.writeZLong(min); + out.writeZLong(max); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index b5c30f54b54f7..cd98f3ab6505e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -532,8 +532,7 @@ public void testStartedShardEntrySerialization() throws Exception { assertThat(deserialized.allocationId, equalTo(allocationId)); assertThat(deserialized.primaryTerm, equalTo(primaryTerm)); assertThat(deserialized.message, equalTo(message)); - assertThat(deserialized.timestampMillisRange, version.onOrAfter(ShardLongFieldRange.LONG_FIELD_RANGE_VERSION_INTRODUCED) ? - equalTo(timestampMillisRange) : sameInstance(ShardLongFieldRange.UNKNOWN)); + assertThat(deserialized.timestampMillisRange, equalTo(timestampMillisRange)); } }