From 6d1693ff49974b1aef9741551df33ad3e851cec4 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 29 Jan 2019 15:09:40 +0000 Subject: [PATCH 01/35] [ML] Prevent submit after autodetect worker is stopped (#37700) Runnables can be submitted to AutodetectProcessManager.AutodetectWorkerExecutorService without error after it has been shutdown which can lead to requests timing out as their handlers are never called by the terminated executor. This change throws an EsRejectedExecutionException if a runnable is submitted after after the shutdown and calls AbstractRunnable.onRejection on any tasks not run. Closes #37108 --- .../autodetect/AutodetectProcessManager.java | 27 ++++++- .../AutodetectProcessManagerTests.java | 70 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 21aa08e14f217..1d8f4f273601f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -77,6 +77,7 @@ import java.nio.file.Path; import java.time.Duration; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -831,7 +832,16 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE } @Override - public void execute(Runnable command) { + public synchronized void execute(Runnable command) { + if (isShutdown()) { + EsRejectedExecutionException rejected = new EsRejectedExecutionException("autodetect worker service has shutdown", true); + if (command instanceof AbstractRunnable) { + ((AbstractRunnable) command).onRejection(rejected); + } else { + throw rejected; + } + } + boolean added = queue.offer(contextHolder.preserveContext(command)); if (added == false) { throw new ElasticsearchStatusException("Unable to submit operation", RestStatus.TOO_MANY_REQUESTS); @@ -851,6 +861,21 @@ void start() { EsExecutors.rethrowErrors(contextHolder.unwrap(runnable)); } } + + synchronized (this) { + // if shutdown with tasks pending notify the handlers + if (queue.isEmpty() == false) { + List notExecuted = new ArrayList<>(); + queue.drainTo(notExecuted); + + for (Runnable runnable : notExecuted) { + if (runnable instanceof AbstractRunnable) { + ((AbstractRunnable) runnable).onRejection( + new EsRejectedExecutionException("unable to process as autodetect worker service has shutdown", true)); + } + } + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } finally { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 82788d4500b09..6b4fd270b1bb7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -54,6 +55,7 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange; import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -72,9 +74,11 @@ import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -125,6 +129,8 @@ public class AutodetectProcessManagerTests extends ESTestCase { private Quantiles quantiles = new Quantiles("foo", new Date(), "state"); private Set filters = new HashSet<>(); + private ThreadPool threadPool; + @Before public void setup() throws Exception { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); @@ -159,8 +165,16 @@ public void setup() throws Exception { handler.accept(buildAutodetectParams()); return null; }).when(jobResultsProvider).getAutodetectParams(any(), any(), any()); + + threadPool = new TestThreadPool("AutodetectProcessManagerTests"); } + @After + public void stopThreadPool() throws InterruptedException { + terminate(threadPool); + } + + public void testMaxOpenJobsSetting_givenDefault() { int maxOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(Settings.EMPTY); assertEquals(20, maxOpenJobs); @@ -690,6 +704,62 @@ public void testAutodetectWorkerExecutorServiceDoesNotSwallowErrors() { } } + public void testAutodetectWorkerExecutorService_SubmitAfterShutdown() { + AutodetectProcessManager.AutodetectWorkerExecutorService executor = + new AutodetectWorkerExecutorService(new ThreadContext(Settings.EMPTY)); + + threadPool.generic().execute(() -> executor.start()); + executor.shutdown(); + expectThrows(EsRejectedExecutionException.class, () -> executor.execute(() -> {})); + } + + public void testAutodetectWorkerExecutorService_TasksNotExecutedCallHandlerOnShutdown() + throws InterruptedException, ExecutionException { + AutodetectProcessManager.AutodetectWorkerExecutorService executor = + new AutodetectWorkerExecutorService(new ThreadContext(Settings.EMPTY)); + + CountDownLatch latch = new CountDownLatch(1); + + Future executorFinished = threadPool.generic().submit(() -> executor.start()); + + // run a task that will block while the others are queued up + executor.execute(() -> { + try { + latch.await(); + } catch (InterruptedException e) { + } + }); + + AtomicBoolean runnableShouldNotBeCalled = new AtomicBoolean(false); + executor.execute(() -> runnableShouldNotBeCalled.set(true)); + + AtomicInteger onFailureCallCount = new AtomicInteger(); + AtomicInteger doRunCallCount = new AtomicInteger(); + for (int i=0; i<2; i++) { + executor.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + onFailureCallCount.incrementAndGet(); + } + + @Override + protected void doRun() { + doRunCallCount.incrementAndGet(); + } + }); + } + + // now shutdown + executor.shutdown(); + latch.countDown(); + executorFinished.get(); + + assertFalse(runnableShouldNotBeCalled.get()); + // the AbstractRunnables should have had their callbacks called + assertEquals(2, onFailureCallCount.get()); + assertEquals(0, doRunCallCount.get()); + } + private AutodetectProcessManager createNonSpyManager(String jobId) { Client client = mock(Client.class); ThreadPool threadPool = mock(ThreadPool.class); From 218df3009a8d3224c416f9ab05893e637714e366 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 29 Jan 2019 10:23:05 -0500 Subject: [PATCH 02/35] Move update and delete by query to use seq# for optimistic concurrency control (#37857) The delete and update by query APIs both offer protection against overriding concurrent user changes to the documents they touch. They currently are using internal versioning. This PR changes that to rely on sequences numbers and primary terms. Relates #37639 Relates #36148 Relates #10708 --- docs/reference/docs/index_.asciidoc | 8 --- .../AbstractAsyncBulkByScrollAction.java | 26 ++++---- .../reindex/AsyncDeleteByQueryAction.java | 26 +++++--- .../index/reindex/TransportReindexAction.java | 17 ++--- .../reindex/TransportUpdateByQueryAction.java | 30 +++++---- .../reindex/remote/RemoteResponseParsers.java | 12 ++-- .../reindex/AsyncBulkByScrollActionTests.java | 9 +-- .../reindex/UpdateByQueryMetadataTests.java | 12 ++-- .../reindex/UpdateByQueryWithScriptTests.java | 3 +- .../test/delete_by_query/10_basic.yml | 64 ++++++++++++++++++- .../test/delete_by_query/40_versioning.yml | 4 ++ .../test/update_by_query/10_basic.yml | 49 +++++++++++++- .../test/update_by_query/40_versioning.yml | 3 + .../reindex/ClientScrollableHitSource.java | 10 +++ .../index/reindex/ScrollableHitSource.java | 32 ++++++++++ 15 files changed, 230 insertions(+), 75 deletions(-) diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 257b88289d87a..e8a681567d622 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -372,14 +372,6 @@ the current document version of 1. If the document was already updated and its version was set to 2 or higher, the indexing command will fail and result in a conflict (409 http status code). -WARNING: External versioning supports the value 0 as a valid version number. -This allows the version to be in sync with an external versioning system -where version numbers start from zero instead of one. It has the side effect -that documents with version number equal to zero can neither be updated -using the <> nor be deleted -using the <> as long as their -version number is equal to zero. - A nice side effect is that there is no need to maintain strict ordering of async indexing operations executed as a result of changes to a source database, as long as version numbers from the source database are used. diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index e55dab1c38f64..9617d2a3774da 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -35,7 +35,6 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.ParentTaskAssigningClient; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -50,6 +49,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.UpdateScript; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -88,7 +88,6 @@ public abstract class AbstractAsyncBulkByScrollActionrequest variables all representing child @@ -111,9 +110,10 @@ public abstract class AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> scriptApplier; - public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, - ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState, - ActionListener listener) { + public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, boolean needsSourceDocumentVersions, + boolean needsSourceDocumentSeqNoAndPrimaryTerm, Logger logger, ParentTaskAssigningClient client, + ThreadPool threadPool, Request mainRequest, ScriptService scriptService, + ActionListener listener) { this.task = task; if (!task.isWorker()) { @@ -125,7 +125,6 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par this.client = client; this.threadPool = threadPool; this.scriptService = scriptService; - this.clusterState = clusterState; this.mainRequest = mainRequest; this.listener = listener; BackoffPolicy backoffPolicy = buildBackoffPolicy(); @@ -137,11 +136,13 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't * another sort. */ - List> sorts = mainRequest.getSearchRequest().source().sorts(); + final SearchSourceBuilder sourceBuilder = mainRequest.getSearchRequest().source(); + List> sorts = sourceBuilder.sorts(); if (sorts == null || sorts.isEmpty()) { - mainRequest.getSearchRequest().source().sort(fieldSort("_doc")); + sourceBuilder.sort(fieldSort("_doc")); } - mainRequest.getSearchRequest().source().version(needsSourceDocumentVersions()); + sourceBuilder.version(needsSourceDocumentVersions); + sourceBuilder.seqNoAndPrimaryTerm(needsSourceDocumentSeqNoAndPrimaryTerm); } /** @@ -153,12 +154,7 @@ public BiFunction, ScrollableHitSource.Hit, RequestWrapper> // The default script applier executes a no-op return (request, searchHit) -> request; } - - /** - * Does this operation need the versions of the source documents? - */ - protected abstract boolean needsSourceDocumentVersions(); - + /** * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle * metadata or scripting. That will be handled by copyMetadata and diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java index c86911649ac34..5026ec0f79c57 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.reindex; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.client.ParentTaskAssigningClient; @@ -31,20 +32,20 @@ * Implementation of delete-by-query using scrolling and bulk. */ public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { + private final boolean useSeqNoForCAS; + public AsyncDeleteByQueryAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); + super(task, + // not all nodes support sequence number powered optimistic concurrency control, we fall back to version + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, + // all nodes support sequence number powered optimistic concurrency control and we can use it + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), + logger, client, threadPool, request, scriptService, listener); + useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been - * changed. - */ - return true; - } @Override protected boolean accept(ScrollableHitSource.Hit doc) { @@ -59,7 +60,12 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc delete.index(doc.getIndex()); delete.type(doc.getType()); delete.id(doc.getId()); - delete.version(doc.getVersion()); + if (useSeqNoForCAS) { + delete.setIfSeqNo(doc.getSeqNo()); + delete.setIfPrimaryTerm(doc.getPrimaryTerm()); + } else { + delete.version(doc.getVersion()); + } return wrap(delete); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 0acc9a7b37dbc..3073b17d5bf95 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -259,16 +259,13 @@ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We only need the source version if we're going to use it when write and we only do that when the destination request uses - * external versioning. - */ - return mainRequest.getDestination().versionType() != VersionType.INTERNAL; + super(task, + /* + * We only need the source version if we're going to use it when write and we only do that when the destination request uses + * external versioning. + */ + request.getDestination().versionType() != VersionType.INTERNAL, + false, logger, client, threadPool, request, scriptService, listener); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index fc86583832619..9ed7744f8a27a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.reindex; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; @@ -81,19 +82,19 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener * Simple implementation of update-by-query using scrolling and bulk. */ static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction { + + private final boolean useSeqNoForCAS; + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { - super(task, logger, client, threadPool, request, scriptService, clusterState, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - /* - * We always need the version of the source document so we can report a version conflict if we try to delete it and it has - * been changed. - */ - return true; + super(task, + // not all nodes support sequence number powered optimistic concurrency control, we fall back to version + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false, + // all nodes support sequence number powered optimistic concurrency control and we can use it + clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0), + logger, client, threadPool, request, scriptService, listener); + useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0); } @Override @@ -112,8 +113,13 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) index.type(doc.getType()); index.id(doc.getId()); index.source(doc.getSource(), doc.getXContentType()); - index.versionType(VersionType.INTERNAL); - index.version(doc.getVersion()); + if (useSeqNoForCAS) { + index.setIfSeqNo(doc.getSeqNo()); + index.setIfPrimaryTerm(doc.getPrimaryTerm()); + } else { + index.versionType(VersionType.INTERNAL); + index.version(doc.getVersion()); + } index.setPipeline(mainRequest.getPipeline()); return wrap(index); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java index c7e814237d6b9..ccd177b9174ab 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java @@ -21,13 +21,9 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.Version; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; -import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; -import org.elasticsearch.index.reindex.ScrollableHitSource.Response; -import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -37,6 +33,10 @@ import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit; +import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; +import org.elasticsearch.index.reindex.ScrollableHitSource.Response; +import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; import org.elasticsearch.search.SearchHits; import java.io.IOException; @@ -90,6 +90,8 @@ private RemoteResponseParsers() {} ParseField routingField = new ParseField("_routing"); ParseField ttlField = new ParseField("_ttl"); ParseField parentField = new ParseField("_parent"); + ParseField seqNoField = new ParseField("_seq_no"); + ParseField primaryTermField = new ParseField("_primary_term"); HIT_PARSER.declareString(BasicHit::setRouting, routingField); // Pre-2.0.0 routing come back in "fields" class Fields { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index e6aee3596b1bd..cd23bf03add59 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -677,13 +677,8 @@ private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeVal private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction { DummyAsyncBulkByScrollAction() { - super(testTask, AsyncBulkByScrollActionTests.this.logger, new ParentTaskAssigningClient(client, localNode, testTask), - client.threadPool(), testRequest, null, null, listener); - } - - @Override - protected boolean needsSourceDocumentVersions() { - return randomBoolean(); + super(testTask, randomBoolean(), randomBoolean(), AsyncBulkByScrollActionTests.this.logger, + new ParentTaskAssigningClient(client, localNode, testTask), client.threadPool(), testRequest, null, listener); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java index 3ce8884ff92fb..95ee787f13f63 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java @@ -19,12 +19,14 @@ package org.elasticsearch.index.reindex; -import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.index.reindex.ScrollableHitSource.Hit; public class UpdateByQueryMetadataTests - extends AbstractAsyncBulkByScrollActionMetadataTestCase { - public void testRoutingIsCopied() throws Exception { + extends AbstractAsyncBulkByScrollActionMetadataTestCase { + + public void testRoutingIsCopied() { IndexRequest index = new IndexRequest(); action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo")); assertEquals("foo", index.routing()); @@ -43,12 +45,12 @@ protected UpdateByQueryRequest request() { private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction { TestAction() { super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null, - UpdateByQueryMetadataTests.this.threadPool, request(), null, null, listener()); + UpdateByQueryMetadataTests.this.threadPool, request(), null, ClusterState.EMPTY_STATE, listener()); } @Override public AbstractAsyncBulkByScrollAction.RequestWrapper copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper request, - Hit doc) { + Hit doc) { return super.copyMetadata(request, doc); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java index 90b78b9e1080d..0eb2a1cfb7d0a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.script.ScriptService; import java.util.Date; @@ -54,6 +55,6 @@ protected UpdateByQueryRequest request() { @Override protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) { return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, - null, listener()); + ClusterState.EMPTY_STATE, listener()); } } diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml index 9f9a99f0f5c2e..7ce6b86c6b525 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml @@ -89,7 +89,11 @@ - is_false: response.task --- -"Response for version conflict": +"Response for version conflict (version powered)": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control + - do: indices.create: index: test @@ -143,6 +147,64 @@ - match: {count: 1} +--- +"Response for version conflict (seq no powered)": + - skip: + version: " - 6.6.99" + reason: reindex moved to rely on sequence numbers for concurrency control + + - do: + indices.create: + index: test + body: + settings: + index.refresh_interval: -1 + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + # Creates a new version for reindex to miss on scan. + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test2" } + + - do: + catch: conflict + delete_by_query: + index: test + body: + query: + match_all: {} + + - match: {deleted: 0} + - match: {version_conflicts: 1} + - match: {batches: 1} + - match: {failures.0.index: test} + - match: {failures.0.type: _doc} + - match: {failures.0.id: "1"} + - match: {failures.0.status: 409} + - match: {failures.0.cause.type: version_conflict_engine_exception} + - match: {failures.0.cause.reason: "/\\[_doc\\]\\[1\\]:.version.conflict,.required.seqNo.\\[\\d+\\]/"} + - match: {failures.0.cause.shard: /\d+/} + - match: {failures.0.cause.index: test} + - gte: { took: 0 } + + - do: + indices.refresh: {} + + - do: + count: + index: test + + - match: {count: 1} + --- "Response for version conflict with conflicts=proceed": - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml index 1e2afacca148e..8448b229b258b 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml @@ -1,5 +1,9 @@ --- "delete_by_query fails to delete documents with version number equal to zero": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control + - do: index: index: index1 diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml index d3881035d924f..eb76681c0b0d1 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml @@ -74,7 +74,10 @@ - is_false: response.deleted --- -"Response for version conflict": +"Response for version conflict (version powered)": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control - do: indices.create: index: test @@ -115,6 +118,50 @@ - match: {failures.0.cause.index: test} - gte: { took: 0 } +--- +"Response for version conflict (seq no powered)": + - skip: + version: " - 6.6.99" + reason: reindex moved to rely on sequence numbers for concurrency control + - do: + indices.create: + index: test + body: + settings: + index.refresh_interval: -1 + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test" } + - do: + indices.refresh: {} + # Creates a new version for reindex to miss on scan. + - do: + index: + index: test + type: _doc + id: 1 + body: { "text": "test2" } + + - do: + catch: conflict + update_by_query: + index: test + - match: {updated: 0} + - match: {version_conflicts: 1} + - match: {batches: 1} + - match: {failures.0.index: test} + - match: {failures.0.type: _doc} + - match: {failures.0.id: "1"} + - match: {failures.0.status: 409} + - match: {failures.0.cause.type: version_conflict_engine_exception} + - match: {failures.0.cause.reason: "/\\[_doc\\]\\[1\\]:.version.conflict,.required.seqNo.\\[\\d+\\]/"} + - match: {failures.0.cause.shard: /\d+/} + - match: {failures.0.cause.index: test} + - gte: { took: 0 } + --- "Response for version conflict with conflicts=proceed": - do: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml index 7d2083f925b99..d14691be53b83 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml @@ -24,6 +24,9 @@ --- "update_by_query fails to update documents with version number equal to zero": + - skip: + version: "6.7.0 - " + reason: reindex moved to rely on sequence numbers for concurrency control - do: index: index: index1 diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java index 7ba3013497990..4fc5770709c1b 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java @@ -241,6 +241,16 @@ public long getVersion() { return delegate.getVersion(); } + @Override + public long getSeqNo() { + return delegate.getSeqNo(); + } + + @Override + public long getPrimaryTerm() { + return delegate.getPrimaryTerm(); + } + @Override public String getRouting() { return fieldValue(RoutingFieldMapper.NAME); diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java index a3901bb7a568b..dc8d69ff4f0d1 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.search.SearchHit; import org.elasticsearch.threadpool.ThreadPool; @@ -190,6 +191,17 @@ public interface Hit { * internal APIs. */ long getVersion(); + + /** + * The sequence number of the match or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if sequence numbers weren't requested. + */ + long getSeqNo(); + + /** + * The primary term of the match or {@link SequenceNumbers#UNASSIGNED_PRIMARY_TERM} if sequence numbers weren't requested. + */ + long getPrimaryTerm(); + /** * The source of the hit. Returns null if the source didn't come back from the search, usually because it source wasn't stored at * all. @@ -217,6 +229,8 @@ public static class BasicHit implements Hit { private BytesReference source; private XContentType xContentType; private String routing; + private long seqNo; + private long primaryTerm; public BasicHit(String index, String type, String id, long version) { this.index = index; @@ -245,6 +259,16 @@ public long getVersion() { return version; } + @Override + public long getSeqNo() { + return seqNo; + } + + @Override + public long getPrimaryTerm() { + return primaryTerm; + } + @Override public BytesReference getSource() { return source; @@ -270,6 +294,14 @@ public BasicHit setRouting(String routing) { this.routing = routing; return this; } + + public void setSeqNo(long seqNo) { + this.seqNo = seqNo; + } + + public void setPrimaryTerm(long primaryTerm) { + this.primaryTerm = primaryTerm; + } } /** From 5f106a27eaaf6deded97b421230065aa3b487409 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 29 Jan 2019 15:41:35 +0000 Subject: [PATCH 03/35] [ML] Add meta information to all ML indices (#37964) This change adds a _meta field storing the version in which the index mappings were last updated to the 3 ML indices that didn't previously have one: - .ml-annotations - .ml-meta - .ml-notifications All other ML indices already had such a _meta field. This field will be useful if we ever need to automatically update the index mappings during a future upgrade. --- .../org/elasticsearch/xpack/core/ml/MlMetaIndex.java | 1 + .../xpack/core/ml/annotations/AnnotationIndex.java | 8 +++++--- .../core/ml/job/persistence/ElasticsearchMappings.java | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index 9014c415f16bb..b1ec651500e0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -29,6 +29,7 @@ public static XContentBuilder docMapping() throws IOException { XContentBuilder builder = jsonBuilder(); builder.startObject(); builder.startObject(TYPE); + ElasticsearchMappings.addMetaInformation(builder); ElasticsearchMappings.addDefaultMapping(builder); builder.startObject(ElasticsearchMappings.PROPERTIES) .startObject(Calendar.ID.getPreferredName()) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 843be6596c380..437aa40c925f2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -109,10 +109,11 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c } public static XContentBuilder annotationsMapping() throws IOException { - return jsonBuilder() + XContentBuilder builder = jsonBuilder() .startObject() - .startObject(ElasticsearchMappings.DOC_TYPE) - .startObject(ElasticsearchMappings.PROPERTIES) + .startObject(ElasticsearchMappings.DOC_TYPE); + ElasticsearchMappings.addMetaInformation(builder); + builder.startObject(ElasticsearchMappings.PROPERTIES) .startObject(Annotation.ANNOTATION.getPreferredName()) .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT) .endObject() @@ -143,5 +144,6 @@ public static XContentBuilder annotationsMapping() throws IOException { .endObject() .endObject() .endObject(); + return builder; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 0eb2e666916dc..d51a8f10e4a5a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -960,10 +960,10 @@ private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) th } public static XContentBuilder auditMessageMapping() throws IOException { - return jsonBuilder() - .startObject() - .startObject(AuditMessage.TYPE.getPreferredName()) - .startObject(PROPERTIES) + XContentBuilder builder = jsonBuilder().startObject() + .startObject(AuditMessage.TYPE.getPreferredName()); + addMetaInformation(builder); + builder.startObject(PROPERTIES) .startObject(Job.ID.getPreferredName()) .field(TYPE, KEYWORD) .endObject() @@ -987,6 +987,7 @@ public static XContentBuilder auditMessageMapping() throws IOException { .endObject() .endObject() .endObject(); + return builder; } static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException { From 1579ac032b690d98817f38be6911e758256bd08f Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 16:43:24 +0000 Subject: [PATCH 04/35] Added missing eclipse-build.gradle files (#37980) Eclipse build files were missing so .eclipse project files were not being generated. Closes #37973 --- libs/geo/src/main/eclipse-build.gradle | 3 +++ libs/geo/src/test/eclipse-build.gradle | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 libs/geo/src/main/eclipse-build.gradle create mode 100644 libs/geo/src/test/eclipse-build.gradle diff --git a/libs/geo/src/main/eclipse-build.gradle b/libs/geo/src/main/eclipse-build.gradle new file mode 100644 index 0000000000000..4864452846b13 --- /dev/null +++ b/libs/geo/src/main/eclipse-build.gradle @@ -0,0 +1,3 @@ + +// this is just shell gradle file for eclipse to have separate projects for geo src and tests +apply from: '../../build.gradle' diff --git a/libs/geo/src/test/eclipse-build.gradle b/libs/geo/src/test/eclipse-build.gradle new file mode 100644 index 0000000000000..8dc16debdde32 --- /dev/null +++ b/libs/geo/src/test/eclipse-build.gradle @@ -0,0 +1,6 @@ + +// this is just shell gradle file for eclipse to have separate projects for geo src and tests +apply from: '../../build.gradle' +dependencies { + testCompile project(':libs:elasticsearch-geo') +} \ No newline at end of file From d05a4b9d14bbcb0f8900d871478d3c97b72c5ba1 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 29 Jan 2019 18:56:20 +0200 Subject: [PATCH 05/35] Get Aliases with wildcard exclusion expression (#34230) This commit adds the code in the HTTP layer that will parse exclusion wildcard expressions. The existing code issues 404s for wildcards as well as explicit indices. But, in general, in an expression with exclude wildcards (-...*) following other include wildcards, there is no way to tell if the include wildcard produced no results or they were subsequently excluded. Therefore, the proposed change is breaking the behavior of 404s for wildcards. Specifically, no 404s will be returned for wildcards, even if they are not followed by exclude wildcards or the exclude wildcards could not possibly exclude what has previously been included. Only explicitly requested aliases will be called out as missing. --- .../test/indices.get_alias/10_basic.yml | 4 +- .../test/indices.get_alias/30_wildcards.yml | 140 +++++++++++++++ .../admin/indices/RestGetAliasesAction.java | 164 ++++++++++-------- .../indices/RestGetAliasesActionTests.java | 134 ++++++++++++++ 4 files changed, 367 insertions(+), 75 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml index 6338598de05d0..f3dc3ac86bc9f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/10_basic.yml @@ -59,7 +59,7 @@ setup: - do: indices.get_alias: - name: _all + name: '*' - match: {test_index.aliases.test_alias: {}} - match: {test_index.aliases.test_blias: {}} @@ -220,7 +220,7 @@ setup: - is_false: test_index_2.aliases.test_blias --- -"Get aliases via /pref*/_alias/{name}": +"Get aliases via /*suf/_alias/{name}": - do: indices.get_alias: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml new file mode 100644 index 0000000000000..08b3009be0e88 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_alias/30_wildcards.yml @@ -0,0 +1,140 @@ +--- +setup: + + - do: + indices.create: + index: test_index + body: + aliases: + test_alias_1: {} + test_alias_2: {} + test_blias_1: {} + test_blias_2: {} + test: {} + +--- +"Get aliases wildcard and inclusion": + - do: + indices.get_alias: + name: test_alias*,test_blias_1 + + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - match: {test_index.aliases.test_blias_1: {}} + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + +--- +"Get aliases wildcard and simple exclusion": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_blias_2,test_alias*,-test_alias_1 + + - is_false: test_index.aliases.test_alias_1 + - match: {test_index.aliases.test_alias_2: {}} + - is_false: test_index.aliases.test_blias_1 + - match: {test_index.aliases.test_blias_2: {}} + - is_false: test_index.aliases.test + +--- +"Get aliases and wildcard exclusion": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias_1,test_blias_1,-test_alias* + + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - match: {test_index.aliases.test_blias_1: {}} + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + + - do: + indices.get_alias: + name: test_blias_2,tes*,-test_alias* + + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - match: {test_index.aliases.test_blias_1: {}} + - match: {test_index.aliases.test_blias_2: {}} + - match: {test_index.aliases.test: {}} + +--- +"Non-existent exclusion alias before wildcard returns 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + catch: missing + indices.get_alias: + name: -test_alias_1,test_alias*,-test_alias_2 + + - match: { 'status': 404} + - match: { 'error': 'alias [-test_alias_1] missing' } + - match: {test_index.aliases.test_alias_1: {}} + - is_false: test_index.aliases.test_alias_2 + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + + - do: + catch: missing + indices.get_alias: + name: -test_alias_1,-non-existing,test_alias*,-test + + - match: { 'status': 404} + - match: { 'error': 'aliases [-non-existing,-test_alias_1] missing' } + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test + +--- +"Missing exclusions does not fire 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias*,-non-existent,test_blias*,-test + + - match: {test_index.aliases.test_alias_1: {}} + - match: {test_index.aliases.test_alias_2: {}} + - match: {test_index.aliases.test_blias_1: {}} + - match: {test_index.aliases.test_blias_2: {}} + - is_false: test_index.aliases.test + +--- +"Exclusion of non wildcarded aliases": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + indices.get_alias: + name: test_alias_1,test_blias_2,-test_alias*,-test_blias_2 + + - match: { '': {}} + +--- +"Wildcard exclusions does not trigger 404": + - skip: + version: " - 6.99.99" + reason: Exclusions in the alias expression are not handled + - do: + catch: missing + indices.get_alias: + name: -non-existent,-non-existent*,-another + + - match: { 'status': 404} + - match: { 'error': 'alias [-non-existent] missing' } + - is_false: test_index.aliases.test_alias_1 + - is_false: test_index.aliases.test_alias_2 + - is_false: test_index.aliases.test_blias_1 + - is_false: test_index.aliases.test_blias_2 + - is_false: test_index.aliases.test diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index 0d6d46e95b602..8cdf9e62b1096 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -25,11 +25,11 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; @@ -41,14 +41,12 @@ import org.elasticsearch.rest.action.RestBuilderListener; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; import java.util.SortedSet; -import java.util.stream.Collectors; +import java.util.TreeSet; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; @@ -75,6 +73,94 @@ public String getName() { return "get_aliases_action"; } + static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String[] requestedAliases, + ImmutableOpenMap> responseAliasMap, XContentBuilder builder) throws Exception { + final Set indicesToDisplay = new HashSet<>(); + final Set returnedAliasNames = new HashSet<>(); + for (final ObjectObjectCursor> cursor : responseAliasMap) { + for (final AliasMetaData aliasMetaData : cursor.value) { + if (aliasesExplicitlyRequested) { + // only display indices that have aliases + indicesToDisplay.add(cursor.key); + } + returnedAliasNames.add(aliasMetaData.alias()); + } + } + // compute explicitly requested aliases that have are not returned in the result + final SortedSet missingAliases = new TreeSet<>(); + // first wildcard index, leading "-" as an alias name after this index means + // that it is an exclusion + int firstWildcardIndex = requestedAliases.length; + for (int i = 0; i < requestedAliases.length; i++) { + if (Regex.isSimpleMatchPattern(requestedAliases[i])) { + firstWildcardIndex = i; + break; + } + } + for (int i = 0; i < requestedAliases.length; i++) { + if (MetaData.ALL.equals(requestedAliases[i]) || Regex.isSimpleMatchPattern(requestedAliases[i]) + || (i > firstWildcardIndex && requestedAliases[i].charAt(0) == '-')) { + // only explicitly requested aliases will be called out as missing (404) + continue; + } + // check if aliases[i] is subsequently excluded + int j = Math.max(i + 1, firstWildcardIndex); + for (; j < requestedAliases.length; j++) { + if (requestedAliases[j].charAt(0) == '-') { + // this is an exclude pattern + if (Regex.simpleMatch(requestedAliases[j].substring(1), requestedAliases[i]) + || MetaData.ALL.equals(requestedAliases[j].substring(1))) { + // aliases[i] is excluded by aliases[j] + break; + } + } + } + if (j == requestedAliases.length) { + // explicitly requested aliases[i] is not excluded by any subsequent "-" wildcard in expression + if (false == returnedAliasNames.contains(requestedAliases[i])) { + // aliases[i] is not in the result set + missingAliases.add(requestedAliases[i]); + } + } + } + + final RestStatus status; + builder.startObject(); + { + if (missingAliases.isEmpty()) { + status = RestStatus.OK; + } else { + status = RestStatus.NOT_FOUND; + final String message; + if (missingAliases.size() == 1) { + message = String.format(Locale.ROOT, "alias [%s] missing", Strings.collectionToCommaDelimitedString(missingAliases)); + } else { + message = String.format(Locale.ROOT, "aliases [%s] missing", Strings.collectionToCommaDelimitedString(missingAliases)); + } + builder.field("error", message); + builder.field("status", status.getStatus()); + } + + for (final ObjectObjectCursor> entry : responseAliasMap) { + if (aliasesExplicitlyRequested == false || (aliasesExplicitlyRequested && indicesToDisplay.contains(entry.key))) { + builder.startObject(entry.key); + { + builder.startObject("aliases"); + { + for (final AliasMetaData alias : entry.value) { + AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS); + } + } + builder.endObject(); + } + builder.endObject(); + } + } + } + builder.endObject(); + return new BytesRestResponse(status, builder); + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { // The TransportGetAliasesAction was improved do the same post processing as is happening here. @@ -94,76 +180,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetAliasesResponse response, XContentBuilder builder) throws Exception { - final ImmutableOpenMap> aliasMap = response.getAliases(); - - final Set aliasNames = new HashSet<>(); - final Set indicesToDisplay = new HashSet<>(); - for (final ObjectObjectCursor> cursor : aliasMap) { - for (final AliasMetaData aliasMetaData : cursor.value) { - aliasNames.add(aliasMetaData.alias()); - if (namesProvided) { - indicesToDisplay.add(cursor.key); - } - } - } - - // first remove requested aliases that are exact matches - final SortedSet difference = Sets.sortedDifference(Arrays.stream(aliases).collect(Collectors.toSet()), aliasNames); - - // now remove requested aliases that contain wildcards that are simple matches - final List matches = new ArrayList<>(); - outer: - for (final String pattern : difference) { - if (pattern.contains("*")) { - for (final String aliasName : aliasNames) { - if (Regex.simpleMatch(pattern, aliasName)) { - matches.add(pattern); - continue outer; - } - } - } - } - difference.removeAll(matches); - - final RestStatus status; - builder.startObject(); - { - if (difference.isEmpty()) { - status = RestStatus.OK; - } else { - status = RestStatus.NOT_FOUND; - final String message; - if (difference.size() == 1) { - message = String.format(Locale.ROOT, "alias [%s] missing", - Strings.collectionToCommaDelimitedString(difference)); - } else { - message = String.format(Locale.ROOT, "aliases [%s] missing", - Strings.collectionToCommaDelimitedString(difference)); - } - builder.field("error", message); - builder.field("status", status.getStatus()); - } - - for (final ObjectObjectCursor> entry : response.getAliases()) { - if (namesProvided == false || (namesProvided && indicesToDisplay.contains(entry.key))) { - builder.startObject(entry.key); - { - builder.startObject("aliases"); - { - for (final AliasMetaData alias : entry.value) { - AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS); - } - } - builder.endObject(); - } - builder.endObject(); - } - } - } - builder.endObject(); - return new BytesRestResponse(status, builder); + return buildRestResponse(namesProvided, aliases, response.getAliases(), builder); } - }); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java new file mode 100644 index 0000000000000..ced52096687a2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.rest.RestStatus.OK; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.hamcrest.Matchers.equalTo; + +public class RestGetAliasesActionTests extends ESTestCase { + +// # Assumes the following setup +// curl -X PUT "localhost:9200/index" -H "Content-Type: application/json" -d' +// { +// "aliases": { +// "foo": {}, +// "foobar": {} +// } +// }' + + public void testBareRequest() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData foobarAliasMetaData = AliasMetaData.builder("foobar").build(); + final AliasMetaData fooAliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(fooAliasMetaData, foobarAliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(false, new String[0], openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{},\"foobar\":{}}}}")); + } + + public void testSimpleAliasWildcardMatchingNothing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*" }, openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } + + public void testMultipleAliasWildcardsSomeMatching() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foobar").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*", "foobar*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foobar\":{}}}}")); + } + + public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foob*", "-foo*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } + + public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo*", "-foob*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{}}}}")); + } + + public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final AliasMetaData aliasMetaData = AliasMetaData.builder("foo").build(); + openMapBuilder.put("index", Arrays.asList(aliasMetaData)); + final String[] aliasPattern; + if (randomBoolean()) { + aliasPattern = new String[] { "missing", "foo*", "-foob*" }; + } else { + aliasPattern = new String[] { "foo*", "-foob*", "missing" }; + } + + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, aliasPattern, openMapBuilder.build(), + xContentBuilder); + assertThat(restResponse.status(), equalTo(NOT_FOUND)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), + equalTo("{\"error\":\"alias [missing] missing\",\"status\":404,\"index\":{\"aliases\":{\"foo\":{}}}}")); + } + + public void testAliasWildcardsExcludeExplicitMissing() throws Exception { + final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final ImmutableOpenMap.Builder> openMapBuilder = ImmutableOpenMap.builder(); + final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo", "foofoo", "-foo*" }, + openMapBuilder.build(), xContentBuilder); + assertThat(restResponse.status(), equalTo(OK)); + assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.content().utf8ToString(), equalTo("{}")); + } +} From 00ace369af0c9b85dca78a57b083c65b7ffd823c Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 11:47:29 -0700 Subject: [PATCH 06/35] Use `CcrRepository` to init follower index (#35719) This commit modifies the put follow index action to use a CcrRepository when creating a follower index. It routes the logic through the snapshot/restore process. A wait_for_active_shards parameter can be used to configure how long to wait before returning the response. --- .../client/CcrRequestConverters.java | 2 + .../client/ccr/PutFollowRequest.java | 18 +- .../java/org/elasticsearch/client/CCRIT.java | 3 +- .../documentation/CCRDocumentationIT.java | 14 +- .../high-level/ccr/put_follow.asciidoc | 2 + .../ccr/apis/follow/get-follow-info.asciidoc | 2 +- .../ccr/apis/follow/get-follow-stats.asciidoc | 2 +- .../apis/follow/post-pause-follow.asciidoc | 2 +- .../apis/follow/post-resume-follow.asciidoc | 2 +- .../ccr/apis/follow/post-unfollow.asciidoc | 2 +- .../ccr/apis/follow/put-follow.asciidoc | 9 +- .../reference/ccr/apis/get-ccr-stats.asciidoc | 2 +- docs/reference/ccr/getting-started.asciidoc | 2 +- .../restore/RestoreClusterStateListener.java | 87 ++++++++ .../TransportRestoreSnapshotAction.java | 43 +--- .../test/ccr/follow_and_unfollow.yml | 1 + .../rest-api-spec/test/ccr/follow_info.yml | 1 + .../rest-api-spec/test/ccr/follow_stats.yml | 1 + .../index_directly_into_follower_index.yml | 1 + .../xpack/ccr/ESCCRRestTestCase.java | 3 +- .../ccr/action/TransportPutFollowAction.java | 193 +++++++++--------- .../xpack/ccr/repository/CcrRepository.java | 18 +- .../xpack/ccr/rest/RestPutFollowAction.java | 4 +- .../elasticsearch/xpack/CcrIntegTestCase.java | 13 +- .../xpack/CcrSingleNodeTestCase.java | 2 + .../xpack/ccr/IndexFollowingIT.java | 147 ++++++++++++- .../xpack/ccr/LocalIndexFollowingIT.java | 6 +- .../action/PutFollowActionRequestTests.java | 3 +- .../core/ccr/action/PutFollowAction.java | 62 +++++- .../indexlifecycle/CCRIndexLifecycleIT.java | 5 +- .../rest-api-spec/api/ccr.follow.json | 7 + 31 files changed, 489 insertions(+), 170 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java index df1e5dc01aef5..526db2a86a761 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java @@ -46,6 +46,8 @@ static Request putFollow(PutFollowRequest putFollowRequest) throws IOException { .addPathPartAsIs("_ccr", "follow") .build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(request); + parameters.withWaitForActiveShards(putFollowRequest.waitForActiveShards()); request.setEntity(createEntity(putFollowRequest, REQUEST_BODY_CONTENT_TYPE)); return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java index 98e9d224564cf..8307b04bd7087 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.ccr; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.Validatable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -36,11 +37,17 @@ public final class PutFollowRequest extends FollowConfig implements Validatable, private final String remoteCluster; private final String leaderIndex; private final String followerIndex; + private final ActiveShardCount waitForActiveShards; public PutFollowRequest(String remoteCluster, String leaderIndex, String followerIndex) { + this(remoteCluster, leaderIndex, followerIndex, ActiveShardCount.NONE); + } + + public PutFollowRequest(String remoteCluster, String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) { this.remoteCluster = Objects.requireNonNull(remoteCluster, "remoteCluster"); this.leaderIndex = Objects.requireNonNull(leaderIndex, "leaderIndex"); this.followerIndex = Objects.requireNonNull(followerIndex, "followerIndex"); + this.waitForActiveShards = waitForActiveShards; } @Override @@ -66,13 +73,18 @@ public String getFollowerIndex() { return followerIndex; } + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; PutFollowRequest that = (PutFollowRequest) o; - return Objects.equals(remoteCluster, that.remoteCluster) && + return Objects.equals(waitForActiveShards, that.waitForActiveShards) && + Objects.equals(remoteCluster, that.remoteCluster) && Objects.equals(leaderIndex, that.leaderIndex) && Objects.equals(followerIndex, that.followerIndex); } @@ -83,7 +95,7 @@ public int hashCode() { super.hashCode(), remoteCluster, leaderIndex, - followerIndex - ); + followerIndex, + waitForActiveShards); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java index 97a379aa16a90..ee2685dee6d92 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ccr.CcrStatsRequest; import org.elasticsearch.client.ccr.CcrStatsResponse; @@ -95,7 +96,7 @@ public void testIndexFollowing() throws Exception { CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); assertThat(response.isAcknowledged(), is(true)); - PutFollowRequest putFollowRequest = new PutFollowRequest("local_cluster", "leader", "follower"); + PutFollowRequest putFollowRequest = new PutFollowRequest("local_cluster", "leader", "follower", ActiveShardCount.ONE); PutFollowResponse putFollowResponse = execute(putFollowRequest, ccrClient::putFollow, ccrClient::putFollowAsync); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java index 9ef01a1a6f7bf..2e54d1c4a1a7c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -97,7 +98,8 @@ public void testPutFollow() throws Exception { PutFollowRequest putFollowRequest = new PutFollowRequest( "local", // <1> "leader", // <2> - "follower" // <3> + "follower", // <3> + ActiveShardCount.ONE // <4> ); // end::ccr-put-follow-request @@ -175,7 +177,7 @@ public void testPauseFollow() throws Exception { String followIndex = "follower"; // Follow index, so that it can be paused: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -241,7 +243,7 @@ public void testResumeFollow() throws Exception { String followIndex = "follower"; // Follow index, so that it can be paused: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -317,7 +319,7 @@ public void testUnfollow() throws Exception { String followIndex = "follower"; // Follow index, pause and close, so that it can be unfollowed: { - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -349,7 +351,7 @@ public void testUnfollow() throws Exception { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(followIndex); assertThat(client.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT).isAcknowledged(), is(true)); - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", followIndex, ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -639,7 +641,7 @@ public void testGetFollowStats() throws Exception { } { // Follow index, so that we can query for follow stats: - PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", "follower"); + PutFollowRequest putFollowRequest = new PutFollowRequest("local", "leader", "follower", ActiveShardCount.ONE); PutFollowResponse putFollowResponse = client.ccr().putFollow(putFollowRequest, RequestOptions.DEFAULT); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); diff --git a/docs/java-rest/high-level/ccr/put_follow.asciidoc b/docs/java-rest/high-level/ccr/put_follow.asciidoc index 839a9bedc6552..2f40bbd5d2b2d 100644 --- a/docs/java-rest/high-level/ccr/put_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/put_follow.asciidoc @@ -20,6 +20,8 @@ include-tagged::{doc-tests-file}[{api}-request] <1> The name of the remote cluster alias. <2> The name of the leader in the remote cluster. <3> The name of the follower index that gets created as part of the put follow API call. +<4> The number of active shard copies to wait for before the put follow API returns a +response, as an `ActiveShardCount` [id="{upid}-{api}-response"] ==== Response diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc index 22418db10887c..212b1167b6e33 100644 --- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc @@ -22,7 +22,7 @@ replication options and whether the follower indices are active or paused. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc index 766f502ff93a3..8c02582e01278 100644 --- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc @@ -21,7 +21,7 @@ following tasks associated with each shard for the specified indices. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc index 0d56ee76bd9b9..f5b0bef7b2994 100644 --- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -24,7 +24,7 @@ following task. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index e8b4cd50f27e7..736061f2bfde8 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -23,7 +23,7 @@ returns, the follower index will resume fetching operations from the leader inde [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc index 6507c04ac5026..c3126d02d1efc 100644 --- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -27,7 +27,7 @@ irreversible operation. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index 3f6156c1e6820..52253d6ad2f4c 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -31,7 +31,7 @@ POST /follower_index/_ccr/pause_follow [source,js] -------------------------------------------------- -PUT //_ccr/follow +PUT //_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "", "leader_index" : "" @@ -43,6 +43,11 @@ PUT //_ccr/follow // TEST[s//remote_cluster/] // TEST[s//leader_index/] +The `wait_for_active_shards` parameter specifies the number of shards to wait on being active +before responding. This defaults to waiting on none of the shards to be active. A shard must +be restored from the leader index being active. Restoring a follower shard requires transferring +all the remote Lucene segment files to the follower index. + ==== Path Parameters `follower_index` (required):: @@ -73,7 +78,7 @@ This example creates a follower index named `follower_index`: [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index", diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index f47e49ee82674..8949de8787fa7 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -22,7 +22,7 @@ shard-level stats as in the <>. [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc index 1af236f7d86fb..7c59b8628052f 100644 --- a/docs/reference/ccr/getting-started.asciidoc +++ b/docs/reference/ccr/getting-started.asciidoc @@ -230,7 +230,7 @@ cluster. [source,js] -------------------------------------------------- -PUT /server-metrics-copy/_ccr/follow +PUT /server-metrics-copy/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "leader", "leader_index" : "server-metrics" diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java new file mode 100644 index 0000000000000..a74aad3ddb586 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreClusterStateListener.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.restore; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.RestoreService; + +import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; + +public class RestoreClusterStateListener implements ClusterStateListener { + + private static final Logger logger = LogManager.getLogger(RestoreClusterStateListener.class); + + private final ClusterService clusterService; + private final String uuid; + private final ActionListener listener; + + + private RestoreClusterStateListener(ClusterService clusterService, RestoreService.RestoreCompletionResponse response, + ActionListener listener) { + this.clusterService = clusterService; + this.uuid = response.getUuid(); + this.listener = listener; + } + + @Override + public void clusterChanged(ClusterChangedEvent changedEvent) { + final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); + final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); + if (prevEntry == null) { + // When there is a master failure after a restore has been started, this listener might not be registered + // on the current master and as such it might miss some intermediary cluster states due to batching. + // Clean up listener in that case and acknowledge completion of restore operation to client. + clusterService.removeListener(this); + listener.onResponse(new RestoreSnapshotResponse(null)); + } else if (newEntry == null) { + clusterService.removeListener(this); + ImmutableOpenMap shards = prevEntry.shards(); + assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); + assert RestoreService.completed(shards) : "expected all restore entries to be completed"; + RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), + prevEntry.indices(), + shards.size(), + shards.size() - RestoreService.failedShards(shards)); + RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); + logger.debug("restore of [{}] completed", prevEntry.snapshot().getSnapshotId()); + listener.onResponse(response); + } else { + // restore not completed yet, wait for next cluster state update + } + } + + /** + * Creates a cluster state listener and registers it with the cluster service. The listener passed as a + * parameter will be called when the restore is complete. + */ + public static void createAndRegisterListener(ClusterService clusterService, RestoreService.RestoreCompletionResponse response, + ActionListener listener) { + clusterService.addListener(new RestoreClusterStateListener(clusterService, response, listener)); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index b362be49b10ab..d8dcc5eb8f846 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -22,26 +22,17 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.RestoreService.RestoreCompletionResponse; -import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; - /** * Transport action for restore snapshot operation */ @@ -86,39 +77,7 @@ protected void masterOperation(final RestoreSnapshotRequest request, final Clust @Override public void onResponse(RestoreCompletionResponse restoreCompletionResponse) { if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) { - final Snapshot snapshot = restoreCompletionResponse.getSnapshot(); - String uuid = restoreCompletionResponse.getUuid(); - - ClusterStateListener clusterStateListener = new ClusterStateListener() { - @Override - public void clusterChanged(ClusterChangedEvent changedEvent) { - final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), uuid); - final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), uuid); - if (prevEntry == null) { - // When there is a master failure after a restore has been started, this listener might not be registered - // on the current master and as such it might miss some intermediary cluster states due to batching. - // Clean up listener in that case and acknowledge completion of restore operation to client. - clusterService.removeListener(this); - listener.onResponse(new RestoreSnapshotResponse(null)); - } else if (newEntry == null) { - clusterService.removeListener(this); - ImmutableOpenMap shards = prevEntry.shards(); - assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state(); - assert RestoreService.completed(shards) : "expected all restore entries to be completed"; - RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), - prevEntry.indices(), - shards.size(), - shards.size() - RestoreService.failedShards(shards)); - RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri); - logger.debug("restore of [{}] completed", snapshot); - listener.onResponse(response); - } else { - // restore not completed yet, wait for next cluster state update - } - } - }; - - clusterService.addListener(clusterStateListener); + RestoreClusterStateListener.createAndRegisterListener(clusterService, restoreCompletionResponse, listener); } else { listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo())); } diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml index f73f5c6dfb2d3..d5cd8ebd4f1ab 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_and_unfollow.yml @@ -37,6 +37,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml index f1e47d830cf97..8383ecd4e6851 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_info.yml @@ -33,6 +33,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml index 5b3e6c18ef29b..220463a60b258 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/follow_stats.yml @@ -36,6 +36,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml index 60c3b404b6f09..62878437c37e3 100644 --- a/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml +++ b/x-pack/plugin/ccr/qa/rest/src/test/resources/rest-api-spec/test/ccr/index_directly_into_follower_index.yml @@ -37,6 +37,7 @@ - do: ccr.follow: index: bar + wait_for_active_shards: 1 body: remote_cluster: local leader_index: foo diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 25fbef7ada73e..6cdb6b37961ff 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -73,7 +73,7 @@ protected static void followIndex(String leaderCluster, String leaderIndex, Stri } protected static void followIndex(RestClient client, String leaderCluster, String leaderIndex, String followIndex) throws IOException { - final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow"); + final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow?wait_for_active_shards=1"); request.setJsonEntity("{\"remote_cluster\": \"" + leaderCluster + "\", \"leader_index\": \"" + leaderIndex + "\", \"read_poll_timeout\": \"10ms\"}"); assertOK(client.performRequest(request)); @@ -186,6 +186,7 @@ protected static Map toMap(String response) { protected static void ensureYellow(String index) throws IOException { Request request = new Request("GET", "/_cluster/health/" + index); request.addParameter("wait_for_status", "yellow"); + request.addParameter("wait_for_active_shards", "1"); request.addParameter("wait_for_no_relocating_shards", "true"); request.addParameter("wait_for_no_initializing_shards", "true"); request.addParameter("timeout", "70s"); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index bca8608904a76..27f3b60fb5291 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -6,52 +6,53 @@ package org.elasticsearch.xpack.ccr.action; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.ResourceAlreadyExistsException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreClusterStateListener; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; public final class TransportPutFollowAction - extends TransportMasterNodeAction { + extends TransportMasterNodeAction { + + private static final Logger logger = LogManager.getLogger(TransportPutFollowAction.class); private final Client client; - private final AllocationService allocationService; - private final ActiveShardsObserver activeShardsObserver; + private final RestoreService restoreService; private final CcrLicenseChecker ccrLicenseChecker; + private final ActiveShardsObserver activeShardsObserver; @Inject public TransportPutFollowAction( @@ -61,7 +62,7 @@ public TransportPutFollowAction( final ActionFilters actionFilters, final IndexNameExpressionResolver indexNameExpressionResolver, final Client client, - final AllocationService allocationService, + final RestoreService restoreService, final CcrLicenseChecker ccrLicenseChecker) { super( PutFollowAction.NAME, @@ -72,9 +73,9 @@ public TransportPutFollowAction( PutFollowAction.Request::new, indexNameExpressionResolver); this.client = client; - this.allocationService = allocationService; - this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); + this.restoreService = restoreService; this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); + this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool); } @Override @@ -96,7 +97,7 @@ protected PutFollowAction.Response read(StreamInput in) throws IOException { protected void masterOperation( final PutFollowAction.Request request, final ClusterState state, - final ActionListener listener) throws Exception { + final ActionListener listener) { if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); return; @@ -111,12 +112,11 @@ protected void masterOperation( remoteCluster, leaderIndex, listener::onFailure, - (historyUUID, leaderIndexMetaData) -> createFollowerIndex(leaderIndexMetaData, historyUUID, request, listener)); + (historyUUID, leaderIndexMetaData) -> createFollowerIndex(leaderIndexMetaData, request, listener)); } private void createFollowerIndex( final IndexMetaData leaderIndexMetaData, - final String[] historyUUIDs, final PutFollowAction.Request request, final ActionListener listener) { if (leaderIndexMetaData == null) { @@ -131,98 +131,107 @@ private void createFollowerIndex( return; } - ActionListener handler = ActionListener.wrap( - result -> { - if (result) { - initiateFollowing(request, listener); - } else { - listener.onResponse(new PutFollowAction.Response(true, false, false)); - } - }, - listener::onFailure); - // Can't use create index api here, because then index templates can alter the mappings / settings. - // And index templates could introduce settings / mappings that are incompatible with the leader index. - clusterService.submitStateUpdateTask("create_following_index", new AckedClusterStateUpdateTask(request, handler) { + final Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowerIndex()) + .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + final String leaderClusterRepoName = CcrRepository.NAME_PREFIX + request.getRemoteCluster(); + final RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) + .indices(request.getLeaderIndex()).indicesOptions(request.indicesOptions()).renamePattern("^(.*)$") + .renameReplacement(request.getFollowRequest().getFollowerIndex()).masterNodeTimeout(request.masterNodeTimeout()) + .indexSettings(settingsBuilder); + + final Client clientWithHeaders = CcrLicenseChecker.wrapClient(this.client, threadPool.getThreadContext().getHeaders()); + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() { @Override - protected Boolean newResponse(final boolean acknowledged) { - return acknowledged; + public void onFailure(Exception e) { + listener.onFailure(e); } @Override - public ClusterState execute(final ClusterState currentState) throws Exception { - String followIndex = request.getFollowRequest().getFollowerIndex(); - IndexMetaData currentIndex = currentState.metaData().index(followIndex); - if (currentIndex != null) { - throw new ResourceAlreadyExistsException(currentIndex.getIndex()); - } + protected void doRun() throws Exception { + restoreService.restoreSnapshot(restoreRequest, new ActionListener() { - MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - IndexMetaData.Builder imdBuilder = IndexMetaData.builder(followIndex); - - // Adding the leader index uuid for each shard as custom metadata: - Map metadata = new HashMap<>(); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", historyUUIDs)); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY, leaderIndexMetaData.getIndexUUID()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY, leaderIndexMetaData.getIndex().getName()); - metadata.put(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, request.getRemoteCluster()); - imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, metadata); - - // Copy all settings, but overwrite a few settings. - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(leaderIndexMetaData.getSettings()); - // Overwriting UUID here, because otherwise we can't follow indices in the same cluster - settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, followIndex); - settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); - settingsBuilder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); - imdBuilder.settings(settingsBuilder); - - // Copy mappings from leader IMD to follow IMD - for (ObjectObjectCursor cursor : leaderIndexMetaData.getMappings()) { - imdBuilder.putMapping(cursor.value); - } - imdBuilder.setRoutingNumShards(leaderIndexMetaData.getRoutingNumShards()); - IndexMetaData followIMD = imdBuilder.build(); - mdBuilder.put(followIMD, false); + @Override + public void onResponse(RestoreService.RestoreCompletionResponse response) { + afterRestoreStarted(clientWithHeaders, request, listener, response); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + }); + } - ClusterState.Builder builder = ClusterState.builder(currentState); - builder.metaData(mdBuilder.build()); - ClusterState updatedState = builder.build(); + private void afterRestoreStarted(Client clientWithHeaders, PutFollowAction.Request request, + ActionListener originalListener, + RestoreService.RestoreCompletionResponse response) { + final ActionListener listener; + if (ActiveShardCount.NONE.equals(request.waitForActiveShards())) { + originalListener.onResponse(new PutFollowAction.Response(true, false, false)); + listener = new ActionListener() { + + @Override + public void onResponse(PutFollowAction.Response response) { + logger.debug("put follow {} completed with {}", request, response); + } - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable()) - .addAsNew(updatedState.metaData().index(request.getFollowRequest().getFollowerIndex())); - updatedState = allocationService.reroute( - ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), - "follow index [" + request.getFollowRequest().getFollowerIndex() + "] created"); + @Override + public void onFailure(Exception e) { + logger.debug(() -> new ParameterizedMessage("put follow {} failed during the restore process", request), e); + } + }; + } else { + listener = originalListener; + } - logger.info("[{}] creating index, cause [ccr_create_and_follow], shards [{}]/[{}]", - followIndex, followIMD.getNumberOfShards(), followIMD.getNumberOfReplicas()); + RestoreClusterStateListener.createAndRegisterListener(clusterService, response, new ActionListener() { + @Override + public void onResponse(RestoreSnapshotResponse restoreSnapshotResponse) { + RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo(); + + if (restoreInfo == null) { + // If restoreInfo is null then it is possible there was a master failure during the + // restore. + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } else if (restoreInfo.failedShards() == 0) { + initiateFollowing(clientWithHeaders, request, listener); + } else { + assert restoreInfo.failedShards() > 0 : "Should have failed shards"; + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } + } - return updatedState; + @Override + public void onFailure(Exception e) { + listener.onFailure(e); } }); } private void initiateFollowing( - final PutFollowAction.Request request, - final ActionListener listener) { + final Client client, + final PutFollowAction.Request request, + final ActionListener listener) { + assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "PutFollowAction does not support DEFAULT."; activeShardsObserver.waitForActiveShards(new String[]{request.getFollowRequest().getFollowerIndex()}, - ActiveShardCount.DEFAULT, request.timeout(), result -> { - if (result) { - client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( - r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), - listener::onFailure - )); - } else { - listener.onResponse(new PutFollowAction.Response(true, false, false)); - } - }, listener::onFailure); + request.waitForActiveShards(), request.timeout(), result -> { + if (result) { + client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( + r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), + listener::onFailure + )); + } else { + listener.onResponse(new PutFollowAction.Response(true, false, false)); + } + }, listener::onFailure); } @Override protected ClusterBlockException checkBlock(final PutFollowAction.Request request, final ClusterState state) { return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getFollowRequest().getFollowerIndex()); } - } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 33a8c64c96138..7ca95a14909ae 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ccr.repository; +import com.carrotsearch.hppc.cursors.IntObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.index.IndexCommit; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; @@ -81,6 +83,7 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit public static final String TYPE = "_ccr_"; public static final String NAME_PREFIX = "_ccr_"; private static final SnapshotId SNAPSHOT_ID = new SnapshotId(LATEST, LATEST); + private static final String IN_SYNC_ALLOCATION_ID = "ccr_restore"; private final RepositoryMetaData metadata; private final CcrSettings ccrSettings; @@ -157,7 +160,7 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind ccrLicenseChecker.fetchLeaderHistoryUUIDs(remoteClient, leaderIndexMetaData, future::onFailure, future::onResponse); String[] leaderHistoryUUIDs = future.actionGet(); - IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData); + IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndex); // Adding the leader index uuid for each shard as custom metadata: Map metadata = new HashMap<>(); metadata.put(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_SHARD_HISTORY_UUIDS, String.join(",", leaderHistoryUUIDs)); @@ -166,6 +169,19 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind metadata.put(Ccr.CCR_CUSTOM_METADATA_REMOTE_CLUSTER_NAME_KEY, remoteClusterAlias); imdBuilder.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, metadata); + imdBuilder.settings(leaderIndexMetaData.getSettings()); + + // Copy mappings from leader IMD to follow IMD + for (ObjectObjectCursor cursor : leaderIndexMetaData.getMappings()) { + imdBuilder.putMapping(cursor.value); + } + + imdBuilder.setRoutingNumShards(leaderIndexMetaData.getRoutingNumShards()); + // We assert that insync allocation ids are not empty in `PrimaryShardAllocator` + for (IntObjectCursor> entry : leaderIndexMetaData.getInSyncAllocationIds()) { + imdBuilder.putInSyncAllocationIds(entry.key, Collections.singleton(IN_SYNC_ALLOCATION_ID)); + } + return imdBuilder.build(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java index 7b21422cb9867..d7a2edd21d26f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ccr.rest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; @@ -38,7 +39,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient static Request createRequest(RestRequest restRequest) throws IOException { try (XContentParser parser = restRequest.contentOrSourceParamParser()) { - return Request.fromXContent(parser, restRequest.param("index")); + ActiveShardCount waitForActiveShards = ActiveShardCount.parseString(restRequest.param("wait_for_active_shards")); + return Request.fromXContent(parser, restRequest.param("index"), waitForActiveShards); } } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 65fd80325e716..4a6c5411737c6 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; @@ -277,8 +278,13 @@ protected final ClusterHealthStatus ensureLeaderGreen(String... indices) { } protected final ClusterHealthStatus ensureFollowerGreen(String... indices) { + return ensureFollowerGreen(false, indices); + } + + protected final ClusterHealthStatus ensureFollowerGreen(boolean waitForNoInitializingShards, String... indices) { logger.info("ensure green follower indices {}", Arrays.toString(indices)); - return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), false, indices); + return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), + waitForNoInitializingShards, indices); } private ClusterHealthStatus ensureColor(TestCluster testCluster, @@ -411,10 +417,15 @@ protected String getIndexSettings(final int numberOfShards, final int numberOfRe } public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex) { + return putFollow(leaderIndex, followerIndex, ActiveShardCount.ONE); + } + + public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) { PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster("leader_cluster"); request.setLeaderIndex(leaderIndex); request.setFollowRequest(resumeFollow(followerIndex)); + request.waitForActiveShards(waitForActiveShards); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index ad8f545fa9dc0..48531c7d28f9a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -98,6 +99,7 @@ protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String request.setRemoteCluster("local"); request.setLeaderIndex(leaderIndex); request.setFollowRequest(getResumeFollowRequest(followerIndex)); + request.waitForActiveShards(ActiveShardCount.ONE); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 648c295efa817..dec671f6e6340 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; @@ -30,7 +31,10 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterIndexHealth; +import org.elasticsearch.cluster.health.ClusterShardHealth; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.service.ClusterService; @@ -91,14 +95,12 @@ public class IndexFollowingIT extends CcrIntegTestCase { public void testFollowIndex() throws Exception { final int numberOfPrimaryShards = randomIntBetween(1, 3); - final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + int numberOfReplicas = between(0, 1); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, numberOfReplicas, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); ensureLeaderYellow("index1"); - final PutFollowAction.Request followRequest = putFollow("index1", "index2"); - followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); - final int firstBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); for (int i = 0; i < firstBatchNumDocs; i++) { @@ -106,6 +108,30 @@ public void testFollowIndex() throws Exception { leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); } + boolean waitOnAll = randomBoolean(); + + final PutFollowAction.Request followRequest; + if (waitOnAll) { + followRequest = putFollow("index1", "index2", ActiveShardCount.ALL); + } else { + followRequest = putFollow("index1", "index2", ActiveShardCount.ONE); + } + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertTrue(response.isFollowIndexCreated()); + assertTrue(response.isFollowIndexShardsAcked()); + assertTrue(response.isIndexFollowingStarted()); + + ClusterHealthRequest healthRequest = Requests.clusterHealthRequest("index2").waitForNoRelocatingShards(true); + ClusterIndexHealth indexHealth = followerClient().admin().cluster().health(healthRequest).actionGet().getIndices().get("index2"); + for (ClusterShardHealth shardHealth : indexHealth.getShards().values()) { + if (waitOnAll) { + assertTrue(shardHealth.isPrimaryActive()); + assertEquals(1 + numberOfReplicas, shardHealth.getActiveShards()); + } else { + assertTrue(shardHealth.isPrimaryActive()); + } + } + final Map firstBatchNumDocsPerShard = new HashMap<>(); final ShardStats[] firstBatchShardStats = leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); @@ -152,6 +178,119 @@ public void testFollowIndex() throws Exception { assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), numberOfPrimaryShards); } + public void testFollowIndexWithConcurrentMappingChanges() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final int firstBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); + for (int i = 0; i < firstBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + AtomicBoolean isRunning = new AtomicBoolean(true); + + // Concurrently index new docs with mapping changes + Thread thread = new Thread(() -> { + int docID = 10000; + char[] chars = "abcdeghijklmnopqrstuvwxyz".toCharArray(); + for (char c : chars) { + if (isRunning.get() == false) { + break; + } + final String source; + long valueToPutInDoc = randomLongBetween(0, 50000); + if (randomBoolean()) { + source = String.format(Locale.ROOT, "{\"%c\":%d}", c, valueToPutInDoc); + } else { + source = String.format(Locale.ROOT, "{\"%c\":\"%d\"}", c, valueToPutInDoc); + } + for (int i = 1; i < 10; i++) { + if (isRunning.get() == false) { + break; + } + leaderClient().prepareIndex("index1", "doc", Long.toString(docID++)).setSource(source, XContentType.JSON).get(); + if (rarely()) { + leaderClient().admin().indices().prepareFlush("index1").setForce(true).get(); + } + } + leaderClient().admin().indices().prepareFlush("index1").setForce(true).setWaitIfOngoing(true).get(); + } + }); + thread.start(); + + final PutFollowAction.Request followRequest = putFollow("index1", "index2", ActiveShardCount.NONE); + followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + + ensureFollowerGreen("index2"); + + for (int i = 0; i < firstBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + + final int secondBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); + for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + + isRunning.set(false); + thread.join(); + } + + public void testFollowIndexWithoutWaitForComplete() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final int firstBatchNumDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs); + for (int i = 0; i < firstBatchNumDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + final PutFollowAction.Request followRequest = putFollow("index1", "index2", ActiveShardCount.NONE); + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + + assertTrue(response.isFollowIndexCreated()); + assertFalse(response.isFollowIndexShardsAcked()); + assertFalse(response.isIndexFollowingStarted()); + + // Check that the index exists, would throw index not found exception if the index is missing + followerClient().admin().indices().prepareGetIndex().addIndices("index2").get(); + ensureFollowerGreen(true, "index2"); + + final Map firstBatchNumDocsPerShard = new HashMap<>(); + final ShardStats[] firstBatchShardStats = + leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); + for (final ShardStats shardStats : firstBatchShardStats) { + if (shardStats.getShardRouting().primary()) { + long value = shardStats.getStats().getIndexing().getTotal().getIndexCount() - 1; + firstBatchNumDocsPerShard.put(shardStats.getShardRouting().shardId(), value); + } + } + + assertBusy(assertTask(numberOfPrimaryShards, firstBatchNumDocsPerShard)); + + for (int i = 0; i < firstBatchNumDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + assertTotalNumberOfOptimizedIndexing(resolveFollowerIndex("index2"), numberOfPrimaryShards, firstBatchNumDocs); + pauseFollow("index2"); + } + public void testSyncMappings() throws Exception { final String leaderIndexSettings = getIndexSettings(2, between(0, 1), singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 2c50411971a1b..f50f17c9e296d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -39,14 +39,14 @@ public void testFollowIndex() throws Exception { assertAcked(client().admin().indices().prepareCreate("leader").setSource(leaderIndexSettings, XContentType.JSON)); ensureGreen("leader"); - final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); - client().execute(PutFollowAction.INSTANCE, followRequest).get(); - final long firstBatchNumDocs = randomIntBetween(2, 64); for (int i = 0; i < firstBatchNumDocs; i++) { client().prepareIndex("leader", "doc").setSource("{}", XContentType.JSON).get(); } + final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); + client().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertBusy(() -> { assertThat(client().prepareSearch("follower").get().getHits().getTotalHits().value, equalTo(firstBatchNumDocs)); }); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java index 726a1c9893a50..d32a773ebe218 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ccr.action; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; @@ -30,7 +31,7 @@ protected PutFollowAction.Request createTestInstance() { @Override protected PutFollowAction.Request doParseInstance(XContentParser parser) throws IOException { - return PutFollowAction.Request.fromXContent(parser, null); + return PutFollowAction.Request.fromXContent(parser, null, ActiveShardCount.DEFAULT); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java index 47240a6a1da10..6ca91bd7f0dad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java @@ -6,10 +6,12 @@ package org.elasticsearch.xpack.core.ccr.action; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.ParseField; @@ -28,10 +30,10 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.FOLLOWER_INDEX_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_READ_REQUESTS; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_WRITE_REQUESTS; +import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; +import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_RETRY_DELAY_FIELD; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_COUNT; import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_SIZE; @@ -105,7 +107,8 @@ public static class Request extends AcknowledgedRequest implements Indi ObjectParser.ValueType.STRING); } - public static Request fromXContent(final XContentParser parser, final String followerIndex) throws IOException { + public static Request fromXContent(final XContentParser parser, final String followerIndex, ActiveShardCount waitForActiveShards) + throws IOException { Request request = PARSER.parse(parser, followerIndex); if (followerIndex != null) { if (request.getFollowRequest().getFollowerIndex() == null) { @@ -116,11 +119,13 @@ public static Request fromXContent(final XContentParser parser, final String fol } } } + request.waitForActiveShards(waitForActiveShards); return request; } private String remoteCluster; private String leaderIndex; + private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE; private ResumeFollowAction.Request followRequest; public Request() { @@ -142,6 +147,27 @@ public void setLeaderIndex(String leaderIndex) { this.leaderIndex = leaderIndex; } + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + + /** + * Sets the number of shard copies that should be active for follower index creation to + * return. Defaults to {@link ActiveShardCount#NONE}, which will not wait for any shards + * to be active. Set this value to {@link ActiveShardCount#DEFAULT} to wait for the primary + * shard to be active. Set this value to {@link ActiveShardCount#ALL} to wait for all shards + * (primary and all replicas) to be active before returning. + * + * @param waitForActiveShards number of active shard copies to wait on + */ + public void waitForActiveShards(ActiveShardCount waitForActiveShards) { + if (waitForActiveShards.equals(ActiveShardCount.DEFAULT)) { + this.waitForActiveShards = ActiveShardCount.NONE; + } else { + this.waitForActiveShards = waitForActiveShards; + } + } + public ResumeFollowAction.Request getFollowRequest() { return followRequest; } @@ -176,6 +202,10 @@ public Request(StreamInput in) throws IOException { super(in); remoteCluster = in.readString(); leaderIndex = in.readString(); + // TODO: Update after backport + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + waitForActiveShards(ActiveShardCount.readFrom(in)); + } followRequest = new ResumeFollowAction.Request(in); } @@ -184,6 +214,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(remoteCluster); out.writeString(leaderIndex); + // TODO: Update after backport + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + waitForActiveShards.writeTo(out); + } followRequest.writeTo(out); } @@ -206,12 +240,23 @@ public boolean equals(Object o) { Request request = (Request) o; return Objects.equals(remoteCluster, request.remoteCluster) && Objects.equals(leaderIndex, request.leaderIndex) && + Objects.equals(waitForActiveShards, request.waitForActiveShards) && Objects.equals(followRequest, request.followRequest); } @Override public int hashCode() { - return Objects.hash(remoteCluster, leaderIndex, followRequest); + return Objects.hash(remoteCluster, leaderIndex, waitForActiveShards, followRequest); + } + + @Override + public String toString() { + return "PutFollowAction.Request{" + + "remoteCluster='" + remoteCluster + '\'' + + ", leaderIndex='" + leaderIndex + '\'' + + ", waitForActiveShards=" + waitForActiveShards + + ", followRequest=" + followRequest + + '}'; } } @@ -280,6 +325,15 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(followIndexCreated, followIndexShardsAcked, indexFollowingStarted); } + + @Override + public String toString() { + return "PutFollowAction.Response{" + + "followIndexCreated=" + followIndexCreated + + ", followIndexShardsAcked=" + followIndexShardsAcked + + ", indexFollowingStarted=" + indexFollowingStarted + + '}'; + } } } diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index 65baeb5f168c4..f8ffce9cd817a 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -61,6 +61,7 @@ public void testBasicCCRAndILMIntegration() throws Exception { // Policy with the same name must exist in follower cluster too: putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7*24)); followIndex(indexName, indexName); + ensureGreen(indexName); // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster: client().performRequest(new Request("PUT", "/" + indexName + "/_alias/logs")); @@ -116,6 +117,7 @@ public void testCCRUnfollowDuringSnapshot() throws Exception { } else if ("follow".equals(targetCluster)) { createNewSingletonPolicy("unfollow-only", "hot", new UnfollowAction(), TimeValue.ZERO); followIndex(indexName, indexName); + ensureGreen(indexName); // Create the repository before taking the snapshot. Request request = new Request("PUT", "/_snapshot/repo"); @@ -210,7 +212,7 @@ public void testCcrAndIlmWithRollover() throws Exception { "\"mappings\": {\"_doc\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}, " + "\"aliases\": {\"" + alias + "\": {\"is_write_index\": true}} }"); assertOK(leaderClient.performRequest(createIndexRequest)); - // Check that the new index is creeg + // Check that the new index is created Request checkIndexRequest = new Request("GET", "/_cluster/health/" + indexName); checkIndexRequest.addParameter("wait_for_status", "green"); checkIndexRequest.addParameter("timeout", "70s"); @@ -226,6 +228,7 @@ public void testCcrAndIlmWithRollover() throws Exception { index(leaderClient, indexName, "1"); assertDocumentExists(leaderClient, indexName, "1"); + ensureGreen(indexName); assertBusy(() -> { assertDocumentExists(client(), indexName, "1"); // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json index 635a4e62683bb..588dd60261252 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json @@ -11,6 +11,13 @@ "required": true, "description": "The name of the follower index" } + }, + "params": { + "wait_for_active_shards": { + "type" : "string", + "description" : "Sets the number of shard copies that must be active before returning. Defaults to 0. Set to `all` for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)", + "default": "0" + } } }, "body": { From 9ca26b7e63b4f75f82842c432377ccee5cdc1e6f Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 29 Jan 2019 10:51:07 -0800 Subject: [PATCH 07/35] Remove more references to type in docs. (#37946) * Update the top-level 'getting started' guide. * Remove custom types from the painless getting started documentation. * Fix an incorrect references to '_doc' in the cardinality query docs. * Update the _update docs to use the typeless API format. --- docs/painless/painless-debugging.asciidoc | 10 ++++---- .../painless-getting-started.asciidoc | 6 ++--- .../metrics/cardinality-aggregation.asciidoc | 4 ++-- docs/reference/docs/update.asciidoc | 24 +++++++++---------- docs/reference/getting-started.asciidoc | 14 +++++------ 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/docs/painless/painless-debugging.asciidoc b/docs/painless/painless-debugging.asciidoc index c141cbc53252a..e335bfe74ebd7 100644 --- a/docs/painless/painless-debugging.asciidoc +++ b/docs/painless/painless-debugging.asciidoc @@ -18,10 +18,10 @@ context available to a {ref}/query-dsl-script-query.html[script query]. [source,js] --------------------------------------------------------- -PUT /hockey/player/1?refresh +PUT /hockey/_doc/1?refresh {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]} -POST /hockey/player/1/_explain +POST /hockey/_explain/1 { "query": { "script": { @@ -31,7 +31,7 @@ POST /hockey/player/1/_explain } --------------------------------------------------------- // CONSOLE -// TEST[s/_explain/_explain?error_trace=false/ catch:/painless_explain_error/] +// TEST[s/_explain\/1/_explain\/1?error_trace=false/ catch:/painless_explain_error/] // The test system sends error_trace=true by default for easier debugging so // we have to override it to get a normal shaped response @@ -58,13 +58,13 @@ in the `_update` API: [source,js] --------------------------------------------------------- -POST /hockey/player/1/_update +POST /hockey/_update/1 { "script": "Debug.explain(ctx._source)" } --------------------------------------------------------- // CONSOLE -// TEST[continued s/_update/_update?error_trace=false/ catch:/painless_explain_error/] +// TEST[continued s/_update\/1/_update\/1?error_trace=false/ catch:/painless_explain_error/] The response looks like: diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc index 936bd8e198c72..f562033471e31 100644 --- a/docs/painless/painless-getting-started.asciidoc +++ b/docs/painless/painless-getting-started.asciidoc @@ -10,7 +10,7 @@ To illustrate how Painless works, let's load some hockey stats into an Elasticse [source,js] ---------------------------------------------------------------- -PUT hockey/player/_bulk?refresh +PUT hockey/_bulk?refresh {"index":{"_id":1}} {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1],"born":"1993/08/13"} {"index":{"_id":2}} @@ -158,7 +158,7 @@ To change player 1's last name to `hockey`, simply set `ctx._source.last` to the [source,js] ---------------------------------------------------------------- -POST hockey/player/1/_update +POST hockey/_update/1 { "script": { "lang": "painless", @@ -176,7 +176,7 @@ the player's nickname, _hockey_. [source,js] ---------------------------------------------------------------- -POST hockey/player/1/_update +POST hockey/_update/1 { "script": { "lang": "painless", diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index a451c6da0db95..405a9d89107f2 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -49,7 +49,7 @@ POST /sales/_search?size=0 "aggs" : { "type_count" : { "cardinality" : { - "field" : "_doc", + "field" : "type", "precision_threshold": 100 <1> } } @@ -214,7 +214,7 @@ POST /sales/_search?size=0 "script" : { "id": "my_script", "params": { - "type_field": "_doc", + "type_field": "type", "promoted_field": "promoted" } } diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 42840b1b0a5ec..64c0f67bc722c 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -32,7 +32,7 @@ Now, we can execute a script that would increment the counter: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.counter += params.count", @@ -51,7 +51,7 @@ will still add it, since it's a list): [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.tags.add(params.tag)", @@ -73,7 +73,7 @@ remove only one occurrence of it: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "if (ctx._source.tags.contains(params.tag)) { ctx._source.tags.remove(ctx._source.tags.indexOf(params.tag)) }", @@ -95,7 +95,7 @@ We can also add a new field to the document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : "ctx._source.new_field = 'value_of_new_field'" } @@ -107,7 +107,7 @@ Or remove a field from the document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : "ctx._source.remove('new_field')" } @@ -121,7 +121,7 @@ the doc if the `tags` field contain `green`, otherwise it does nothing [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "if (ctx._source.tags.contains(params.tag)) { ctx.op = 'delete' } else { ctx.op = 'none' }", @@ -148,7 +148,7 @@ existing document: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -169,7 +169,7 @@ By default updates that don't change anything detect that they don't change anyt [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -204,7 +204,7 @@ You can disable this behavior by setting "detect_noop": false like this: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" @@ -225,7 +225,7 @@ will be inserted as a new document. If the document does exist, then the [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "script" : { "source": "ctx._source.counter += params.count", @@ -251,7 +251,7 @@ or not -- i.e. the script handles initializing the document instead of the [source,js] -------------------------------------------------- -POST sessions/session/dh3sgudg8gsrgl/_update +POST sessions/_update/dh3sgudg8gsrgl { "scripted_upsert":true, "script" : { @@ -280,7 +280,7 @@ value: [source,js] -------------------------------------------------- -POST test/_doc/1/_update +POST test/_update/1 { "doc" : { "name" : "new_name" diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index d8656f7ac4c25..bee2ae5194477 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -65,9 +65,7 @@ A type used to be a logical category/partition of your index to allow you to sto [float] === Document -A document is a basic unit of information that can be indexed. For example, you can have a document for a single customer, another document for a single product, and yet another for a single order. This document is expressed in http://json.org/[JSON] (JavaScript Object Notation) which is a ubiquitous internet data interchange format. - -Within an index/type, you can store as many documents as you want. Note that although a document physically resides in an index, a document actually must be indexed/assigned to a type inside an index. +A document is a basic unit of information that can be indexed. For example, you can have a document for a single customer, another document for a single product, and yet another for a single order. This document is expressed in http://json.org/[JSON] (JavaScript Object Notation) which is a ubiquitous internet data interchange format. Within an index, you can store as many documents as you want. [[getting-started-shards-and-replicas]] [float] @@ -496,7 +494,7 @@ If we study the above commands carefully, we can actually see a pattern of how w [source,js] -------------------------------------------------- - /// + /// -------------------------------------------------- // NOTCONSOLE @@ -572,7 +570,7 @@ This example shows how to update our previous document (ID of 1) by changing the [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "doc": { "name": "Jane Doe" } } @@ -584,7 +582,7 @@ This example shows how to update our previous document (ID of 1) by changing the [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "doc": { "name": "Jane Doe", "age": 20 } } @@ -596,7 +594,7 @@ Updates can also be performed by using simple scripts. This example uses a scrip [source,js] -------------------------------------------------- -POST /customer/_doc/1/_update?pretty +POST /customer/_update/1?pretty { "script" : "ctx._source.age += 5" } @@ -719,7 +717,7 @@ yellow open bank l7sSYV2cQXmu6_4rJWVIww 5 1 1000 0 12 // TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/] // TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ _cat] -Which means that we just successfully bulk indexed 1000 documents into the bank index (under the `_doc` type). +Which means that we just successfully bulk indexed 1000 documents into the bank index. [[getting-started-search-API]] === The Search API From 34d61d3231261794a58a918fce9642cf6699fbab Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 29 Jan 2019 12:51:34 -0600 Subject: [PATCH 08/35] ML: ignore unknown fields for JobTaskState (#37982) --- .../elasticsearch/xpack/core/ml/job/config/JobTaskState.java | 3 +-- .../elasticsearch/xpack/ml/job/config/JobTaskStateTests.java | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java index d979b897ad43a..08f73d791e53f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobTaskState.java @@ -29,8 +29,7 @@ public class JobTaskState implements PersistentTaskState { private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(NAME, - args -> new JobTaskState((JobState) args[0], (Long) args[1])); + new ConstructingObjectParser<>(NAME, true, args -> new JobTaskState((JobState) args[0], (Long) args[1])); static { PARSER.declareField(constructorArg(), p -> { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java index 4dfd1965804e5..26560f1034f9e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java @@ -27,4 +27,9 @@ protected Writeable.Reader instanceReader() { protected JobTaskState doParseInstance(XContentParser parser) { return JobTaskState.fromXContent(parser); } + + @Override + protected boolean supportsUnknownFields() { + return true; + } } From 697b2fbe520942a2212e1778b5b251138db93425 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 29 Jan 2019 21:10:03 +0200 Subject: [PATCH 09/35] Remove implicit index monitor privilege (#37774) Restricted indices (currently only .security-6 and .security) are special internal indices that require setting the `allow_restricted_indices` flag on every index permission that covers them. If this flag is `false` (default) the permission will not cover these and actions against them will not be authorized. However, the monitoring APIs were the only exception to this rule. This exception is herein forfeited and index monitoring privileges have to be granted explicitly, using the `allow_restricted_indices` flag on the permission, as is the case for any other index privilege. --- .../migration/migrate_7_0/api.asciidoc | 12 ++ .../authz/permission/IndicesPermission.java | 8 +- .../authz/store/ReservedRolesStoreTests.java | 25 ++++ .../MultipleIndicesPermissionsTests.java | 108 +++++++++++++- .../test/SecuritySettingsSource.java | 1 + .../authz/AuthorizationServiceTests.java | 141 +++++++++++------- 6 files changed, 230 insertions(+), 65 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc index 3972be43685cc..bb151edb778e2 100644 --- a/docs/reference/migration/migrate_7_0/api.asciidoc +++ b/docs/reference/migration/migrate_7_0/api.asciidoc @@ -126,3 +126,15 @@ removed. The `_termvector` endpoint was deprecated in 2.0 and has now been removed. The endpoint `_termvectors` (plural) should be used instead. + +[float] +==== When {security-features} are enabled, index monitoring APIs over restricted indices are not authorized implicitly anymore + +Restricted indices (currently only `.security-6` and `.security`) are special internal +indices that require setting the `allow_restricted_indices` flag on every index +permission that covers them. If this flag is `false` (default) the permission +will not cover these and actions against them will not be authorized. +However, the monitoring APIs were the only exception to this rule. This exception +has been forfeited and index monitoring privileges have to be granted explicitly, +using the `allow_restricted_indices` flag on the permission (as any other index +privilege). diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 4c2a479721a2a..27fa8b2cd9da0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -259,12 +259,8 @@ private boolean check(String action) { private boolean check(String action, String index) { assert index != null; - return check(action) && (indexNameMatcher.test(index) - && (allowRestrictedIndices - // all good if it is not restricted - || (false == RestrictedIndicesNames.NAMES_SET.contains(index)) - // allow monitor as a special case, even for restricted - || IndexPrivilege.MONITOR.predicate().test(action))); + return check(action) && indexNameMatcher.test(index) + && (allowRestrictedIndices || (false == RestrictedIndicesNames.NAMES_SET.contains(index))); } boolean hasQuery() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 8711a6c318e58..35e2043acd809 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -144,6 +144,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.hasEntry; @@ -563,9 +564,33 @@ public void testRemoteMonitoringCollectorRole() { assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndexAction.NAME) .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false)); + assertMonitoringOnRestrictedIndices(remoteMonitoringAgentRole); + assertNoAccessAllowed(remoteMonitoringAgentRole, RestrictedIndicesNames.NAMES_SET); } + private void assertMonitoringOnRestrictedIndices(Role role) { + final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build(); + final MetaData metaData = new MetaData.Builder() + .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX) + .settings(indexSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build()) + .build(), true) + .build(); + final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); + final List indexMonitoringActionNamesList = Arrays.asList(IndicesStatsAction.NAME, IndicesSegmentsAction.NAME, + GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME); + for (final String indexMonitoringActionName : indexMonitoringActionNamesList) { + final Map authzMap = role.indices().authorize(indexMonitoringActionName, + Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData, + fieldPermissionsCache); + assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true)); + assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true)); + } + } + public void testReportingUserRole() { final TransportRequest request = mock(TransportRequest.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index 675300e25760e..da03e9ffe3d1e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -7,14 +7,27 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.core.security.authc.support.Hasher; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; +import org.junit.After; +import org.junit.Before; import java.util.Collections; @@ -25,11 +38,29 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; public class MultipleIndicesPermissionsTests extends SecurityIntegTestCase { protected static final SecureString USERS_PASSWD = new SecureString("passwd".toCharArray()); + @Before + public void waitForSecurityIndexWritable() throws Exception { + // adds a dummy user to the native realm to force .security index creation + securityClient().preparePutUser("dummy_user", "password".toCharArray(), Hasher.BCRYPT, "missing_role").get(); + assertSecurityIndexActive(); + } + + @After + public void cleanupSecurityIndex() throws Exception { + super.deleteSecurityIndex(); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + @Override protected String configRoles() { return SecuritySettingsSource.TEST_ROLE + ":\n" + @@ -49,6 +80,12 @@ protected String configRoles() { " - names: 'a'\n" + " privileges: [all]\n" + "\n" + + "role_monitor_all_unrestricted_indices:\n" + + " cluster: [monitor]\n" + + " indices:\n" + + " - names: '*'\n" + + " privileges: [monitor]\n" + + "\n" + "role_b:\n" + " indices:\n" + " - names: 'b'\n" + @@ -60,14 +97,16 @@ protected String configUsers() { final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); return SecuritySettingsSource.CONFIG_STANDARD_USER + "user_a:" + usersPasswdHashed + "\n" + - "user_ab:" + usersPasswdHashed + "\n"; + "user_ab:" + usersPasswdHashed + "\n" + + "user_monitor:" + usersPasswdHashed + "\n"; } @Override protected String configUsersRoles() { return SecuritySettingsSource.CONFIG_STANDARD_USER_ROLES + "role_a:user_a,user_ab\n" + - "role_b:user_ab\n"; + "role_b:user_ab\n" + + "role_monitor_all_unrestricted_indices:user_monitor\n"; } public void testSingleRole() throws Exception { @@ -127,6 +166,71 @@ public void testSingleRole() throws Exception { assertHitCount(searchResponse, 1); } + public void testMonitorRestrictedWildcards() throws Exception { + + IndexResponse indexResponse = index("foo", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + indexResponse = index("foobar", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + indexResponse = index("foobarfoo", "type", jsonBuilder() + .startObject() + .field("name", "value") + .endObject()); + assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); + + refresh(); + + final Client client = client() + .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user_monitor", USERS_PASSWD))); + + final GetSettingsResponse getSettingsResponse = client.admin().indices().prepareGetSettings(randomFrom("*", "_all", "foo*")).get(); + assertThat(getSettingsResponse.getIndexToSettings().size(), is(3)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foo"), is(true)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foobar"), is(true)); + assertThat(getSettingsResponse.getIndexToSettings().containsKey("foobarfoo"), is(true)); + + final IndicesShardStoresResponse indicesShardsStoresResponse = client.admin().indices() + .prepareShardStores(randomFrom("*", "_all", "foo*")).setShardStatuses("all").get(); + assertThat(indicesShardsStoresResponse.getStoreStatuses().size(), is(3)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foo"), is(true)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foobar"), is(true)); + assertThat(indicesShardsStoresResponse.getStoreStatuses().containsKey("foobarfoo"), is(true)); + + final UpgradeStatusResponse upgradeStatusResponse = client.admin().indices().prepareUpgradeStatus(randomFrom("*", "_all", "foo*")) + .get(); + assertThat(upgradeStatusResponse.getIndices().size(), is(3)); + assertThat(upgradeStatusResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final IndicesStatsResponse indicesStatsResponse = client.admin().indices().prepareStats(randomFrom("*", "_all", "foo*")).get(); + assertThat(indicesStatsResponse.getIndices().size(), is(3)); + assertThat(indicesStatsResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final IndicesSegmentResponse indicesSegmentResponse = client.admin().indices().prepareSegments("*").get(); + assertThat(indicesSegmentResponse.getIndices().size(), is(3)); + assertThat(indicesSegmentResponse.getIndices().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + final RecoveryResponse indicesRecoveryResponse = client.admin().indices().prepareRecoveries("*").get(); + assertThat(indicesRecoveryResponse.shardRecoveryStates().size(), is(3)); + assertThat(indicesRecoveryResponse.shardRecoveryStates().keySet(), containsInAnyOrder("foo", "foobar", "foobarfoo")); + + // test _cat/indices with wildcards that cover unauthorized indices (".security" in this case) + RequestOptions.Builder optionsBuilder = RequestOptions.DEFAULT.toBuilder(); + optionsBuilder.addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue("user_monitor", USERS_PASSWD)); + RequestOptions options = optionsBuilder.build(); + Request catIndicesRequest = new Request("GET", "/_cat/indices/" + randomFrom("*", "_all", "foo*")); + catIndicesRequest.setOptions(options); + Response catIndicesResponse = getRestClient().performRequest(catIndicesRequest); + assertThat(catIndicesResponse.getStatusLine().getStatusCode() < 300, is(true)); + } + public void testMultipleRoles() throws Exception { IndexResponse indexResponse = index("a", "type", jsonBuilder() .startObject() diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index adeb4a7f86569..d72130db2f571 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -78,6 +78,7 @@ public class SecuritySettingsSource extends NodeConfigurationSource { " cluster: [ ALL ]\n" + " indices:\n" + " - names: '*'\n" + + " allow_restricted_indices: true\n" + " privileges: [ ALL ]\n"; private final Path parentFolder; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index dccc8f3ce587a..171e11614c5f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -73,6 +73,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -151,7 +152,8 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.hamcrest.Matchers.arrayContaining; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -804,7 +806,7 @@ public void testRunAsRequestWithValidPermissions() { verifyNoMoreInteractions(auditTrail); } - public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { + public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndices() { RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"}, new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null); final Authentication authentication = createAuthentication(new User("all_access_user", "all_access")); @@ -812,29 +814,41 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(),true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(BulkAction.NAME + "[s]", - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + requests.add( + new Tuple<>(BulkAction.NAME + "[s]", new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); + .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(UpdateSettingsAction.NAME, + new UpdateSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + // cannot execute monitor operations + requests.add(new Tuple<>(IndicesStatsAction.NAME, + new IndicesStatsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add( - new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SECURITY_INDEX_NAME))); + new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(IndicesSegmentsAction.NAME, + new IndicesSegmentsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(GetSettingsAction.NAME, + new GetSettingsRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(IndicesShardStoresAction.NAME, + new IndicesShardStoresRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); + requests.add(new Tuple<>(UpgradeStatusAction.NAME, + new UpgradeStatusRequest().indices(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); for (Tuple requestTuple : requests) { String action = requestTuple.v1(); @@ -847,12 +861,12 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { } // we should allow waiting for the health of the index or any index if the user has this permission - ClusterHealthRequest request = new ClusterHealthRequest(SECURITY_INDEX_NAME); + ClusterHealthRequest request = new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); // multiple indices - request = new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"); + request = new ClusterHealthRequest(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX, "foo", "bar"); authorize(authentication, ClusterHealthAction.NAME, request); verify(auditTrail).accessGranted(requestId, authentication, ClusterHealthAction.NAME, request, new String[]{role.getName()}); verifyNoMoreInteractions(auditTrail); @@ -863,17 +877,24 @@ public void testNonXPackUserCannotExecuteOperationAgainstSecurityIndex() { assertEquals(IndicesAndAliasesResolver.NO_INDICES_OR_ALIASES_LIST, Arrays.asList(searchRequest.indices())); } - public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurityIndex() { - RoleDescriptor role = new RoleDescriptor("all access", new String[]{"all"}, - new IndicesPrivileges[]{IndicesPrivileges.builder().indices("*").privileges("all").build()}, null); - final Authentication authentication = createAuthentication(new User("all_access_user", "all_access")); - roleMap.put("all_access", role); + public void testMonitoringOperationsAgainstSecurityIndexRequireAllowRestricted() { + final RoleDescriptor restrictedMonitorRole = new RoleDescriptor("restricted_monitor", null, + new IndicesPrivileges[] { IndicesPrivileges.builder().indices("*").privileges("monitor").build() }, null); + final RoleDescriptor unrestrictedMonitorRole = new RoleDescriptor("unrestricted_monitor", null, new IndicesPrivileges[] { + IndicesPrivileges.builder().indices("*").privileges("monitor").allowRestrictedIndices(true).build() }, null); + roleMap.put("restricted_monitor", restrictedMonitorRole); + roleMap.put("unrestricted_monitor", unrestrictedMonitorRole); + final Authentication restrictedUserAuthn = createAuthentication(new User("restricted_user", "restricted_monitor")); + final Authentication unrestrictedUserAuthn = createAuthentication(new User("unrestricted_user", "unrestricted_monitor")); ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); @@ -882,16 +903,18 @@ public void testGrantedNonXPackUserCanExecuteMonitoringOperationsAgainstSecurity requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SECURITY_INDEX_NAME))); requests.add(new Tuple<>(IndicesSegmentsAction.NAME, new IndicesSegmentsRequest().indices(SECURITY_INDEX_NAME))); requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(IndicesShardStoresAction.NAME, - new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); - requests.add(new Tuple<>(UpgradeStatusAction.NAME, - new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(IndicesShardStoresAction.NAME, new IndicesShardStoresRequest().indices(SECURITY_INDEX_NAME))); + requests.add(new Tuple<>(UpgradeStatusAction.NAME, new UpgradeStatusRequest().indices(SECURITY_INDEX_NAME))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); final TransportRequest request = requestTuple.v2(); - authorize(authentication, action, request); - verify(auditTrail).accessGranted(requestId, authentication, action, request, new String[]{role.getName()}); + assertThrowsAuthorizationException(() -> authorize(restrictedUserAuthn, action, request), action, "restricted_user"); + verify(auditTrail).accessDenied(requestId, restrictedUserAuthn, action, request, new String[] { "restricted_monitor" }); + verifyNoMoreInteractions(auditTrail); + authorize(unrestrictedUserAuthn, action, request); + verify(auditTrail).accessGranted(requestId, unrestrictedUserAuthn, action, request, new String[] { "unrestricted_monitor" }); + verifyNoMoreInteractions(auditTrail); } } @@ -901,34 +924,35 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() { ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); List> requests = new ArrayList<>(); - requests.add(new Tuple<>(DeleteAction.NAME, - new DeleteRequest(SECURITY_INDEX_NAME, "type", "id"))); + requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, DeleteRequest::new))); - requests.add(new Tuple<>(UpdateAction.NAME, - new UpdateRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndexAction.NAME, - new IndexRequest(SECURITY_INDEX_NAME, "type", "id"))); + createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), DeleteRequest::new))); + requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "id"))); + requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(BulkAction.NAME + "[s]", - createBulkShardRequest(SECURITY_INDEX_NAME, IndexRequest::new))); - requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SECURITY_INDEX_NAME))); + createBulkShardRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), IndexRequest::new))); + requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SECURITY_INDEX_NAME, "type", "id"))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); requests.add(new Tuple<>(TermVectorsAction.NAME, - new TermVectorsRequest(SECURITY_INDEX_NAME, "type", "id"))); - requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest() - .addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)))); - requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SECURITY_INDEX_NAME))); + new TermVectorsRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "type", "id"))); + requests.add(new Tuple<>(IndicesAliasesAction.NAME, + new IndicesAliasesRequest().addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_INDEX)))); + requests.add( + new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)))); requests.add(new Tuple<>(ClusterHealthAction.NAME, - new ClusterHealthRequest(SECURITY_INDEX_NAME, "foo", "bar"))); + new ClusterHealthRequest(randomFrom(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX), "foo", "bar"))); for (final Tuple requestTuple : requests) { final String action = requestTuple.v1(); @@ -946,9 +970,12 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() ClusterState state = mock(ClusterState.class); when(clusterService.state()).thenReturn(state); when(state.metaData()).thenReturn(MetaData.builder() - .put(new IndexMetaData.Builder(SECURITY_INDEX_NAME) - .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) - .numberOfShards(1).numberOfReplicas(0).build(), true) + .put(new IndexMetaData.Builder(INTERNAL_SECURITY_INDEX) + .putAlias(new AliasMetaData.Builder(SECURITY_INDEX_NAME).build()) + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(), true) .build()); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); @@ -956,7 +983,7 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndexWithWildcard() SearchRequest request = new SearchRequest("_all"); authorize(createAuthentication(superuser), action, request); verify(auditTrail).accessGranted(requestId, authentication, action, request, superuser.roles()); - assertThat(request.indices(), arrayContaining(".security")); + assertThat(request.indices(), arrayContainingInAnyOrder(INTERNAL_SECURITY_INDEX, SECURITY_INDEX_NAME)); } public void testAnonymousRolesAreAppliedToOtherUsers() { From 8e5f9c4b142f6ed0b5b0424c7de5d6c309989164 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Jan 2019 11:18:30 -0800 Subject: [PATCH 10/35] Add OS/architecture classifier to distributions (#37881) This commit adds classifiers to the distributions indicating the OS (for archives) and platform. The current OSes are for windows, darwin (ie macos) and linux. This change will allow future OS/architecture specific changes to the distributions. Note the docs using distribution links have been updated, but will be reworked in a followup to make OS specific instructions for the archives. --- buildSrc/build.gradle | 8 ++- .../gradle/test/ClusterFormationTasks.groovy | 43 ++++++++++++--- .../gradle/vagrant/VagrantTestPlugin.groovy | 8 +-- distribution/archives/build.gradle | 24 +++++++-- .../{oss-tar => darwin-tar}/build.gradle | 0 .../{oss-zip => linux-tar}/build.gradle | 0 .../{tar => oss-darwin-tar}/build.gradle | 0 .../{zip => oss-linux-tar}/build.gradle | 0 .../archives/oss-windows-zip/build.gradle | 2 + .../archives/windows-zip/build.gradle | 2 + distribution/bwc/build.gradle | 33 ++++++++++-- distribution/docker/build.gradle | 4 +- distribution/packages/build.gradle | 4 +- docs/reference/setup/install/deb.asciidoc | 12 ++--- docs/reference/setup/install/rpm.asciidoc | 12 ++--- .../setup/install/zip-targz.asciidoc | 24 ++++----- .../setup/install/zip-windows.asciidoc | 4 +- .../packaging/test/ArchiveTestCase.java | 8 +-- .../packaging/test/DefaultTarTests.java | 2 +- .../test/DefaultWindowsServiceTests.java | 2 +- .../packaging/test/DefaultZipTests.java | 2 +- .../packaging/test/OssTarTests.java | 2 +- .../test/OssWindowsServiceTests.java | 2 +- .../packaging/test/OssZipTests.java | 2 +- .../test/RpmPreservationTestCase.java | 3 -- .../packaging/util/Archives.java | 2 +- .../packaging/util/Distribution.java | 52 ++++++++++++++----- .../packaging/util/Platforms.java | 1 + .../resources/packaging/utils/packages.bash | 10 +++- .../test/resources/packaging/utils/tar.bash | 2 +- settings.gradle | 10 ++-- 31 files changed, 196 insertions(+), 84 deletions(-) rename distribution/archives/{oss-tar => darwin-tar}/build.gradle (100%) rename distribution/archives/{oss-zip => linux-tar}/build.gradle (100%) rename distribution/archives/{tar => oss-darwin-tar}/build.gradle (100%) rename distribution/archives/{zip => oss-linux-tar}/build.gradle (100%) create mode 100644 distribution/archives/oss-windows-zip/build.gradle create mode 100644 distribution/archives/windows-zip/build.gradle diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 0acdc95c86bbf..2b5e4f2d24d1f 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -183,8 +183,12 @@ if (project != rootProject) { } dependencies { - distribution project(':distribution:archives:zip') - distribution project(':distribution:archives:oss-zip') + distribution project(':distribution:archives:windows-zip') + distribution project(':distribution:archives:oss-windows-zip') + distribution project(':distribution:archives:darwin-tar') + distribution project(':distribution:archives:oss-darwin-tar') + distribution project(':distribution:archives:linux-tar') + distribution project(':distribution:archives:oss-linux-tar') } String localDownloads = "${rootProject.buildDir}/local-downloads" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index e38cb854a10b0..2fffd67215581 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -191,13 +191,21 @@ class ClusterFormationTasks { throw new GradleException("Unknown distribution: ${distro} in project ${project.path}") } Version version = Version.fromString(elasticsearchVersion) - String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo + String os = getOs() + String classifier = "${os}-x86_64" + String packaging = os.equals('windows') ? 'zip' : 'tar.gz' String artifactName = 'elasticsearch' if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) { artifactName += '-oss' } - String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip' Object dependency + String snapshotProject = "${os}-${os.equals('windows') ? 'zip' : 'tar'}" + if (version.before("7.0.0")) { + snapshotProject = "zip" + } + if (distro.equals("oss")) { + snapshotProject = "oss-" + snapshotProject + } boolean internalBuild = project.hasProperty('bwcVersions') VersionCollection.UnreleasedVersionInfo unreleasedInfo = null if (project.hasProperty('bwcVersions')) { @@ -205,11 +213,16 @@ class ClusterFormationTasks { unreleasedInfo = project.bwcVersions.unreleasedInfo(version) } if (unreleasedInfo != null) { - dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) + dependency = project.dependencies.project( + path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject) } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) { dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}") } else { - dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip" + if (version.before('7.0.0')) { + classifier = "" // for bwc, before we had classifiers + } + // group does not matter as it is not used when we pull from the ivy repo that points to the download service + dependency = "dnm:${artifactName}:${elasticsearchVersion}${classifier}@${packaging}" } project.dependencies.add(configuration.name, dependency) } @@ -335,8 +348,15 @@ class ClusterFormationTasks { the elasticsearch source tree then this should be the version of elasticsearch built by the source tree. If it isn't then Bad Things(TM) will happen. */ Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) { - from { - project.zipTree(configuration.singleFile) + if (getOs().equals("windows")) { + from { + project.zipTree(configuration.singleFile) + } + } else { + // macos and linux use tar + from { + project.tarTree(project.resources.gzip(configuration.singleFile)) + } } into node.baseDir } @@ -948,4 +968,15 @@ class ClusterFormationTasks { PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin') return extension.name } + + /** Find the current OS */ + static String getOs() { + String os = "linux" + if (Os.FAMILY_WINDOWS) { + os = "windows" + } else if (Os.FAMILY_MAC) { + os = "darwin" + } + return os + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 9f5364b78a896..8f54d63f4ca14 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -53,10 +53,10 @@ class VagrantTestPlugin implements Plugin { /** All distributions to bring into test VM, whether or not they are used **/ static final List DISTRIBUTIONS = unmodifiableList([ - 'archives:tar', - 'archives:oss-tar', - 'archives:zip', - 'archives:oss-zip', + 'archives:linux-tar', + 'archives:oss-linux-tar', + 'archives:windows-zip', + 'archives:oss-windows-zip', 'packages:rpm', 'packages:oss-rpm', 'packages:deb', diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 7ab7e671ac214..3723e31b27f1e 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -105,13 +105,15 @@ task buildIntegTestZip(type: Zip) { with archiveFiles(transportModulesFiles, 'zip', true) } -task buildZip(type: Zip) { +task buildWindowsZip(type: Zip) { configure(commonZipConfig) + archiveClassifier = 'windows-x86_64' with archiveFiles(modulesFiles(false), 'zip', false) } -task buildOssZip(type: Zip) { +task buildOssWindowsZip(type: Zip) { configure(commonZipConfig) + archiveClassifier = 'windows-x86_64' with archiveFiles(modulesFiles(true), 'zip', true) } @@ -122,13 +124,27 @@ Closure commonTarConfig = { fileMode 0644 } -task buildTar(type: Tar) { +task buildDarwinTar(type: Tar) { configure(commonTarConfig) + archiveClassifier = 'darwin-x86_64' with archiveFiles(modulesFiles(false), 'tar', false) } -task buildOssTar(type: Tar) { +task buildOssDarwinTar(type: Tar) { configure(commonTarConfig) + archiveClassifier = 'darwin-x86_64' + with archiveFiles(modulesFiles(true), 'tar', true) +} + +task buildLinuxTar(type: Tar) { + configure(commonTarConfig) + archiveClassifier = 'linux-x86_64' + with archiveFiles(modulesFiles(false), 'tar', false) +} + +task buildOssLinuxTar(type: Tar) { + configure(commonTarConfig) + archiveClassifier = 'linux-x86_64' with archiveFiles(modulesFiles(true), 'tar', true) } diff --git a/distribution/archives/oss-tar/build.gradle b/distribution/archives/darwin-tar/build.gradle similarity index 100% rename from distribution/archives/oss-tar/build.gradle rename to distribution/archives/darwin-tar/build.gradle diff --git a/distribution/archives/oss-zip/build.gradle b/distribution/archives/linux-tar/build.gradle similarity index 100% rename from distribution/archives/oss-zip/build.gradle rename to distribution/archives/linux-tar/build.gradle diff --git a/distribution/archives/tar/build.gradle b/distribution/archives/oss-darwin-tar/build.gradle similarity index 100% rename from distribution/archives/tar/build.gradle rename to distribution/archives/oss-darwin-tar/build.gradle diff --git a/distribution/archives/zip/build.gradle b/distribution/archives/oss-linux-tar/build.gradle similarity index 100% rename from distribution/archives/zip/build.gradle rename to distribution/archives/oss-linux-tar/build.gradle diff --git a/distribution/archives/oss-windows-zip/build.gradle b/distribution/archives/oss-windows-zip/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/oss-windows-zip/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/archives/windows-zip/build.gradle b/distribution/archives/windows-zip/build.gradle new file mode 100644 index 0000000000000..4a6dde5fc0c92 --- /dev/null +++ b/distribution/archives/windows-zip/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// distribution is done in the parent project. diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 4557e0eb1dc4e..0ee597b449a41 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -122,17 +122,34 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre Map artifactFiles = [:] List projectDirs = [] - List projects = ['zip', 'deb', 'rpm'] + List projects = ['deb', 'rpm'] + if (bwcVersion.onOrAfter('7.0.0')) { + projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) + } else { + projects.add('zip') + } + for (String projectName : projects) { String baseDir = "distribution" + String classifier = "" + String extension = projectName + if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { + int index = projectName.indexOf('-') + classifier = "-${projectName.substring(0, index)}-x86_64" + extension = projectName.substring(index + 1) + if (extension.equals('tar')) { + extension += '.gz' + } + } if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += projectName == 'zip' ? '/archives' : '/packages' + baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' // add oss variant first projectDirs.add("${baseDir}/oss-${projectName}") - artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}")) + artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) } projectDirs.add("${baseDir}/${projectName}") - artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}")) + artifactFiles.put(projectName, + file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")) } Closure createRunBwcGradleTask = { name, extraConfig -> @@ -216,9 +233,15 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre String artifactFileName = artifactFile.name String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' String suffix = artifactFile.toString()[-3..-1] + int archIndex = artifactFileName.indexOf('x86_64') + String classifier = '' + if (archIndex != -1) { + int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2) + classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64" + } configurations.create(projectName) artifacts { - it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion]) + it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcVersion]) } } // make sure no dependencies were added to assemble; we want it to be a no-op diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 4d228e3c44fdc..8a5cea936bbcd 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -13,8 +13,8 @@ configurations { } dependencies { - dockerSource project(path: ":distribution:archives:tar") - ossDockerSource project(path: ":distribution:archives:oss-tar") + dockerSource project(path: ":distribution:archives:linux-tar") + ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } ext.expansions = { oss -> diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 0b573ed9bad13..ad0f4a5cdec08 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -101,8 +101,9 @@ Closure commonPackageConfig(String type, boolean oss) { return { dependsOn "process${oss ? 'Oss' : ''}${type.capitalize()}Files" packageName "elasticsearch${oss ? '-oss' : ''}" + arch (type == 'deb' ? 'amd64' : 'X86_64') // Follow elasticsearch's file naming convention - archiveName "${packageName}-${project.version}.${type}" + archiveName "${packageName}-${project.version}-${archString}.${type}" String prefix = "${oss ? 'oss-' : ''}${type}" destinationDir = file("${prefix}/build/distributions") @@ -333,7 +334,6 @@ Closure commonRpmConfig(boolean oss) { packager 'Elasticsearch' version = project.version.replace('-', '_') release = '1' - arch 'NOARCH' os 'LINUX' distribution 'Elasticsearch' vendor 'Elasticsearch' diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index c5046f51ba112..97b4762338936 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -150,17 +150,17 @@ The Debian package for Elasticsearch v{version} can be downloaded from the websi ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha512 -shasum -a 512 -c elasticsearch-{version}.deb.sha512 <1> -sudo dpkg -i elasticsearch-{version}.deb +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb.sha512 +shasum -a 512 -c elasticsearch-{version}-amd64.deb.sha512 <1> +sudo dpkg -i elasticsearch-{version}-amd64.deb -------------------------------------------- <1> Compares the SHA of the downloaded Debian package and the published checksum, which should output - `elasticsearch-{version}.deb: OK`. + `elasticsearch-{version}-amd64.deb: OK`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.deb +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-amd64.deb endif::[] diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index 11add03573db1..a450e202b6896 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -135,17 +135,17 @@ The RPM for Elasticsearch v{version} can be downloaded from the website and inst ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha512 -shasum -a 512 -c elasticsearch-{version}.rpm.sha512 <1> -sudo rpm --install elasticsearch-{version}.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm.sha512 +shasum -a 512 -c elasticsearch-{version}-x86_64.rpm.sha512 <1> +sudo rpm --install elasticsearch-{version}-x86_64.rpm -------------------------------------------- <1> Compares the SHA of the downloaded RPM and the published checksum, which should output - `elasticsearch-{version}.rpm: OK`. + `elasticsearch-{version}-x86_64.rpm: OK`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.rpm +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-x86_64.rpm endif::[] diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc index 735ca5b4ea0d1..d532438103754 100644 --- a/docs/reference/setup/install/zip-targz.asciidoc +++ b/docs/reference/setup/install/zip-targz.asciidoc @@ -32,19 +32,19 @@ The `.zip` archive for Elasticsearch v{version} can be downloaded and installed ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha512 -shasum -a 512 -c elasticsearch-{version}.zip.sha512 <1> -unzip elasticsearch-{version}.zip +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip.sha512 +shasum -a 512 -c elasticsearch-{version}-windows-x86_64.zip.sha512 <1> +unzip elasticsearch-{version}-windows-x86_64.zip cd elasticsearch-{version}/ <2> -------------------------------------------- <1> Compares the SHA of the downloaded `.zip` archive and the published checksum, which should output - `elasticsearch-{version}.zip: OK`. + `elasticsearch-{version}-windows-x86_64.zip: OK`. <2> This directory is known as `$ES_HOME`. Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip endif::[] @@ -64,19 +64,19 @@ The `.tar.gz` archive for Elasticsearch v{version} can be downloaded and install ["source","sh",subs="attributes"] -------------------------------------------- -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz -wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha512 -shasum -a 512 -c elasticsearch-{version}.tar.gz.sha512 <1> -tar -xzf elasticsearch-{version}.tar.gz +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz.sha512 +shasum -a 512 -c elasticsearch-{version}-linux-x86_64.tar.gz.sha512 <1> +tar -xzf elasticsearch-{version}-linux-x86_64.tar.gz cd elasticsearch-{version}/ <2> -------------------------------------------- <1> Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output - `elasticsearch-{version}.tar.gz: OK`. + `elasticsearch-{version}-linux-x86_64.tar.gz: OK`. <2> This directory is known as `$ES_HOME`. Alternatively, you can download the following package, which includes only Apache 2.0 licensed code: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.tar.gz +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-linux-x86_64.tar.gz endif::[] diff --git a/docs/reference/setup/install/zip-windows.asciidoc b/docs/reference/setup/install/zip-windows.asciidoc index 254fb63f6157d..967b449bc972b 100644 --- a/docs/reference/setup/install/zip-windows.asciidoc +++ b/docs/reference/setup/install/zip-windows.asciidoc @@ -31,11 +31,11 @@ endif::[] ifeval::["{release-state}"!="unreleased"] -Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip +Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip Alternatively, you can download the following package, which contains only features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip +https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip Unzip it with your favourite unzip tool. This will create a folder called +elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 05b3603628adb..7f2299d1fc3ea 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -285,7 +285,7 @@ public void test90SecurityCliPackaging() { final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); Platforms.onLinux(() -> { final Result result = sh.run(bin.elasticsearchCertutil + " help"); @@ -296,7 +296,7 @@ public void test90SecurityCliPackaging() { final Result result = sh.run(bin.elasticsearchCertutil + " help"); assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); }); - } else if (distribution().equals(Distribution.OSS_TAR) || distribution().equals(Distribution.OSS_ZIP)) { + } else if (distribution().equals(Distribution.OSS_LINUX) || distribution().equals(Distribution.OSS_WINDOWS)) { assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); } } @@ -312,7 +312,7 @@ public void test100ElasticsearchShardCliPackaging() { assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards")); }; - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { Platforms.onLinux(action); Platforms.onWindows(action); } @@ -330,7 +330,7 @@ public void test110ElasticsearchNodeCliPackaging() { containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes")); }; - if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) { + if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { Platforms.onLinux(action); Platforms.onWindows(action); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java index 9b359a329c1bb..b6c633c0e72a2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultTarTests.java @@ -25,6 +25,6 @@ public class DefaultTarTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_TAR; + return Distribution.DEFAULT_LINUX; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java index 072c3da68868f..6fedd3b89db70 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultWindowsServiceTests.java @@ -25,6 +25,6 @@ public class DefaultWindowsServiceTests extends WindowsServiceTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_ZIP; + return Distribution.DEFAULT_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java index d9a6353a8c6f3..852535188cf9b 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/DefaultZipTests.java @@ -25,6 +25,6 @@ public class DefaultZipTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.DEFAULT_ZIP; + return Distribution.DEFAULT_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java index 86637fc9d48e3..3e72f1da5cbb0 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssTarTests.java @@ -25,6 +25,6 @@ public class OssTarTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_TAR; + return Distribution.OSS_LINUX; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java index f4de95f67b6db..bfa220c6aaf22 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssWindowsServiceTests.java @@ -25,6 +25,6 @@ public class OssWindowsServiceTests extends WindowsServiceTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_ZIP; + return Distribution.OSS_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java index b6cd1e596a09b..418ba6d344650 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/OssZipTests.java @@ -25,6 +25,6 @@ public class OssZipTests extends ArchiveTestCase { @Override protected Distribution distribution() { - return Distribution.OSS_ZIP; + return Distribution.OSS_WINDOWS; } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java index 4787766ae3b61..36558ea2429cc 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/RpmPreservationTestCase.java @@ -43,10 +43,7 @@ import static org.elasticsearch.packaging.util.Platforms.isRPM; import static org.elasticsearch.packaging.util.Platforms.isSystemd; import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index db48ed753b271..ed579c35baf56 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -70,7 +70,7 @@ public static Installation installArchive(Distribution distribution, Path fullIn final Path baseInstallPath = fullInstallPath.getParent(); final Path extractedPath = baseInstallPath.resolve("elasticsearch-" + version); - assertThat("distribution file must exist", Files.exists(distributionFile), is(true)); + assertThat("distribution file must exist: " + distributionFile.toString(), Files.exists(distributionFile), is(true)); assertThat("elasticsearch must not already be installed", lsGlob(baseInstallPath, "elasticsearch*"), empty()); if (distribution.packaging == Distribution.Packaging.TAR) { diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java index 7e1067fdb351c..5ed1bcb5a3d1f 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Distribution.java @@ -19,28 +19,46 @@ package org.elasticsearch.packaging.util; +import java.util.Locale; + public enum Distribution { - OSS_TAR(Packaging.TAR, Flavor.OSS), - OSS_ZIP(Packaging.ZIP, Flavor.OSS), - OSS_DEB(Packaging.DEB, Flavor.OSS), - OSS_RPM(Packaging.RPM, Flavor.OSS), + OSS_LINUX(Packaging.TAR, Platform.LINUX, Flavor.OSS), + OSS_WINDOWS(Packaging.ZIP, Platform.WINDOWS, Flavor.OSS), + OSS_DARWIN(Packaging.TAR, Platform.DARWIN, Flavor.OSS), + OSS_DEB(Packaging.DEB, Platform.LINUX, Flavor.OSS), + OSS_RPM(Packaging.RPM, Platform.LINUX, Flavor.OSS), - DEFAULT_TAR(Packaging.TAR, Flavor.DEFAULT), - DEFAULT_ZIP(Packaging.ZIP, Flavor.DEFAULT), - DEFAULT_DEB(Packaging.DEB, Flavor.DEFAULT), - DEFAULT_RPM(Packaging.RPM, Flavor.DEFAULT); + DEFAULT_LINUX(Packaging.TAR, Platform.LINUX, Flavor.DEFAULT), + DEFAULT_WINDOWS(Packaging.ZIP, Platform.WINDOWS, Flavor.DEFAULT), + DEFAULT_DARWIN(Packaging.TAR, Platform.DARWIN, Flavor.DEFAULT), + DEFAULT_DEB(Packaging.DEB, Platform.LINUX, Flavor.DEFAULT), + DEFAULT_RPM(Packaging.RPM, Platform.LINUX, Flavor.DEFAULT); public final Packaging packaging; + public final Platform platform; public final Flavor flavor; - Distribution(Packaging packaging, Flavor flavor) { + Distribution(Packaging packaging, Platform platform, Flavor flavor) { this.packaging = packaging; + this.platform = platform; this.flavor = flavor; } public String filename(String version) { - return flavor.name + "-" + version + packaging.extension; + String architecture = ""; + if (version.startsWith("6.") == false) { + + if (packaging == Packaging.DEB) { + architecture = "-amd64"; + } else { + if (packaging != Packaging.RPM) { + architecture = "-" + platform.toString(); + } + architecture += "-x86_64"; + } + } + return flavor.name + "-" + version + architecture + packaging.extension; } public boolean isDefault() { @@ -53,8 +71,8 @@ public boolean isOSS() { public enum Packaging { - TAR(".tar.gz", Platforms.LINUX), - ZIP(".zip", true), + TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN), + ZIP(".zip", Platforms.WINDOWS), DEB(".deb", Platforms.isDPKG()), RPM(".rpm", Platforms.isRPM()); @@ -70,6 +88,16 @@ public enum Packaging { } } + public enum Platform { + LINUX, + WINDOWS, + DARWIN; + + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + public enum Flavor { OSS("elasticsearch-oss"), diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index c7ca1284ca69a..dbac9c88d26c9 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -27,6 +27,7 @@ public class Platforms { public static final String OS_NAME = System.getProperty("os.name"); public static final boolean LINUX = OS_NAME.startsWith("Linux"); public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); + public static final boolean DARWIN = OS_NAME.startsWith("Mac OS X"); public static String getOsRelease() { if (LINUX) { diff --git a/qa/vagrant/src/test/resources/packaging/utils/packages.bash b/qa/vagrant/src/test/resources/packaging/utils/packages.bash index d86f1c64e2e69..0285e3682b3f7 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/packages.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/packages.bash @@ -73,10 +73,16 @@ install_package() { ;; esac done + local rpm_classifier="-x86_64" + local deb_classifier="-amd64" + if [[ $version == 6* ]]; then + rpm_classifier="" + deb_classifier="" + fi if is_rpm; then - rpm $rpmCommand $PACKAGE_NAME-$version.rpm + rpm $rpmCommand $PACKAGE_NAME-$version$rpm_classifier.rpm elif is_dpkg; then - run dpkg $dpkgCommand -i $PACKAGE_NAME-$version.deb + run dpkg $dpkgCommand -i $PACKAGE_NAME-$version$deb_classifier.deb [[ "$status" -eq 0 ]] || { echo "dpkg failed:" echo "$output" diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/src/test/resources/packaging/utils/tar.bash index eb2e39274e29e..142215a703682 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/tar.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/tar.bash @@ -40,7 +40,7 @@ install_archive() { echo "Unpacking tarball to $ESHOME" rm -rf /tmp/untar mkdir -p /tmp/untar - tar -xzpf "${PACKAGE_NAME}-${version}.tar.gz" -C /tmp/untar + tar -xzpf "${PACKAGE_NAME}-${version}-linux-x86_64.tar.gz" -C /tmp/untar find /tmp/untar -depth -type d -name 'elasticsearch*' -exec mv {} "$ESHOME" \; > /dev/null diff --git a/settings.gradle b/settings.gradle index d5eebd6f66fe2..18f5f63332e00 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,10 +16,12 @@ List projects = [ 'client:benchmark', 'benchmarks', 'distribution:archives:integ-test-zip', - 'distribution:archives:oss-zip', - 'distribution:archives:zip', - 'distribution:archives:oss-tar', - 'distribution:archives:tar', + 'distribution:archives:oss-windows-zip', + 'distribution:archives:windows-zip', + 'distribution:archives:oss-darwin-tar', + 'distribution:archives:darwin-tar', + 'distribution:archives:oss-linux-tar', + 'distribution:archives:linux-tar', 'distribution:docker', 'distribution:packages:oss-deb', 'distribution:packages:deb', From e9332331a39efdb42be204c7df0f04d4088d0d6e Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 29 Jan 2019 21:20:09 +0200 Subject: [PATCH 11/35] SQL: Make error msg for validation of 2nd arg of PERCENTILE[_RANK] consistent (#37937) Use `first` and `second` instead of `1st` and `2nd`. --- .../xpack/sql/expression/function/aggregate/Percentile.java | 2 +- .../sql/expression/function/aggregate/PercentileRank.java | 2 +- .../sql/analysis/analyzer/VerifierErrorMessagesTests.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java index 295932cd99c5e..76c7bda320012 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Percentile.java @@ -43,7 +43,7 @@ public Percentile replaceChildren(List newChildren) { @Override protected TypeResolution resolveType() { if (!percent.foldable()) { - return new TypeResolution(format(null, "2nd argument of PERCENTILE must be a constant, received [{}]", + return new TypeResolution(format(null, "Second argument of PERCENTILE must be a constant, received [{}]", Expressions.name(percent))); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java index 92bc794b248da..b30b38a01b6c5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/PercentileRank.java @@ -43,7 +43,7 @@ public Expression replaceChildren(List newChildren) { @Override protected TypeResolution resolveType() { if (!value.foldable()) { - return new TypeResolution(format(null, "2nd argument of PERCENTILE_RANK must be a constant, received [{}]", + return new TypeResolution(format(null, "Second argument of PERCENTILE_RANK must be a constant, received [{}]", Expressions.name(value))); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 3316b179f50cb..4279910e0e03b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -538,12 +538,12 @@ public void testAggsInHistogram() { } public void testErrorMessageForPercentileWithSecondArgBasedOnAField() { - assertEquals("1:8: 2nd argument of PERCENTILE must be a constant, received [ABS(int)]", + assertEquals("1:8: Second argument of PERCENTILE must be a constant, received [ABS(int)]", error("SELECT PERCENTILE(int, ABS(int)) FROM test")); } public void testErrorMessageForPercentileRankWithSecondArgBasedOnAField() { - assertEquals("1:8: 2nd argument of PERCENTILE_RANK must be a constant, received [ABS(int)]", + assertEquals("1:8: Second argument of PERCENTILE_RANK must be a constant, received [ABS(int)]", error("SELECT PERCENTILE_RANK(int, ABS(int)) FROM test")); } } From f3f9cabd67681009f5eadee369b1dc1e24d8c67d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 12:29:06 -0700 Subject: [PATCH 12/35] Add timeout for ccr recovery action (#37840) This is related to #35975. It adds a action timeout setting that allows timeouts to be applied to the individual transport actions that are used during a ccr recovery. --- .../test/transport/MockTransportService.java | 10 +++ .../transport/StubbableConnectionManager.java | 2 + .../test/transport/StubbableTransport.java | 8 ++ .../elasticsearch/xpack/ccr/CcrSettings.java | 19 +++++ .../xpack/ccr/repository/CcrRepository.java | 41 ++++++---- .../elasticsearch/xpack/CcrIntegTestCase.java | 5 +- .../xpack/ccr/CcrRepositoryIT.java | 77 +++++++++++++++++++ .../CcrRestoreSourceServiceTests.java | 2 +- 8 files changed, 145 insertions(+), 19 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 1ff1fa37fc120..469a4cfb4d55e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -406,6 +406,16 @@ public boolean addSendBehavior(TransportAddress transportAddress, StubbableTrans return transport().addSendBehavior(transportAddress, sendBehavior); } + /** + * Adds a send behavior that is the default send behavior. + * + * @return {@code true} if no default send behavior was registered + */ + public boolean addSendBehavior(StubbableTransport.SendRequestBehavior behavior) { + return transport().setDefaultSendBehavior(behavior); + } + + /** * Adds a new connect behavior that is used for creating connections with the given delegate service. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java index 994037be07a92..41ac87f0af576 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java @@ -69,7 +69,9 @@ public boolean setDefaultNodeConnectedBehavior(NodeConnectedBehavior behavior) { } public void clearBehaviors() { + defaultGetConnectionBehavior = ConnectionManager::getConnection; getConnectionBehaviors.clear(); + defaultNodeConnectedBehavior = ConnectionManager::nodeConnected; nodeConnectedBehaviors.clear(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java index 4f0df85fd50c8..673ed49387570 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java @@ -55,6 +55,12 @@ public StubbableTransport(Transport transport) { this.delegate = transport; } + boolean setDefaultSendBehavior(SendRequestBehavior sendBehavior) { + SendRequestBehavior prior = defaultSendRequest; + defaultSendRequest = sendBehavior; + return prior == null; + } + public boolean setDefaultConnectBehavior(OpenConnectionBehavior openConnectionBehavior) { OpenConnectionBehavior prior = this.defaultConnectBehavior; this.defaultConnectBehavior = openConnectionBehavior; @@ -70,7 +76,9 @@ boolean addConnectBehavior(TransportAddress transportAddress, OpenConnectionBeha } void clearBehaviors() { + this.defaultSendRequest = null; sendBehaviors.clear(); + this.defaultConnectBehavior = null; connectBehaviors.clear(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java index d6262a7711dad..625429dc0abc6 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java @@ -57,6 +57,13 @@ public final class CcrSettings { Setting.timeSetting("ccr.indices.recovery.recovery_activity_timeout", TimeValue.timeValueSeconds(60), Setting.Property.Dynamic, Setting.Property.NodeScope); + /** + * The timeout value to use for requests made as part of ccr recovery process. + * */ + public static final Setting INDICES_RECOVERY_ACTION_TIMEOUT_SETTING = + Setting.positiveTimeSetting("ccr.indices.recovery.internal_action_timeout", TimeValue.timeValueSeconds(60), + Property.Dynamic, Property.NodeScope); + /** * The settings defined by CCR. * @@ -67,6 +74,7 @@ static List> getSettings() { XPackSettings.CCR_ENABLED_SETTING, CCR_FOLLOWING_INDEX_SETTING, RECOVERY_MAX_BYTES_PER_SECOND, + INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT, CCR_WAIT_FOR_METADATA_TIMEOUT); @@ -74,12 +82,15 @@ static List> getSettings() { private final CombinedRateLimiter ccrRateLimiter; private volatile TimeValue recoveryActivityTimeout; + private volatile TimeValue recoveryActionTimeout; public CcrSettings(Settings settings, ClusterSettings clusterSettings) { this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings); + this.recoveryActionTimeout = INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.get(settings); this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings)); clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout); + clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, this::setRecoveryActionTimeout); } private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) { @@ -90,6 +101,10 @@ private void setRecoveryActivityTimeout(TimeValue recoveryActivityTimeout) { this.recoveryActivityTimeout = recoveryActivityTimeout; } + private void setRecoveryActionTimeout(TimeValue recoveryActionTimeout) { + this.recoveryActionTimeout = recoveryActionTimeout; + } + public CombinedRateLimiter getRateLimiter() { return ccrRateLimiter; } @@ -97,4 +112,8 @@ public CombinedRateLimiter getRateLimiter() { public TimeValue getRecoveryActivityTimeout() { return recoveryActivityTimeout; } + + public TimeValue getRecoveryActionTimeout() { + return recoveryActionTimeout; + } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 7ca95a14909ae..bcf0e5f6dc6e9 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -127,7 +127,8 @@ public RepositoryMetaData getMetadata() { public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) { assert SNAPSHOT_ID.equals(snapshotId) : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId"; Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); - ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).setNodes(true).get(); + ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).setNodes(true) + .get(ccrSettings.getRecoveryActionTimeout()); ImmutableOpenMap indicesMap = response.getState().metaData().indices(); ArrayList indices = new ArrayList<>(indicesMap.size()); indicesMap.keysIt().forEachRemaining(indices::add); @@ -141,7 +142,8 @@ public MetaData getSnapshotGlobalMetaData(SnapshotId snapshotId) { Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); // We set a single dummy index name to avoid fetching all the index data ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest("dummy_index_name"); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); return clusterState.getState().metaData(); } @@ -152,13 +154,14 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); // Validates whether the leader cluster has been configured properly: PlainActionFuture future = PlainActionFuture.newFuture(); IndexMetaData leaderIndexMetaData = clusterState.getState().metaData().index(leaderIndex); ccrLicenseChecker.fetchLeaderHistoryUUIDs(remoteClient, leaderIndexMetaData, future::onFailure, future::onResponse); - String[] leaderHistoryUUIDs = future.actionGet(); + String[] leaderHistoryUUIDs = future.actionGet(ccrSettings.getRecoveryActionTimeout()); IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndex); // Adding the leader index uuid for each shard as custom metadata: @@ -188,7 +191,8 @@ public IndexMetaData getSnapshotIndexMetaData(SnapshotId snapshotId, IndexId ind @Override public RepositoryData getRepositoryData() { Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); - ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true).get(); + ClusterStateResponse response = remoteClient.admin().cluster().prepareState().clear().setMetaData(true) + .get(ccrSettings.getRecoveryActionTimeout()); MetaData remoteMetaData = response.getState().getMetaData(); Map copiedSnapshotIds = new HashMap<>(); @@ -298,7 +302,8 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Ve private void maybeUpdateMappings(Client localClient, Client remoteClient, Index leaderIndex, IndexSettings followerIndexSettings) { ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex.getName()); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest).actionGet(); + ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) + .actionGet(ccrSettings.getRecoveryActionTimeout()); IndexMetaData leaderIndexMetadata = clusterState.getState().metaData().getIndexSafe(leaderIndex); long leaderMappingVersion = leaderIndexMetadata.getMappingVersion(); @@ -306,7 +311,7 @@ private void maybeUpdateMappings(Client localClient, Client remoteClient, Index Index followerIndex = followerIndexSettings.getIndex(); MappingMetaData mappingMetaData = leaderIndexMetadata.mapping(); PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData); - localClient.admin().indices().putMapping(putMappingRequest).actionGet(); + localClient.admin().indices().putMapping(putMappingRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } } @@ -314,9 +319,9 @@ private RestoreSession openSession(String repositoryName, Client remoteClient, S RecoveryState recoveryState) { String sessionUUID = UUIDs.randomBase64UUID(); PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse response = remoteClient.execute(PutCcrRestoreSessionAction.INSTANCE, - new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(); + new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(ccrSettings.getRecoveryActionTimeout()); return new RestoreSession(repositoryName, remoteClient, sessionUUID, response.getNode(), indexShard, recoveryState, - response.getStoreFileMetaData(), ccrSettings.getRateLimiter(), throttledTime::inc); + response.getStoreFileMetaData(), ccrSettings, throttledTime::inc); } private static class RestoreSession extends FileRestoreContext implements Closeable { @@ -327,18 +332,18 @@ private static class RestoreSession extends FileRestoreContext implements Closea private final String sessionUUID; private final DiscoveryNode node; private final Store.MetadataSnapshot sourceMetaData; - private final CombinedRateLimiter rateLimiter; + private final CcrSettings ccrSettings; private final LongConsumer throttleListener; RestoreSession(String repositoryName, Client remoteClient, String sessionUUID, DiscoveryNode node, IndexShard indexShard, - RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, CombinedRateLimiter rateLimiter, + RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, CcrSettings ccrSettings, LongConsumer throttleListener) { super(repositoryName, indexShard, SNAPSHOT_ID, recoveryState, BUFFER_SIZE); this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.sourceMetaData = sourceMetaData; - this.rateLimiter = rateLimiter; + this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; } @@ -354,14 +359,14 @@ void restoreFiles() throws IOException { @Override protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) { - return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), rateLimiter, throttleListener); + return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), ccrSettings, throttleListener); } @Override public void close() { ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = - remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(); + remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } } @@ -372,17 +377,19 @@ private static class RestoreFileInputStream extends InputStream { private final DiscoveryNode node; private final StoreFileMetaData fileToRecover; private final CombinedRateLimiter rateLimiter; + private final CcrSettings ccrSettings; private final LongConsumer throttleListener; private long pos = 0; private RestoreFileInputStream(Client remoteClient, String sessionUUID, DiscoveryNode node, StoreFileMetaData fileToRecover, - CombinedRateLimiter rateLimiter, LongConsumer throttleListener) { + CcrSettings ccrSettings, LongConsumer throttleListener) { this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.fileToRecover = fileToRecover; - this.rateLimiter = rateLimiter; + this.ccrSettings = ccrSettings; + this.rateLimiter = ccrSettings.getRateLimiter(); this.throttleListener = throttleListener; } @@ -407,7 +414,7 @@ public int read(byte[] bytes, int off, int len) throws IOException { String fileName = fileToRecover.name(); GetCcrRestoreFileChunkRequest request = new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileName, bytesRequested); GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse response = - remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(); + remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(ccrSettings.getRecoveryActionTimeout()); BytesReference fileChunk = response.getChunk(); int bytesReceived = fileChunk.length(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 4a6c5411737c6..c4fdeb116ae86 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -65,7 +65,9 @@ import org.elasticsearch.test.NodeConfigurationSource; import org.elasticsearch.test.TestCluster; import org.elasticsearch.test.discovery.TestZenDiscovery; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.nio.MockNioTransportPlugin; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.ccr.LocalStateCcr; import org.elasticsearch.xpack.ccr.index.engine.FollowingEngine; @@ -120,7 +122,8 @@ public final void startClusters() throws Exception { stopClusters(); Collection> mockPlugins = Arrays.asList(ESIntegTestCase.TestSeedPlugin.class, - TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class, getTestTransportPlugin()); + TestZenDiscovery.TestPlugin.class, MockHttpTransport.TestPlugin.class, MockTransportService.TestPlugin.class, + MockNioTransportPlugin.class); InternalTestCluster leaderCluster = new InternalTestCluster(randomLong(), createTempDir(), true, true, numberOfNodesPerCluster(), numberOfNodesPerCluster(), UUIDs.randomBase64UUID(random()), createNodeConfigurationSource(null), 0, "leader", mockPlugins, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index ee02241978cfe..f22857939e0d1 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ccr; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; @@ -34,8 +36,10 @@ import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.CcrIntegTestCase; +import org.elasticsearch.xpack.ccr.action.repositories.GetCcrRestoreFileChunkAction; import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -51,6 +55,7 @@ import static org.elasticsearch.snapshots.RestoreService.restoreInProgress; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; // TODO: Fold this integration test into a more expansive integration test as more bootstrap from remote work // TODO: is completed. @@ -308,6 +313,78 @@ public void testRateLimitingIsEmployed() throws Exception { } } + public void testIndividualActionsTimeout() throws Exception { + ClusterUpdateSettingsRequest settingsRequest = new ClusterUpdateSettingsRequest(); + TimeValue timeValue = TimeValue.timeValueMillis(100); + settingsRequest.persistentSettings(Settings.builder().put(CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getKey(), timeValue)); + assertAcked(followerClient().admin().cluster().updateSettings(settingsRequest).actionGet()); + + String leaderClusterRepoName = CcrRepository.NAME_PREFIX + "leader_cluster"; + String leaderIndex = "index1"; + String followerIndex = "index2"; + + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex).setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderGreen(leaderIndex); + + List transportServices = new ArrayList<>(); + + for (TransportService transportService : getFollowerCluster().getDataOrMasterNodeInstances(TransportService.class)) { + MockTransportService mockTransportService = (MockTransportService) transportService; + transportServices.add(mockTransportService); + mockTransportService.addSendBehavior((connection, requestId, action, request, options) -> { + if (action.equals(GetCcrRestoreFileChunkAction.NAME) == false) { + connection.sendRequest(requestId, action, request, options); + } + }); + } + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get(); + + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, followerIndex) + .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) + .indices(leaderIndex).indicesOptions(indicesOptions).renamePattern("^(.*)$") + .renameReplacement(followerIndex).masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) + .indexSettings(settingsBuilder); + + final RestoreService restoreService = getFollowerCluster().getCurrentMasterNodeInstance(RestoreService.class); + final ClusterService clusterService = getFollowerCluster().getCurrentMasterNodeInstance(ClusterService.class); + PlainActionFuture future = PlainActionFuture.newFuture(); + restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); + + // Depending on when the timeout occurs this can fail in two ways. If it times-out when fetching + // metadata this will throw an exception. If it times-out when restoring a shard, the shard will + // be marked as failed. Either one is a success for the purpose of this test. + try { + RestoreInfo restoreInfo = future.actionGet(); + assertEquals(0, restoreInfo.successfulShards()); + assertEquals(numberOfPrimaryShards, restoreInfo.failedShards()); + } catch (Exception e) { + assertThat(ExceptionsHelper.unwrapCause(e), instanceOf(ElasticsearchTimeoutException.class)); + } + + + for (MockTransportService transportService : transportServices) { + transportService.clearAllRules(); + } + + settingsRequest = new ClusterUpdateSettingsRequest(); + TimeValue defaultValue = CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getDefault(Settings.EMPTY); + settingsRequest.persistentSettings(Settings.builder().put(CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.getKey(), + defaultValue)); + assertAcked(followerClient().admin().cluster().updateSettings(settingsRequest).actionGet()); + } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37887") public void testFollowerMappingIsUpdated() throws IOException { String leaderClusterRepoName = CcrRepository.NAME_PREFIX + "leader_cluster"; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java index 5f352788d9597..3035b96b5bcac 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java @@ -40,7 +40,7 @@ public void setUp() throws Exception { Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); taskQueue = new DeterministicTaskQueue(settings, random()); Set> registeredSettings = Sets.newHashSet(CcrSettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, - CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND); + CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND, CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING); ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, registeredSettings); restoreSourceService = new CcrRestoreSourceService(taskQueue.getThreadPool(), new CcrSettings(Settings.EMPTY, clusterSettings)); } From 193017672a3a283a2e5bb85f1002f656a4e5e156 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 29 Jan 2019 20:31:24 +0100 Subject: [PATCH 13/35] Handle completion suggestion without contexts This change fixes the handling of completion suggestion without contexts. Relates #36996 --- .../suggest/completion/TopSuggestGroupDocsCollector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java index 3dfb38bef9dd4..30c642dcdeb3b 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/TopSuggestGroupDocsCollector.java @@ -60,7 +60,9 @@ public void collect(int docID, CharSequence key, CharSequence context, float sco int globalDoc = docID + docBase; boolean isNewDoc = docContexts.containsKey(globalDoc) == false; List contexts = docContexts.computeIfAbsent(globalDoc, k -> new ArrayList<>()); - contexts.add(context); + if (context != null) { + contexts.add(context); + } if (isNewDoc) { super.collect(docID, key, context, score); } From b889221f75048bebbc7084bf3b3f341eac30e34c Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 20:52:41 +0000 Subject: [PATCH 14/35] Types removal - deprecate include_type_name with index templates (#37484) Added deprecation warnings for use of include_type_name in put/get index templates. HLRC changes: GetIndexTemplateRequest has a new client-side class which is a copy of server's GetIndexTemplateResponse but modified to be typeless. PutIndexTemplateRequest has a new client-side counterpart which doesn't use types in the mappings Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 109 ++++- .../client/IndicesRequestConverters.java | 41 +- .../indices/GetIndexTemplatesResponse.java | 90 ++++ .../client/indices/IndexTemplateMetaData.java | 300 ++++++++++++ .../indices/PutIndexTemplateRequest.java | 453 ++++++++++++++++++ .../elasticsearch/client/IndicesClientIT.java | 243 +++++++++- .../client/IndicesRequestConvertersTests.java | 62 ++- .../client/RestHighLevelClientTests.java | 13 +- .../IndicesClientDocumentationIT.java | 74 ++- .../GetIndexTemplatesResponseTests.java | 136 ++++++ .../indices/PutIndexTemplateRequestTests.java | 116 +++++ .../high-level/indices/put_template.asciidoc | 10 +- .../upgrades/FullClusterRestartIT.java | 4 + .../get/GetIndexTemplatesResponse.java | 2 +- .../indices/RestGetIndexTemplateAction.java | 9 + .../indices/RestPutIndexTemplateAction.java | 6 + .../RestPutIndexTemplateActionTests.java | 63 ++- 17 files changed, 1623 insertions(+), 108 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index c93c9c67e8f7f..2f5bd65fba189 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -49,8 +49,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -61,7 +59,9 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.rest.RestStatus; @@ -908,6 +908,7 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques AcknowledgedResponse::fromXContent, listener, emptySet()); } + /** * Puts an index template using the Index Templates API. * See Index Templates API @@ -916,9 +917,13 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} + * instead which introduces a new request object without types. */ - public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options) throws IOException { + @Deprecated + public AcknowledgedResponse putTemplate( + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, emptySet()); } @@ -930,9 +935,44 @@ public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplate * @param putIndexTemplateRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * @deprecated This old form of request allows types in mappings. + * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} + * instead which introduces a new request object without types. */ - public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, - ActionListener listener) { + @Deprecated + public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, + AcknowledgedResponse::fromXContent, listener, emptySet()); + } + + + /** + * Puts an index template using the Index Templates API. + * See Index Templates API + * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public AcknowledgedResponse putTemplate( + PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, + AcknowledgedResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously puts an index template using the Index Templates API. + * See Index Templates API + * on elastic.co + * @param putIndexTemplateRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } @@ -968,33 +1008,74 @@ public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, Reques } /** - * Gets index templates using the Index Templates API + * Gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, where the + * mapping definition is nested under the type name. * See Index Templates API * on elastic.co * @param getIndexTemplatesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * {@link #getIndexTemplate(GetIndexTemplatesRequest, RequestOptions)} instead which returns a new response object */ - public GetIndexTemplatesResponse getTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, - RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, - options, GetIndexTemplatesResponse::fromXContent, emptySet()); + @Deprecated + public org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate( + GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplatesWithDocumentTypes, + options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, emptySet()); } + + /** + * Gets index templates using the Index Templates API + * See Index Templates API + * on elastic.co + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param getIndexTemplatesRequest the request + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetIndexTemplatesResponse getIndexTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplates, + options, GetIndexTemplatesResponse::fromXContent, emptySet()); + } /** - * Asynchronously gets index templates using the Index Templates API + * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, + * where the mapping definition is nested under the type name. * See Index Templates API * on elastic.co * @param getIndexTemplatesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * {@link #getIndexTemplateAsync(GetIndexTemplatesRequest, RequestOptions, ActionListener)} instead which returns a new response object */ + @Deprecated public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplatesWithDocumentTypes, + options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, listener, emptySet()); + } + + /** + * Asynchronously gets index templates using the Index Templates API + * See Index Templates API + * on elastic.co + * @param getIndexTemplatesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, + IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); - } + } /** * Uses the Index Templates API to determine if index templates exist diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 4889ead93b715..13bc2b8c149db 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -43,13 +43,13 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.FreezeIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.common.Strings; @@ -416,7 +416,13 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr return request; } - static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { + /** + * @deprecated This uses the old form of PutIndexTemplateRequest which uses types. + * Use (@link {@link #putTemplate(PutIndexTemplateRequest)} instead + */ + @Deprecated + static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest) + throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") .addPathPart(putIndexTemplateRequest.name()).build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); @@ -433,6 +439,22 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro return request; } + static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") + .addPathPart(putIndexTemplateRequest.name()).build(); + Request request = new Request(HttpPut.METHOD_NAME, endpoint); + RequestConverters.Params params = new RequestConverters.Params(request); + params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); + if (putIndexTemplateRequest.create()) { + params.putParam("create", Boolean.TRUE.toString()); + } + if (Strings.hasText(putIndexTemplateRequest.cause())) { + params.putParam("cause", putIndexTemplateRequest.cause()); + } + request.setEntity(RequestConverters.createEntity(putIndexTemplateRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws IOException { String[] indices = validateQueryRequest.indices() == null ? Strings.EMPTY_ARRAY : validateQueryRequest.indices(); String[] types = validateQueryRequest.types() == null || indices.length <= 0 ? Strings.EMPTY_ARRAY : validateQueryRequest.types(); @@ -458,7 +480,16 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) { return request; } + @Deprecated + static Request getTemplatesWithDocumentTypes(GetIndexTemplatesRequest getIndexTemplatesRequest) { + return getTemplates(getIndexTemplatesRequest, true); + } + static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { + return getTemplates(getIndexTemplatesRequest, false); + } + + private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest, boolean includeTypeName) { final String endpoint = new RequestConverters.EndpointBuilder() .addPathPartAsIs("_template") .addCommaSeparatedPathParts(getIndexTemplatesRequest.names()) @@ -467,9 +498,11 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { final RequestConverters.Params params = new RequestConverters.Params(request); params.withLocal(getIndexTemplatesRequest.isLocal()); params.withMasterTimeout(getIndexTemplatesRequest.getMasterNodeTimeout()); - params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + if (includeTypeName) { + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + } return request; - } + } static Request templatesExist(IndexTemplatesExistRequest indexTemplatesExistRequest) { final String endpoint = new RequestConverters.EndpointBuilder() diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java new file mode 100644 index 0000000000000..ef283b31c0634 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + + +public class GetIndexTemplatesResponse { + + @Override + public String toString() { + List thisList = new ArrayList<>(this.indexTemplates); + thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return "GetIndexTemplatesResponse [indexTemplates=" + thisList + "]"; + } + + private final List indexTemplates; + + GetIndexTemplatesResponse() { + indexTemplates = new ArrayList<>(); + } + + GetIndexTemplatesResponse(List indexTemplates) { + this.indexTemplates = indexTemplates; + } + + public List getIndexTemplates() { + return indexTemplates; + } + + + public static GetIndexTemplatesResponse fromXContent(XContentParser parser) throws IOException { + final List templates = new ArrayList<>(); + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + final IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()); + templates.add(templateMetaData); + } + } + return new GetIndexTemplatesResponse(templates); + } + + @Override + public int hashCode() { + List sortedList = new ArrayList<>(this.indexTemplates); + sortedList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return Objects.hash(sortedList); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + // To compare results we need to make sure the templates are listed in the same order + GetIndexTemplatesResponse other = (GetIndexTemplatesResponse) obj; + List thisList = new ArrayList<>(this.indexTemplates); + List otherList = new ArrayList<>(other.indexTemplates); + thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + otherList.sort(Comparator.comparing(IndexTemplateMetaData::name)); + return Objects.equals(thisList, otherList); + } + + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java new file mode 100644 index 0000000000000..12fc747ab3473 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java @@ -0,0 +1,300 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class IndexTemplateMetaData { + + + private final String name; + + private final int order; + + /** + * The version is an arbitrary number managed by the user so that they can easily and quickly verify the existence of a given template. + * Expected usage: + * 

+     * PUT /_template/my_template
+     * {
+     *   "index_patterns": ["my_index-*"],
+     *   "mappings": { ... },
+     *   "version": 1
+     * }
+     *
+ * Then, some process from the user can occasionally verify that the template exists with the appropriate version without having to + * check the template's content: + * 

+     * GET /_template/my_template?filter_path=*.version
+     *
+ */ + @Nullable + private final Integer version; + + private final List patterns; + + private final Settings settings; + + private final MappingMetaData mappings; + + private final ImmutableOpenMap aliases; + + public IndexTemplateMetaData(String name, int order, Integer version, + List patterns, Settings settings, + MappingMetaData mappings, + ImmutableOpenMap aliases) { + if (patterns == null || patterns.isEmpty()) { + throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns); + } + this.name = name; + this.order = order; + this.version = version; + this.patterns= patterns; + this.settings = settings; + this.mappings = mappings; + this.aliases = aliases; + } + + public String name() { + return this.name; + } + + public int order() { + return this.order; + } + + @Nullable + public Integer version() { + return version; + } + + public List patterns() { + return this.patterns; + } + + public Settings settings() { + return this.settings; + } + + public MappingMetaData mappings() { + return this.mappings; + } + + public ImmutableOpenMap aliases() { + return this.aliases; + } + + public static Builder builder(String name) { + return new Builder(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IndexTemplateMetaData that = (IndexTemplateMetaData) o; + + if (order != that.order) return false; + if (!Objects.equals(mappings, that.mappings)) return false; + if (!name.equals(that.name)) return false; + if (!settings.equals(that.settings)) return false; + if (!patterns.equals(that.patterns)) return false; + + return Objects.equals(version, that.version); + } + + @Override + public int hashCode() { + return Objects.hash(name, order, version, patterns, settings, mappings); + } + + public static class Builder { + + private static final Set VALID_FIELDS = Sets.newHashSet( + "template", "order", "mappings", "settings", "index_patterns", "aliases", "version"); + + private String name; + + private int order; + + private Integer version; + + private List indexPatterns; + + private Settings settings = Settings.Builder.EMPTY_SETTINGS; + + private MappingMetaData mappings; + + private final ImmutableOpenMap.Builder aliases; + + public Builder(String name) { + this.name = name; + mappings = null; + aliases = ImmutableOpenMap.builder(); + } + + public Builder(IndexTemplateMetaData indexTemplateMetaData) { + this.name = indexTemplateMetaData.name(); + order(indexTemplateMetaData.order()); + version(indexTemplateMetaData.version()); + patterns(indexTemplateMetaData.patterns()); + settings(indexTemplateMetaData.settings()); + + mappings = indexTemplateMetaData.mappings(); + aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases()); + } + + public Builder order(int order) { + this.order = order; + return this; + } + + public Builder version(Integer version) { + this.version = version; + return this; + } + + public Builder patterns(List indexPatterns) { + this.indexPatterns = indexPatterns; + return this; + } + + + public Builder settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + public Builder settings(Settings settings) { + this.settings = settings; + return this; + } + + public Builder mapping(MappingMetaData mappings) { + this.mappings = mappings; + return this; + } + + public Builder putAlias(AliasMetaData aliasMetaData) { + aliases.put(aliasMetaData.alias(), aliasMetaData); + return this; + } + + public Builder putAlias(AliasMetaData.Builder aliasMetaData) { + aliases.put(aliasMetaData.alias(), aliasMetaData.build()); + return this; + } + + public IndexTemplateMetaData build() { + return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings, aliases.build()); + } + + + public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException { + Builder builder = new Builder(templateName); + + String currentFieldName = skipTemplateName(parser); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("settings".equals(currentFieldName)) { + Settings.Builder templateSettingsBuilder = Settings.builder(); + templateSettingsBuilder.put(Settings.fromXContent(parser)); + templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX); + builder.settings(templateSettingsBuilder.build()); + } else if ("mappings".equals(currentFieldName)) { + Map mapping = parser.map(); + if (mapping.isEmpty() == false) { + MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping); + builder.mapping(md); + } + } else if ("aliases".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); + } + } else { + throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("mappings".equals(currentFieldName)) { + // The server-side IndexTemplateMetaData has toXContent impl that can return mappings + // in an array but also a comment saying this never happens with typeless APIs. + throw new ElasticsearchParseException("Invalid response format - " + + "mappings are not expected to be returned in an array", currentFieldName); + } else if ("index_patterns".equals(currentFieldName)) { + List index_patterns = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + index_patterns.add(parser.text()); + } + builder.patterns(index_patterns); + } + } else if (token.isValue()) { + // Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009) + if("template".equals(currentFieldName)) { + builder.patterns(Collections.singletonList(parser.text())); + } else if ("order".equals(currentFieldName)) { + builder.order(parser.intValue()); + } else if ("version".equals(currentFieldName)) { + builder.version(parser.intValue()); + } + } + } + return builder.build(); + } + + private static String skipTemplateName(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + token = parser.nextToken(); + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if (VALID_FIELDS.contains(currentFieldName)) { + return currentFieldName; + } else { + // we just hit the template name, which should be ignored and we move on + parser.nextToken(); + } + } + } + + return null; + } + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java new file mode 100644 index 0000000000000..5f22691b046eb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java @@ -0,0 +1,453 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.XContentMapValues; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; + +/** + * A request to create an index template. + */ +public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContent { + + private String name; + + private String cause = ""; + + private List indexPatterns; + + private int order; + + private boolean create; + + private Settings settings = EMPTY_SETTINGS; + + private BytesReference mappings = null; + + private final Set aliases = new HashSet<>(); + + private Integer version; + + /** + * Constructs a new put index template request with the provided name. + */ + public PutIndexTemplateRequest(String name) { + this.name(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (indexPatterns == null || indexPatterns.size() == 0) { + validationException = addValidationError("index patterns are missing", validationException); + } + return validationException; + } + + /** + * Sets the name of the index template. + */ + public PutIndexTemplateRequest name(String name) { + if(name == null) { + throw new IllegalArgumentException("Name cannot be null"); + } + this.name = name; + return this; + } + + /** + * The name of the index template. + */ + public String name() { + return this.name; + } + + public PutIndexTemplateRequest patterns(List indexPatterns) { + this.indexPatterns = indexPatterns; + return this; + } + + public List patterns() { + return this.indexPatterns; + } + + public PutIndexTemplateRequest order(int order) { + this.order = order; + return this; + } + + public int order() { + return this.order; + } + + public PutIndexTemplateRequest version(Integer version) { + this.version = version; + return this; + } + + public Integer version() { + return this.version; + } + + /** + * Set to {@code true} to force only creation, not an update of an index template. If it already + * exists, it will fail with an {@link IllegalArgumentException}. + */ + public PutIndexTemplateRequest create(boolean create) { + this.create = create; + return this; + } + + public boolean create() { + return create; + } + + /** + * The settings to create the index template with. + */ + public PutIndexTemplateRequest settings(Settings settings) { + this.settings = settings; + return this; + } + + /** + * The settings to create the index template with. + */ + public PutIndexTemplateRequest settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + /** + * The settings to create the index template with (either json/yaml format). + */ + public PutIndexTemplateRequest settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * The settings to create the index template with (either json or yaml format). + */ + public PutIndexTemplateRequest settings(Map source) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.map(source); + settings(Strings.toString(builder), XContentType.JSON); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + return this; + } + + public Settings settings() { + return this.settings; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType The type of content contained within the source + */ + public PutIndexTemplateRequest mapping(String source, XContentType xContentType) { + internalMapping(XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2()); + return this; + } + + /** + * The cause for this index template creation. + */ + public PutIndexTemplateRequest cause(String cause) { + this.cause = cause; + return this; + } + + public String cause() { + return this.cause; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + */ + public PutIndexTemplateRequest mapping(XContentBuilder source) { + internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), + true, source.contentType()).v2()); + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + * @param xContentType the source content type + */ + public PutIndexTemplateRequest mapping(BytesReference source, XContentType xContentType) { + internalMapping(XContentHelper.convertToMap(source, true, xContentType).v2()); + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param source The mapping source + */ + public PutIndexTemplateRequest mapping(Map source) { + return internalMapping(source); + } + + private PutIndexTemplateRequest internalMapping(Map source) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.map(source); + Objects.requireNonNull(builder.contentType()); + try { + mappings = new BytesArray( + XContentHelper.convertToJson(BytesReference.bytes(builder), false, false, builder.contentType())); + return this; + } catch (IOException e) { + throw new UncheckedIOException("failed to convert source to json", e); + } + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + public BytesReference mappings() { + return this.mappings; + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { + try { + return source(BytesReference.bytes(templateBuilder), templateBuilder.contentType()); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to build json for template request", e); + } + } + + /** + * The template source definition. + */ + @SuppressWarnings("unchecked") + public PutIndexTemplateRequest source(Map templateSource) { + Map source = templateSource; + for (Map.Entry entry : source.entrySet()) { + String name = entry.getKey(); + if (name.equals("template")) { + if(entry.getValue() instanceof String) { + patterns(Collections.singletonList((String) entry.getValue())); + } + } else if (name.equals("index_patterns")) { + if(entry.getValue() instanceof String) { + patterns(Collections.singletonList((String) entry.getValue())); + } else if (entry.getValue() instanceof List) { + List elements = ((List) entry.getValue()).stream().map(Object::toString).collect(Collectors.toList()); + patterns(elements); + } else { + throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings"); + } + } else if (name.equals("order")) { + order(XContentMapValues.nodeIntegerValue(entry.getValue(), order())); + } else if ("version".equals(name)) { + if ((entry.getValue() instanceof Integer) == false) { + throw new IllegalArgumentException("Malformed [version] value, should be an integer"); + } + version((Integer)entry.getValue()); + } else if (name.equals("settings")) { + if ((entry.getValue() instanceof Map) == false) { + throw new IllegalArgumentException("Malformed [settings] section, should include an inner object"); + } + settings((Map) entry.getValue()); + } else if (name.equals("mappings")) { + Map mappings = (Map) entry.getValue(); + mapping(mappings); + } else if (name.equals("aliases")) { + aliases((Map) entry.getValue()); + } else { + throw new ElasticsearchParseException("unknown key [{}] in the template ", name); + } + } + return this; + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { + return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * The template source definition. + */ + public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { + return source(XContentHelper.convertToMap(source, true, xContentType).v2()); + } + + + public Set aliases() { + return this.aliases; + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(Map source) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.map(source); + return aliases(BytesReference.bytes(builder)); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(XContentBuilder source) { + return aliases(BytesReference.bytes(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(String source) { + return aliases(new BytesArray(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public PutIndexTemplateRequest aliases(BytesReference source) { + // EMPTY is safe here because we never call namedObject + try (XContentParser parser = XContentHelper + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source)) { + //move to the first alias + parser.nextToken(); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + alias(Alias.fromXContent(parser)); + } + return this; + } catch(IOException e) { + throw new ElasticsearchParseException("Failed to parse aliases", e); + } + } + + /** + * Adds an alias that will be added when the index gets created. + * + * @param alias The metadata for the new alias + * @return the index template creation request + */ + public PutIndexTemplateRequest alias(Alias alias) { + aliases.add(alias); + return this; + } + + @Override + public String[] indices() { + return indexPatterns.toArray(new String[indexPatterns.size()]); + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictExpand(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("index_patterns", indexPatterns); + builder.field("order", order); + if (version != null) { + builder.field("version", version); + } + + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + + if (mappings != null) { + builder.field("mappings"); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappings.utf8ToString())) { + builder.copyCurrentStructure(parser); + } + } + + builder.startObject("aliases"); + for (Alias alias : aliases) { + alias.toXContent(builder, params); + } + builder.endObject(); + + return builder; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 708cb6687ce9e..306929d78a67a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -54,8 +54,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.index.IndexRequest; @@ -72,12 +70,14 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; @@ -86,6 +86,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; @@ -97,6 +98,8 @@ import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; +import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import java.io.IOException; import java.util.Arrays; @@ -1400,8 +1403,9 @@ public void testIndexPutSettingNonExistent() throws IOException { } @SuppressWarnings("unchecked") - public void testPutTemplate() throws Exception { - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest() + public void testPutTemplateWithTypes() throws Exception { + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() .name("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) .order(10) @@ -1411,7 +1415,9 @@ public void testPutTemplate() throws Exception { .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) + ); assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); Map templates = getAsMap("/_template/my-template"); @@ -1425,6 +1431,94 @@ public void testPutTemplate() throws Exception { assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); } + + @SuppressWarnings("unchecked") + public void testPutTemplate() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("{ \"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); + + Map templates = getAsMap("/_template/my-template"); + assertThat(templates.keySet(), hasSize(1)); + assertThat(extractValue("my-template.order", templates), equalTo(10)); + assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); + assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); + assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); + assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); + assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); + assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); + } + + public void testPutTemplateWithTypesUsingUntypedAPI() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("{ " + + "\"my_doc_type\":{" + + "\"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}" + + "}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + + ElasticsearchStatusException badMappingError = expectThrows(ElasticsearchStatusException.class, + () -> execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync)); + assertThat(badMappingError.getDetailedMessage(), + containsString("Root mapping definition has unsupported parameters: [my_doc_type")); + } + + @SuppressWarnings("unchecked") + public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() + .name("my-template") + .patterns(Arrays.asList("pattern-1", "name-*")) + .order(10) + .create(randomBoolean()) + .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) + .mapping("my_doc_type", + // Note that the declared type is missing from the mapping + "{ " + + "\"properties\":{" + + "\"host_name\": {\"type\":\"keyword\"}," + + "\"description\": {\"type\":\"text\"}" + + "}" + + "}", XContentType.JSON) + .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); + + AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) + ); + assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); + + Map templates = getAsMap("/_template/my-template"); + assertThat(templates.keySet(), hasSize(1)); + assertThat(extractValue("my-template.order", templates), equalTo(10)); + assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); + assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); + assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); + assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); + assertThat(extractValue("my-template.mappings.properties.description.type", templates), equalTo("text")); + assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); + assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); + } public void testPutTemplateBadRequests() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -1489,68 +1583,168 @@ public void testInvalidValidateQuery() throws IOException{ assertFalse(response.isValid()); } + // Tests the deprecated form of the API that returns templates with doc types (using the server-side's GetIndexTemplateResponse) + public void testCRUDIndexTemplateWithTypes() throws Exception { + RestHighLevelClient client = highLevelClient(); + + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate1 = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-1") + .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); + assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync + , expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) + .isAcknowledged(), equalTo(true)); + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate2 = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-2") + .patterns(Arrays.asList("pattern-2", "name-2")) + .mapping("custom_doc_type", "name", "type=text") + .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, + expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) + .isAcknowledged(), equalTo(true)); + + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate1 = execute( + new GetIndexTemplatesRequest("template-1"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); + org.elasticsearch.cluster.metadata.IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); + assertThat(template1.name(), equalTo("template-1")); + assertThat(template1.patterns(), contains("pattern-1", "name-1")); + assertTrue(template1.aliases().containsKey("alias-1")); + + //Check the typed version of the call + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate2 = + execute(new GetIndexTemplatesRequest("template-2"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); + org.elasticsearch.cluster.metadata.IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); + assertThat(template2.name(), equalTo("template-2")); + assertThat(template2.patterns(), contains("pattern-2", "name-2")); + assertTrue(template2.aliases().isEmpty()); + assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + // Ugly deprecated form of API requires use of doc type to get at mapping object which is CompressedXContent + assertTrue(template2.mappings().containsKey("custom_doc_type")); + + List names = randomBoolean() + ? Arrays.asList("*-1", "template-2") + : Arrays.asList("template-*"); + GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getBoth = execute( + getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getBoth.getIndexTemplates(), hasSize(2)); + assertThat(getBoth.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName).toArray(), + arrayContainingInAnyOrder("template-1", "template-2")); + + GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getAll = execute( + getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); + assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); + assertThat(getAll.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName) + .collect(Collectors.toList()), + hasItems("template-1", "template-2")); + + assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), + client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + + assertThat(execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates(), hasSize(1)); + assertThat(execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates() + .get(0).name(), equalTo("template-2")); + + assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), + client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); + assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), + client.indices()::getTemplate, client.indices()::getTemplateAsync, + expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE))).status(), equalTo(RestStatus.NOT_FOUND)); + } + + public void testCRUDIndexTemplate() throws Exception { RestHighLevelClient client = highLevelClient(); - PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest().name("template-1") + PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest("template-1") .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), equalTo(true)); - PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest().name("template-2") + PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest("template-2") .patterns(Arrays.asList("pattern-2", "name-2")) + .mapping("{\"properties\": { \"name\": { \"type\": \"text\" }}}", XContentType.JSON) .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), - equalTo(true)); + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync) + .isAcknowledged(), equalTo(true)); - GetIndexTemplatesResponse getTemplate1 = execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getTemplate1 = execute( + new GetIndexTemplatesRequest("template-1"), + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); assertThat(template1.name(), equalTo("template-1")); assertThat(template1.patterns(), contains("pattern-1", "name-1")); assertTrue(template1.aliases().containsKey("alias-1")); - + GetIndexTemplatesResponse getTemplate2 = execute(new GetIndexTemplatesRequest("template-2"), - client.indices()::getTemplate, client.indices()::getTemplateAsync); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); assertThat(template2.name(), equalTo("template-2")); assertThat(template2.patterns(), contains("pattern-2", "name-2")); assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + // New API returns a MappingMetaData class rather than CompressedXContent for the mapping + assertTrue(template2.mappings().sourceAsMap().containsKey("properties")); + @SuppressWarnings("unchecked") + Map props = (Map) template2.mappings().sourceAsMap().get("properties"); + assertTrue(props.containsKey("name")); + + List names = randomBoolean() ? Arrays.asList("*-1", "template-2") : Arrays.asList("template-*"); GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); - GetIndexTemplatesResponse getBoth = execute(getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getBoth = execute( + getBothRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getBoth.getIndexTemplates(), hasSize(2)); - assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).toArray(), + assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::name).toArray(), arrayContainingInAnyOrder("template-1", "template-2")); GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); - GetIndexTemplatesResponse getAll = execute(getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); + GetIndexTemplatesResponse getAll = execute( + getAllRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); - assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).collect(Collectors.toList()), + assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::name) + .collect(Collectors.toList()), hasItems("template-1", "template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates(), hasSize(1)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates(), hasSize(1)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates().get(0).name(), equalTo("template-2")); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates() + .get(0).name(), equalTo("template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); } public void testIndexTemplatesExist() throws Exception { @@ -1559,8 +1753,7 @@ public void testIndexTemplatesExist() throws Exception { { for (String suffix : Arrays.asList("1", "2")) { - final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest() - .name("template-" + suffix) + final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("template-" + suffix) .patterns(Arrays.asList("pattern-" + suffix, "name-" + suffix)) .alias(new Alias("alias-" + suffix)); assertTrue(execute(putRequest, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 409690eaa6997..6d873ec2b944c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; @@ -53,6 +52,7 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; import org.elasticsearch.common.CheckedFunction; @@ -60,6 +60,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.junit.Assert; @@ -921,14 +922,16 @@ public void testIndexPutSettings() throws IOException { Assert.assertEquals(expectedParams, request.getParameters()); } - public void testPutTemplateRequest() throws Exception { + public void testPutTemplateRequestWithTypes() throws Exception { Map names = new HashMap<>(); names.put("log", "log"); names.put("template#1", "template%231"); names.put("-#template", "-%23template"); names.put("foo^bar", "foo%5Ebar"); - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest().name(ESTestCase.randomFrom(names.keySet())) + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() + .name(ESTestCase.randomFrom(names.keySet())) .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); if (ESTestCase.randomBoolean()) { putTemplateRequest.order(ESTestCase.randomInt()); @@ -939,14 +942,15 @@ public void testPutTemplateRequest() throws Exception { if (ESTestCase.randomBoolean()) { putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); } + Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { putTemplateRequest.mapping("doc-" + ESTestCase.randomInt(), "field-" + ESTestCase.randomInt(), "type=" + ESTestCase.randomFrom("text", "keyword")); } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); } - Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { expectedParams.put("create", Boolean.TRUE.toString()); putTemplateRequest.create(true); @@ -955,9 +959,8 @@ public void testPutTemplateRequest() throws Exception { String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); putTemplateRequest.cause(cause); expectedParams.put("cause", cause); - } + } RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); @@ -965,6 +968,49 @@ public void testPutTemplateRequest() throws Exception { RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); } + public void testPutTemplateRequest() throws Exception { + Map names = new HashMap<>(); + names.put("log", "log"); + names.put("template#1", "template%231"); + names.put("-#template", "-%23template"); + names.put("foo^bar", "foo%5Ebar"); + + PutIndexTemplateRequest putTemplateRequest = + new PutIndexTemplateRequest(ESTestCase.randomFrom(names.keySet())) + .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); + if (ESTestCase.randomBoolean()) { + putTemplateRequest.order(ESTestCase.randomInt()); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.version(ESTestCase.randomInt()); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); + } + Map expectedParams = new HashMap<>(); + if (ESTestCase.randomBoolean()) { + putTemplateRequest.mapping("{ \"properties\": { \"field-" + ESTestCase.randomInt() + + "\" : { \"type\" : \"" + ESTestCase.randomFrom("text", "keyword") + "\" }}}", XContentType.JSON); + } + if (ESTestCase.randomBoolean()) { + putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); + } + if (ESTestCase.randomBoolean()) { + expectedParams.put("create", Boolean.TRUE.toString()); + putTemplateRequest.create(true); + } + if (ESTestCase.randomBoolean()) { + String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); + putTemplateRequest.cause(cause); + expectedParams.put("cause", cause); + } + RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); + + Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); + Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); + Assert.assertThat(request.getParameters(), equalTo(expectedParams)); + RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); + } public void testValidateQuery() throws Exception { String[] indices = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); String[] types = ESTestCase.randomBoolean() ? ESTestCase.generateRandomStringArray(5, 5, false, false) : null; @@ -1012,9 +1058,9 @@ public void testGetTemplateRequest() throws Exception { Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomMasterTimeout(getTemplatesRequest::setMasterNodeTimeout, expectedParams); RequestConvertersTests.setRandomLocal(getTemplatesRequest::setLocal, expectedParams); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); - Request request = IndicesRequestConverters.getTemplates(getTemplatesRequest); + Request request = IndicesRequestConverters.getTemplatesWithDocumentTypes(getTemplatesRequest); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.stream().map(encodes::get).collect(Collectors.joining(",")))); Assert.assertThat(request.getParameters(), equalTo(expectedParams)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index a20d78f939da5..945c1df19efa3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -744,6 +744,14 @@ public void testApiNamingConventions() throws Exception { .collect(Collectors.groupingBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet()))); + // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate + // impl will replace the existing getTemplate method. + // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` + // looking like it doesn't have a valid implementatation when it does. + apiUnsupported.remove("indices.get_template"); + + + for (Map.Entry> entry : methods.entrySet()) { String apiName = entry.getKey(); @@ -776,7 +784,10 @@ public void testApiNamingConventions() throws Exception { apiName.startsWith("security.") == false && apiName.startsWith("index_lifecycle.") == false && apiName.startsWith("ccr.") == false && - apiName.endsWith("freeze") == false) { + apiName.endsWith("freeze") == false && + // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we + // can get rid of 7.0's deprecated "getTemplate" + apiName.equals("indices.get_index_template") == false) { apiNotFound.add(apiName); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index cfd5ac4de2f82..83244b90b950f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -55,8 +55,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; @@ -76,11 +74,13 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -2094,13 +2094,11 @@ public void testPutTemplate() throws Exception { { // tag::put-template-request-mappings-json - request.mapping("_doc", // <1> + request.mapping(// <1> "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + "}", // <2> @@ -2111,14 +2109,16 @@ public void testPutTemplate() throws Exception { { //tag::put-template-request-mappings-map Map jsonMap = new HashMap<>(); - Map message = new HashMap<>(); - message.put("type", "text"); - Map properties = new HashMap<>(); - properties.put("message", message); - Map mapping = new HashMap<>(); - mapping.put("properties", properties); - jsonMap.put("_doc", mapping); - request.mapping("_doc", jsonMap); // <1> + { + Map properties = new HashMap<>(); + { + Map message = new HashMap<>(); + message.put("type", "text"); + properties.put("message", message); + } + jsonMap.put("properties", properties); + } + request.mapping(jsonMap); // <1> //end::put-template-request-mappings-map assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } @@ -2127,31 +2127,21 @@ public void testPutTemplate() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); { - builder.startObject("_doc"); + builder.startObject("properties"); { - builder.startObject("properties"); + builder.startObject("message"); { - builder.startObject("message"); - { - builder.field("type", "text"); - } - builder.endObject(); + builder.field("type", "text"); } builder.endObject(); } builder.endObject(); } builder.endObject(); - request.mapping("_doc", builder); // <1> + request.mapping(builder); // <1> //end::put-template-request-mappings-xcontent assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } - { - //tag::put-template-request-mappings-shortcut - request.mapping("_doc", "message", "type=text"); // <1> - //end::put-template-request-mappings-shortcut - assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); - } // tag::put-template-request-aliases request.alias(new Alias("twitter_alias").filter(QueryBuilders.termQuery("user", "kimchy"))); // <1> @@ -2177,11 +2167,9 @@ public void testPutTemplate() throws Exception { " \"number_of_shards\": 1\n" + " },\n" + " \"mappings\": {\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + " },\n" + @@ -2244,13 +2232,11 @@ public void testGetTemplates() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template"); putRequest.patterns(Arrays.asList("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3).put("index.number_of_replicas", 1)); - putRequest.mapping("_doc", + putRequest.mapping( "{\n" + - " \"_doc\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + - " }\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + " }\n" + " }\n" + "}", XContentType.JSON); @@ -2269,7 +2255,7 @@ public void testGetTemplates() throws Exception { // end::get-templates-request-masterTimeout // tag::get-templates-execute - GetIndexTemplatesResponse getTemplatesResponse = client.indices().getTemplate(request, RequestOptions.DEFAULT); + GetIndexTemplatesResponse getTemplatesResponse = client.indices().getIndexTemplate(request, RequestOptions.DEFAULT); // end::get-templates-execute // tag::get-templates-response @@ -2299,7 +2285,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-templates-execute-async - client.indices().getTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.indices().getIndexTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-templates-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java new file mode 100644 index 0000000000000..d2f0c3d7eba88 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class GetIndexTemplatesResponseTests extends ESTestCase { + + static final String mappingString = "{\"properties\":{" + + "\"f1\": {\"type\":\"text\"}," + + "\"f2\": {\"type\":\"keyword\"}" + + "}}"; + + + public void testFromXContent() throws IOException { + xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent, + GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false) + .assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances) + .shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"}) + .test(); + } + + private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) { + assertEquals(expectedInstance, newInstance); + // Check there's no doc types at the root of the mapping + Map expectedMap = XContentHelper.convertToMap( + new BytesArray(mappingString), true, XContentType.JSON).v2(); + for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) { + MappingMetaData mappingMD = template.mappings(); + if(mappingMD!=null) { + Map mappingAsMap = mappingMD.sourceAsMap(); + assertEquals(expectedMap, mappingAsMap); + } + } + } + + static GetIndexTemplatesResponse createTestInstance() { + List templates = new ArrayList<>(); + int numTemplates = between(0, 10); + for (int t = 0; t < numTemplates; t++) { + IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder("template-" + t); + templateBuilder.patterns(IntStream.range(0, between(1, 5)).mapToObj(i -> "pattern-" + i).collect(Collectors.toList())); + int numAlias = between(0, 5); + for (int i = 0; i < numAlias; i++) { + templateBuilder.putAlias(AliasMetaData.builder(randomAlphaOfLengthBetween(1, 10))); + } + if (randomBoolean()) { + templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); + } + if (randomBoolean()) { + templateBuilder.order(randomInt()); + } + if (randomBoolean()) { + templateBuilder.version(between(0, 100)); + } + if (randomBoolean()) { + try { + Map map = XContentHelper.convertToMap(new BytesArray(mappingString), true, XContentType.JSON).v2(); + MappingMetaData mapping = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, map); + templateBuilder.mapping(mapping); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + templates.add(templateBuilder.build()); + } + return new GetIndexTemplatesResponse(templates); + } + + // As the client class GetIndexTemplatesResponse doesn't have toXContent method, adding this method here only for the test + static void toXContent(GetIndexTemplatesResponse response, XContentBuilder builder) throws IOException { + + //Create a server-side counterpart for the client-side class and call toXContent on it + + List serverIndexTemplates = new ArrayList<>(); + List clientIndexTemplates = response.getIndexTemplates(); + for (IndexTemplateMetaData clientITMD : clientIndexTemplates) { + org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder serverTemplateBuilder = + org.elasticsearch.cluster.metadata.IndexTemplateMetaData.builder(clientITMD.name()); + + serverTemplateBuilder.patterns(clientITMD.patterns()); + + Iterator aliases = clientITMD.aliases().valuesIt(); + aliases.forEachRemaining((a)->serverTemplateBuilder.putAlias(a)); + + serverTemplateBuilder.settings(clientITMD.settings()); + serverTemplateBuilder.order(clientITMD.order()); + serverTemplateBuilder.version(clientITMD.version()); + if (clientITMD.mappings() != null) { + serverTemplateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, clientITMD.mappings().source()); + } + serverIndexTemplates.add(serverTemplateBuilder.build()); + + } + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse serverResponse = new + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates); + serverResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java new file mode 100644 index 0000000000000..8aab973982fc0 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.Is.is; + +public class PutIndexTemplateRequestTests extends AbstractXContentTestCase { + public void testValidateErrorMessage() throws Exception { + expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest(null)); + expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest("test").name(null)); + PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); + ActionRequestValidationException withoutPattern = request.validate(); + assertThat(withoutPattern.getMessage(), containsString("index patterns are missing")); + + request.name("foo"); + ActionRequestValidationException withoutIndexPatterns = request.validate(); + assertThat(withoutIndexPatterns.validationErrors(), hasSize(1)); + assertThat(withoutIndexPatterns.getMessage(), containsString("index patterns are missing")); + + request.patterns(Collections.singletonList("test-*")); + ActionRequestValidationException noError = request.validate(); + assertThat(noError, is(nullValue())); + } + + @Override + protected PutIndexTemplateRequest createTestInstance() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); + if (randomBoolean()) { + request.version(randomInt()); + } + if (randomBoolean()) { + request.order(randomInt()); + } + request.patterns(Arrays.asList(generateRandomStringArray(20, 100, false, false))); + int numAlias = between(0, 5); + for (int i = 0; i < numAlias; i++) { + // some ASCII or Latin-1 control characters, especially newline, can lead to + // problems with yaml parsers, that's why we filter them here (see #30911) + Alias alias = new Alias(randomRealisticUnicodeOfLengthBetween(1, 10).replaceAll("\\p{Cc}", "")); + if (randomBoolean()) { + alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); + } + if (randomBoolean()) { + alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); + } + request.alias(alias); + } + if (randomBoolean()) { + try { + request.mapping(XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("field-" + randomInt()).field("type", randomFrom("keyword", "text")).endObject() + .endObject().endObject()); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + if (randomBoolean()) { + request.settings(Settings.builder().put("setting1", randomLong()).put("setting2", randomTimeValue()).build()); + } + return request; + } + + @Override + protected PutIndexTemplateRequest doParseInstance(XContentParser parser) throws IOException { + return new PutIndexTemplateRequest("test").source(parser.map()); + } + + @Override + protected void assertEqualInstances(PutIndexTemplateRequest expected, PutIndexTemplateRequest actual) { + assertNotSame(expected, actual); + assertThat(actual.version(), equalTo(expected.version())); + assertThat(actual.order(), equalTo(expected.order())); + assertThat(actual.patterns(), equalTo(expected.patterns())); + assertThat(actual.aliases(), equalTo(expected.aliases())); + assertThat(actual.mappings(), equalTo(expected.mappings())); + assertThat(actual.settings(), equalTo(expected.settings())); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/docs/java-rest/high-level/indices/put_template.asciidoc b/docs/java-rest/high-level/indices/put_template.asciidoc index 7618fc8ce7af5..3e3954308736d 100644 --- a/docs/java-rest/high-level/indices/put_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_template.asciidoc @@ -39,8 +39,7 @@ template's patterns. -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-mappings-json] -------------------------------------------------- -<1> The type to define -<2> The mapping for this type, provided as a JSON string +<1> The mapping, provided as a JSON string The mapping source can be provided in different ways in addition to the `String` example shown above: @@ -59,13 +58,6 @@ include-tagged::{doc-tests-file}[{api}-request-mappings-xcontent] <1> Mapping source provided as an `XContentBuilder` object, the Elasticsearch built-in helpers to generate JSON content -["source","java",subs="attributes,callouts,macros"] --------------------------------------------------- -include-tagged::{doc-tests-file}[{api}-request-mappings-shortcut] --------------------------------------------------- -<1> Mapping source provided as `Object` key-pairs, which gets converted to -JSON format - ==== Aliases The aliases of the template will define aliasing to the index whose name matches the template's patterns. A placeholder `{index}` can be used in an alias of a template. diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 43bb7401dba58..1b2503ccb99d5 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -35,6 +35,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.rest.action.document.RestUpdateAction; @@ -921,6 +923,7 @@ public void testSnapshotRestore() throws IOException { // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); + createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } client().performRequest(createTemplateRequest); @@ -1122,6 +1125,7 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); + getTemplateRequest.setOptions(expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } Map getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9749aaa05b1a4..e5cd947f0f046 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -43,7 +43,7 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont indexTemplates = new ArrayList<>(); } - GetIndexTemplatesResponse(List indexTemplates) { + public GetIndexTemplatesResponse(List indexTemplates) { this.indexTemplates = indexTemplates; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 50370797aa6f2..707378eec4cf6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -19,10 +19,12 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; @@ -47,6 +49,10 @@ public class RestGetIndexTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMETERS = Collections.unmodifiableSet(Sets.union( Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER), Settings.FORMAT_PARAMS)); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestGetIndexTemplateAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in get index template requests is deprecated."; public RestGetIndexTemplateAction(final Settings settings, final RestController controller) { super(settings); @@ -65,6 +71,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); + } getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 2b72a724a8906..445d393fef82b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -42,6 +42,9 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger( LogManager.getLogger(RestPutIndexTemplateAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in put index template requests is deprecated."+ + " The parameter will be removed in the next major version."; public RestPutIndexTemplateAction(Settings settings, RestController controller) { super(settings); @@ -57,6 +60,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name")); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); + } if (request.hasParam("template")) { deprecationLogger.deprecated("Deprecated parameter[template] used, replaced by [index_patterns]"); putRequest.patterns(Collections.singletonList(request.param("template"))); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java index ac0eb8f0d81a6..d1900aaee84d5 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.rest.action.admin.indices; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -31,8 +32,12 @@ import org.junit.Before; import java.io.IOException; +import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + public class RestPutIndexTemplateActionTests extends RestActionTestCase { private RestPutIndexTemplateAction action; @@ -45,7 +50,8 @@ public void testPrepareTypelessRequest() throws IOException { XContentBuilder content = XContentFactory.jsonBuilder().startObject() .startObject("mappings") .startObject("properties") - .startObject("field").field("type", "keyword").endObject() + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() .endObject() .endObject() .startObject("aliases") @@ -58,6 +64,11 @@ public void testPrepareTypelessRequest() throws IOException { .withPath("/_template/_some_template") .withContent(BytesReference.bytes(content), XContentType.JSON) .build(); + action.prepareRequest(request, mock(NodeClient.class)); + + // Internally the above prepareRequest method calls prepareRequestSource to inject a + // default type into the mapping. Here we test that this does what is expected by + // explicitly calling that same helper function boolean includeTypeName = false; Map source = action.prepareRequestSource(request, includeTypeName); @@ -65,7 +76,8 @@ public void testPrepareTypelessRequest() throws IOException { .startObject("mappings") .startObject("_doc") .startObject("properties") - .startObject("field").field("type", "keyword").endObject() + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() .endObject() .endObject() .endObject() @@ -78,4 +90,51 @@ public void testPrepareTypelessRequest() throws IOException { assertEquals(expectedContentAsMap, source); } + + public void testIncludeTypeName() throws IOException { + XContentBuilder typedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withParams(params) + .withPath("/_template/_some_template") + .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .build(); + action.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); + boolean includeTypeName = true; + Map source = action.prepareRequestSource(request, includeTypeName); + + XContentBuilder expectedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + Map expectedContentAsMap = XContentHelper.convertToMap( + BytesReference.bytes(expectedContent), true, expectedContent.contentType()).v2(); + + assertEquals(expectedContentAsMap, source); + } } From 0470ee1fcc757296e3f7f1f168a3a98539c01f44 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 29 Jan 2019 21:24:08 +0000 Subject: [PATCH 15/35] Docs fix - missing callout --- .../client/documentation/IndicesClientDocumentationIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 83244b90b950f..d358655f2355a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2101,7 +2101,7 @@ public void testPutTemplate() throws Exception { " \"type\": \"text\"\n" + " }\n" + " }\n" + - "}", // <2> + "}", XContentType.JSON); // end::put-template-request-mappings-json assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); From 55b916afc04e7867d6fc5070445c3551b8d5a364 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 29 Jan 2019 15:58:31 -0700 Subject: [PATCH 16/35] Ensure task metadata not null in follow test (#37993) This commit fixes a potential race in the IndexFollowingIT. Currently it is possible that we fetch the task metadata, it is null, and that throws a null pointer exception. Assertbusy does not catch null pointer exceptions. This commit assertions that the metadata is not null. --- .../test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index dec671f6e6340..55fcb6ace89fd 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -983,6 +983,7 @@ private CheckedRunnable assertTask(final int numberOfPrimaryShards, f return () -> { final ClusterState clusterState = followerClient().admin().cluster().prepareState().get().getState(); final PersistentTasksCustomMetaData taskMetadata = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + assertNotNull(taskMetadata); ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setDetailed(true); From f5b9b4d89c9e28dba020cd53c39b582fb01d95bd Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 30 Jan 2019 15:32:45 +1100 Subject: [PATCH 17/35] Add version 6.6.1 (#37975) --- server/src/main/java/org/elasticsearch/Version.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index b66630344a7ea..e520d714bb931 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -116,10 +116,10 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_6_5_3 = new Version(V_6_5_3_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); public static final int V_6_5_4_ID = 6050499; public static final Version V_6_5_4 = new Version(V_6_5_4_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); - public static final int V_6_5_5_ID = 6050599; - public static final Version V_6_5_5 = new Version(V_6_5_5_ID, org.apache.lucene.util.Version.LUCENE_7_5_0); public static final int V_6_6_0_ID = 6060099; public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); + public static final int V_6_6_1_ID = 6060199; + public static final Version V_6_6_1 = new Version(V_6_6_1_ID, org.apache.lucene.util.Version.LUCENE_7_6_0); public static final int V_6_7_0_ID = 6070099; public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0); public static final int V_7_0_0_ID = 7000099; @@ -142,10 +142,10 @@ public static Version fromId(int id) { return V_7_0_0; case V_6_7_0_ID: return V_6_7_0; + case V_6_6_1_ID: + return V_6_6_1; case V_6_6_0_ID: return V_6_6_0; - case V_6_5_5_ID: - return V_6_5_5; case V_6_5_4_ID: return V_6_5_4; case V_6_5_3_ID: From 57823c484fddafb65accce261621a3b051e647db Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 30 Jan 2019 06:22:31 +0100 Subject: [PATCH 18/35] Streamline S3 Repository- and Client-Settings (#37393) * Make repository settings override static settings * Cache clients according to settings * Introduce custom implementations for the AWS credentials here to be able to use them as part of a hash key --- docs/plugins/repository-s3.asciidoc | 26 ++++ .../repositories/s3/S3BasicCredentials.java | 62 +++++++++ .../s3/S3BasicSessionCredentials.java | 57 ++++++++ .../repositories/s3/S3BlobStore.java | 14 +- .../repositories/s3/S3ClientSettings.java | 126 +++++++++++++----- .../repositories/s3/S3Repository.java | 50 +------ .../repositories/s3/S3Service.java | 104 ++++++++++----- .../s3/AwsS3ServiceImplTests.java | 6 +- .../s3/S3BlobStoreRepositoryTests.java | 2 - .../repositories/s3/S3BlobStoreTests.java | 9 +- .../s3/S3ClientSettingsTests.java | 32 ++++- .../repositories/s3/S3RepositoryTests.java | 6 +- .../20_repository_permanent_credentials.yml | 80 +++++++++++ 13 files changed, 444 insertions(+), 130 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java create mode 100644 plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index b0b87dda792fe..e7ab83ca6e69b 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -221,6 +221,32 @@ The following settings are supported: currently supported by the plugin. For more information about the different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] +NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated: + +In addition to the above settings, you may also specify all non-secure client settings in the repository settings. +In this case, the client settings found in the repository settings will be merged with those of the named client used by the repository. +Conflicts between client and repository settings are resolved by the repository settings taking precedence over client settings. + +For example: + +[source,js] +---- +PUT _snapshot/my_s3_repository +{ + "type": "s3", + "settings": { + "client": "my_client_name", + "bucket": "my_bucket_name", + "endpoint": "my.s3.endpoint" + } +} +---- +// CONSOLE +// TEST[skip:we don't have s3 set up while testing this] + +This sets up a repository that uses all client settings from the client `my_client_named` except for the `endpoint` that is overridden +to `my.s3.endpoint` by the repository settings. + [[repository-s3-permissions]] ===== Recommended S3 Permissions diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java new file mode 100644 index 0000000000000..f6dfb692d1de2 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicCredentials.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.s3; + +import com.amazonaws.auth.AWSCredentials; + +import java.util.Objects; + +class S3BasicCredentials implements AWSCredentials { + + private final String accessKey; + + private final String secretKey; + + S3BasicCredentials(String accessKey, String secretKey) { + this.accessKey = accessKey; + this.secretKey = secretKey; + } + + @Override + public final String getAWSAccessKeyId() { + return accessKey; + } + + @Override + public final String getAWSSecretKey() { + return secretKey; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final S3BasicCredentials that = (S3BasicCredentials) o; + return accessKey.equals(that.accessKey) && secretKey.equals(that.secretKey); + } + + @Override + public int hashCode() { + return Objects.hash(accessKey, secretKey); + } +} diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java new file mode 100644 index 0000000000000..4057403aa2c0f --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BasicSessionCredentials.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.s3; + +import com.amazonaws.auth.AWSSessionCredentials; + +import java.util.Objects; + +final class S3BasicSessionCredentials extends S3BasicCredentials implements AWSSessionCredentials { + + private final String sessionToken; + + S3BasicSessionCredentials(String accessKey, String secretKey, String sessionToken) { + super(accessKey, secretKey); + this.sessionToken = sessionToken; + } + + @Override + public String getSessionToken() { + return sessionToken; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final S3BasicSessionCredentials that = (S3BasicSessionCredentials) o; + return sessionToken.equals(that.sessionToken) && + getAWSAccessKeyId().equals(that.getAWSAccessKeyId()) && + getAWSSecretKey().equals(that.getAWSSecretKey()); + } + + @Override + public int hashCode() { + return Objects.hash(sessionToken, getAWSAccessKeyId(), getAWSSecretKey()); + } +} diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 27f2b305abd32..d4df4094fcf92 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -25,6 +25,7 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -39,8 +40,6 @@ class S3BlobStore implements BlobStore { private final S3Service service; - private final String clientName; - private final String bucket; private final ByteSizeValue bufferSize; @@ -51,15 +50,18 @@ class S3BlobStore implements BlobStore { private final StorageClass storageClass; - S3BlobStore(S3Service service, String clientName, String bucket, boolean serverSideEncryption, - ByteSizeValue bufferSize, String cannedACL, String storageClass) { + private final RepositoryMetaData repositoryMetaData; + + S3BlobStore(S3Service service, String bucket, boolean serverSideEncryption, + ByteSizeValue bufferSize, String cannedACL, String storageClass, + RepositoryMetaData repositoryMetaData) { this.service = service; - this.clientName = clientName; this.bucket = bucket; this.serverSideEncryption = serverSideEncryption; this.bufferSize = bufferSize; this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); + this.repositoryMetaData = repositoryMetaData; } @Override @@ -68,7 +70,7 @@ public String toString() { } public AmazonS3Reference clientReference() { - return service.client(clientName); + return service.client(repositoryMetaData); } public String bucket() { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 58fca161415a4..ea45fbaf93dd3 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -21,9 +21,6 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -36,6 +33,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -46,6 +44,9 @@ final class S3ClientSettings { // prefix for s3 client settings private static final String PREFIX = "s3.client."; + /** Placeholder client name for normalizing client settings in the repository settings. */ + private static final String PLACEHOLDER_CLIENT = "placeholder"; + /** The access key (ie login id) for connecting to s3. */ static final Setting.AffixSetting ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key", key -> SecureSetting.secureString(key, null)); @@ -95,7 +96,7 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Credentials to authenticate with s3. */ - final AWSCredentials credentials; + final S3BasicCredentials credentials; /** The s3 endpoint the client should talk to, or empty string to use the default. */ final String endpoint; @@ -126,7 +127,7 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; - protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries) { this.credentials = credentials; @@ -141,6 +142,51 @@ protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol this.throttleRetries = throttleRetries; } + /** + * Overrides the settings in this instance with settings found in repository metadata. + * + * @param metadata RepositoryMetaData + * @return S3ClientSettings + */ + S3ClientSettings refine(RepositoryMetaData metadata) { + final Settings repoSettings = metadata.settings(); + // Normalize settings to placeholder client settings prefix so that we can use the affix settings directly + final Settings normalizedSettings = + Settings.builder().put(repoSettings).normalizePrefix(PREFIX + PLACEHOLDER_CLIENT + '.').build(); + final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint); + + final Protocol newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol); + final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost); + final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxyPort); + final int newReadTimeoutMillis = Math.toIntExact( + getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); + final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); + final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); + final S3BasicCredentials newCredentials; + if (checkDeprecatedCredentials(repoSettings)) { + newCredentials = loadDeprecatedCredentials(repoSettings); + } else { + newCredentials = credentials; + } + if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) + && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries + && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)) { + return this; + } + return new S3ClientSettings( + newCredentials, + newEndpoint, + newProtocol, + newProxyHost, + newProxyPort, + proxyUsername, + proxyPassword, + newReadTimeoutMillis, + newMaxRetries, + newThrottleRetries + ); + } + /** * Load all client settings from the given settings. * @@ -175,24 +221,24 @@ static boolean checkDeprecatedCredentials(Settings repositorySettings) { } // backcompat for reading keys out of repository settings (clusterState) - static BasicAWSCredentials loadDeprecatedCredentials(Settings repositorySettings) { + private static S3BasicCredentials loadDeprecatedCredentials(Settings repositorySettings) { assert checkDeprecatedCredentials(repositorySettings); try (SecureString key = S3Repository.ACCESS_KEY_SETTING.get(repositorySettings); SecureString secret = S3Repository.SECRET_KEY_SETTING.get(repositorySettings)) { - return new BasicAWSCredentials(key.toString(), secret.toString()); + return new S3BasicCredentials(key.toString(), secret.toString()); } } - static AWSCredentials loadCredentials(Settings settings, String clientName) { + private static S3BasicCredentials loadCredentials(Settings settings, String clientName) { try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING); SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING)) { if (accessKey.length() != 0) { if (secretKey.length() != 0) { if (sessionToken.length() != 0) { - return new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); + return new S3BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); } else { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + return new S3BasicCredentials(accessKey.toString(), secretKey.toString()); } } else { throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]"); @@ -212,34 +258,48 @@ static AWSCredentials loadCredentials(Settings settings, String clientName) { // pkg private for tests /** Parse settings for a single client. */ static S3ClientSettings getClientSettings(final Settings settings, final String clientName) { - final AWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); - return getClientSettings(settings, clientName, credentials); - } - - static S3ClientSettings getClientSettings(final Settings settings, final String clientName, final AWSCredentials credentials) { try (SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) { return new S3ClientSettings( - credentials, - getConfigValue(settings, clientName, ENDPOINT_SETTING), - getConfigValue(settings, clientName, PROTOCOL_SETTING), - getConfigValue(settings, clientName, PROXY_HOST_SETTING), - getConfigValue(settings, clientName, PROXY_PORT_SETTING), - proxyUsername.toString(), - proxyPassword.toString(), - Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), - getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) + S3ClientSettings.loadCredentials(settings, clientName), + getConfigValue(settings, clientName, ENDPOINT_SETTING), + getConfigValue(settings, clientName, PROTOCOL_SETTING), + getConfigValue(settings, clientName, PROXY_HOST_SETTING), + getConfigValue(settings, clientName, PROXY_PORT_SETTING), + proxyUsername.toString(), + proxyPassword.toString(), + Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), + getConfigValue(settings, clientName, MAX_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) ); } } - static S3ClientSettings getClientSettings(final RepositoryMetaData metadata, final AWSCredentials credentials) { - final Settings.Builder builder = Settings.builder(); - for (final String key : metadata.settings().keySet()) { - builder.put(PREFIX + "provided" + "." + key, metadata.settings().get(key)); + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; } - return getClientSettings(builder.build(), "provided", credentials); + final S3ClientSettings that = (S3ClientSettings) o; + return proxyPort == that.proxyPort && + readTimeoutMillis == that.readTimeoutMillis && + maxRetries == that.maxRetries && + throttleRetries == that.throttleRetries && + Objects.equals(credentials, that.credentials) && + Objects.equals(endpoint, that.endpoint) && + protocol == that.protocol && + Objects.equals(proxyHost, that.proxyHost) && + Objects.equals(proxyUsername, that.proxyUsername) && + Objects.equals(proxyPassword, that.proxyPassword); + } + + @Override + public int hashCode() { + return Objects.hash(credentials, endpoint, protocol, proxyHost, proxyPort, proxyUsername, proxyPassword, + readTimeoutMillis, maxRetries, throttleRetries); } private static T getConfigValue(Settings settings, String clientName, @@ -248,4 +308,10 @@ private static T getConfigValue(Settings settings, String clientName, return concreteSetting.get(settings); } + private static T getRepoSettingOrDefault(Setting.AffixSetting setting, Settings normalizedSettings, T defaultValue) { + if (setting.getConcreteSettingForNamespace(PLACEHOLDER_CLIENT).exists(normalizedSettings)) { + return getConfigValue(normalizedSettings, PLACEHOLDER_CLIENT, setting); + } + return defaultValue; + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index e0e34e40f3cf8..b1d29d89a59c0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.auth.BasicAWSCredentials; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.elasticsearch.cluster.metadata.RepositoryMetaData; @@ -148,8 +147,6 @@ class S3Repository extends BlobStoreRepository { */ static final Setting BASE_PATH_SETTING = Setting.simpleString("base_path"); - private final Settings settings; - private final S3Service service; private final String bucket; @@ -168,9 +165,7 @@ class S3Repository extends BlobStoreRepository { private final String cannedACL; - private final String clientName; - - private final AmazonS3Reference reference; + private final RepositoryMetaData repositoryMetaData; /** * Constructs an s3 backed repository @@ -180,9 +175,10 @@ class S3Repository extends BlobStoreRepository { final NamedXContentRegistry namedXContentRegistry, final S3Service service) { super(metadata, settings, namedXContentRegistry); - this.settings = settings; this.service = service; + this.repositoryMetaData = metadata; + // Parse and validate the user's S3 Storage Class setting this.bucket = BUCKET_SETTING.get(metadata.settings()); if (bucket == null) { @@ -211,24 +207,10 @@ class S3Repository extends BlobStoreRepository { this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings()); this.cannedACL = CANNED_ACL_SETTING.get(metadata.settings()); - this.clientName = CLIENT_NAME.get(metadata.settings()); - - if (CLIENT_NAME.exists(metadata.settings()) && S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { - logger.warn( - "ignoring use of named client [{}] for repository [{}] as insecure credentials were specified", - clientName, - metadata.name()); - } - if (S3ClientSettings.checkDeprecatedCredentials(metadata.settings())) { // provided repository settings deprecationLogger.deprecated("Using s3 access/secret key from repository settings. Instead " + "store these in named clients and the elasticsearch keystore for secure settings."); - final BasicAWSCredentials insecureCredentials = S3ClientSettings.loadDeprecatedCredentials(metadata.settings()); - final S3ClientSettings s3ClientSettings = S3ClientSettings.getClientSettings(metadata, insecureCredentials); - this.reference = new AmazonS3Reference(service.buildClient(s3ClientSettings)); - } else { - reference = null; } logger.debug( @@ -243,21 +225,7 @@ class S3Repository extends BlobStoreRepository { @Override protected S3BlobStore createBlobStore() { - if (reference != null) { - assert S3ClientSettings.checkDeprecatedCredentials(metadata.settings()) : metadata.name(); - return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass) { - @Override - public AmazonS3Reference clientReference() { - if (reference.tryIncRef()) { - return reference; - } else { - throw new IllegalStateException("S3 client is closed"); - } - } - }; - } else { - return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); - } + return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, repositoryMetaData); } // only use for testing @@ -286,14 +254,4 @@ protected boolean isCompress() { protected ByteSizeValue chunkSize() { return chunkSize; } - - @Override - protected void doClose() { - if (reference != null) { - assert S3ClientSettings.checkDeprecatedCredentials(metadata.settings()) : metadata.name(); - reference.decRef(); - } - super.doClose(); - } - } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index a5ee861d0c38b..b0c8a6198130a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -22,18 +22,20 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; -import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.internal.Constants; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.settings.Settings; import java.io.Closeable; import java.io.IOException; @@ -43,56 +45,96 @@ class S3Service implements Closeable { - private static final Logger logger = LogManager.getLogger(S3Service.class); - private volatile Map clientsCache = emptyMap(); - private volatile Map clientsSettings = emptyMap(); + private volatile Map clientsCache = emptyMap(); + + /** + * Client settings calculated from static configuration and settings in the keystore. + */ + private volatile Map staticClientSettings = MapBuilder.newMapBuilder() + .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default")).immutableMap(); + + /** + * Client settings derived from those in {@link #staticClientSettings} by combining them with settings + * in the {@link RepositoryMetaData}. + */ + private volatile Map> derivedClientSettings = emptyMap(); /** * Refreshes the settings for the AmazonS3 clients and clears the cache of * existing clients. New clients will be build using these new settings. Old * clients are usable until released. On release they will be destroyed instead - * to being returned to the cache. + * of being returned to the cache. */ - public synchronized Map refreshAndClearCache(Map clientsSettings) { + public synchronized void refreshAndClearCache(Map clientsSettings) { // shutdown all unused clients // others will shutdown on their respective release releaseCachedClients(); - final Map prevSettings = this.clientsSettings; - this.clientsSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); - assert this.clientsSettings.containsKey("default") : "always at least have 'default'"; - // clients are built lazily by {@link client(String)} - return prevSettings; + this.staticClientSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); + derivedClientSettings = emptyMap(); + assert this.staticClientSettings.containsKey("default") : "always at least have 'default'"; + // clients are built lazily by {@link client} } /** - * Attempts to retrieve a client by name from the cache. If the client does not - * exist it will be created. + * Attempts to retrieve a client by its repository metadata and settings from the cache. + * If the client does not exist it will be created. */ - public AmazonS3Reference client(String clientName) { - AmazonS3Reference clientReference = clientsCache.get(clientName); - if ((clientReference != null) && clientReference.tryIncRef()) { - return clientReference; - } - synchronized (this) { - clientReference = clientsCache.get(clientName); - if ((clientReference != null) && clientReference.tryIncRef()) { + public AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { + final S3ClientSettings clientSettings = settings(repositoryMetaData); + { + final AmazonS3Reference clientReference = clientsCache.get(clientSettings); + if (clientReference != null && clientReference.tryIncRef()) { return clientReference; } - final S3ClientSettings clientSettings = clientsSettings.get(clientName); - if (clientSettings == null) { - throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " - + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); + } + synchronized (this) { + final AmazonS3Reference existing = clientsCache.get(clientSettings); + if (existing != null && existing.tryIncRef()) { + return existing; } - logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint); - clientReference = new AmazonS3Reference(buildClient(clientSettings)); + final AmazonS3Reference clientReference = new AmazonS3Reference(buildClient(clientSettings)); clientReference.incRef(); - clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientName, clientReference).immutableMap(); + clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap(); return clientReference; } } + /** + * Either fetches {@link S3ClientSettings} for a given {@link RepositoryMetaData} from cached settings or creates them + * by overriding static client settings from {@link #staticClientSettings} with settings found in the repository metadata. + * @param repositoryMetaData Repository Metadata + * @return S3ClientSettings + */ + private S3ClientSettings settings(RepositoryMetaData repositoryMetaData) { + final String clientName = S3Repository.CLIENT_NAME.get(repositoryMetaData.settings()); + final S3ClientSettings staticSettings = staticClientSettings.get(clientName); + if (staticSettings != null) { + { + final S3ClientSettings existing = derivedClientSettings.getOrDefault(staticSettings, emptyMap()).get(repositoryMetaData); + if (existing != null) { + return existing; + } + } + synchronized (this) { + final Map derivedSettings = + derivedClientSettings.getOrDefault(staticSettings, emptyMap()); + final S3ClientSettings existing = derivedSettings.get(repositoryMetaData); + if (existing != null) { + return existing; + } + final S3ClientSettings newSettings = staticSettings.refine(repositoryMetaData); + derivedClientSettings = MapBuilder.newMapBuilder(derivedClientSettings).put( + staticSettings, MapBuilder.newMapBuilder(derivedSettings).put(repositoryMetaData, newSettings).immutableMap() + ).immutableMap(); + return newSettings; + } + } + throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " + + Strings.collectionToDelimitedString(staticClientSettings.keySet(), ",")); + } + // proxy for testing AmazonS3 buildClient(final S3ClientSettings clientSettings) { final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); @@ -141,17 +183,17 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { - final AWSCredentials credentials = clientSettings.credentials; + final S3BasicCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using instance profile credentials"); return new PrivilegedInstanceProfileCredentialsProvider(); } else { logger.debug("Using basic key/secret credentials"); - return new StaticCredentialsProvider(credentials); + return new AWSStaticCredentialsProvider(credentials); } } - protected synchronized void releaseCachedClients() { + private synchronized void releaseCachedClients() { // the clients will shutdown when they will not be used anymore for (final AmazonS3Reference clientReference : clientsCache.values()) { clientReference.decRef(); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index 0c14f44d8b613..2f5982b77ce62 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -22,7 +22,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; @@ -61,7 +61,7 @@ public void testAWSCredentialsFromKeystore() { final String clientName = clientNamePrefix + i; final S3ClientSettings someClientSettings = allClientsSettings.get(clientName); final AWSCredentialsProvider credentialsProvider = S3Service.buildCredentials(logger, someClientSettings); - assertThat(credentialsProvider, instanceOf(StaticCredentialsProvider.class)); + assertThat(credentialsProvider, instanceOf(AWSStaticCredentialsProvider.class)); assertThat(credentialsProvider.getCredentials().getAWSAccessKeyId(), is(clientName + "_aws_access_key")); assertThat(credentialsProvider.getCredentials().getAWSSecretKey(), is(clientName + "_aws_secret_key")); } @@ -83,7 +83,7 @@ public void testSetDefaultCredential() { // test default exists and is an Instance provider final S3ClientSettings defaultClientSettings = allClientsSettings.get("default"); final AWSCredentialsProvider defaultCredentialsProvider = S3Service.buildCredentials(logger, defaultClientSettings); - assertThat(defaultCredentialsProvider, instanceOf(StaticCredentialsProvider.class)); + assertThat(defaultCredentialsProvider, instanceOf(AWSStaticCredentialsProvider.class)); assertThat(defaultCredentialsProvider.getCredentials().getAWSAccessKeyId(), is(awsAccessKey)); assertThat(defaultCredentialsProvider.getCredentials().getAWSSecretKey(), is(awsSecretKey)); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index b4c2f81a3f8b8..739452dc178c4 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -59,7 +59,6 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa private static final ConcurrentMap blobs = new ConcurrentHashMap<>(); private static String bucket; - private static String client; private static ByteSizeValue bufferSize; private static boolean serverSideEncryption; private static String cannedACL; @@ -68,7 +67,6 @@ public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCa @BeforeClass public static void setUpRepositorySettings() { bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - client = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); bufferSize = new ByteSizeValue(randomIntBetween(5, 50), ByteSizeUnit.MB); serverSideEncryption = randomBoolean(); if (randomBoolean()) { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java index a44ad706b2361..6640a0dc4d71c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java @@ -22,8 +22,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.repositories.ESBlobStoreTestCase; @@ -114,15 +116,14 @@ public static S3BlobStore randomMockS3BlobStore() { storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); } - final String theClientName = randomAlphaOfLength(4); final AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass); final S3Service service = new S3Service() { @Override - public synchronized AmazonS3Reference client(String clientName) { - assert theClientName.equals(clientName); + public synchronized AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { return new AmazonS3Reference(client); } }; - return new S3BlobStore(service, theClientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); + return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, + new RepositoryMetaData(bucket, "s3", Settings.EMPTY)); } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index e629f43f8a3d3..53740672df329 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -21,8 +21,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -103,7 +102,7 @@ public void testCredentialsTypeWithAccessKeyAndSecretKey() { secureSettings.setString("s3.client.default.secret_key", "secret_key"); final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); final S3ClientSettings defaultSettings = settings.get("default"); - BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials; + S3BasicCredentials credentials = defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); assertThat(credentials.getAWSSecretKey(), is("secret_key")); } @@ -115,9 +114,34 @@ public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { secureSettings.setString("s3.client.default.session_token", "session_token"); final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); final S3ClientSettings defaultSettings = settings.get("default"); - BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials; + S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) defaultSettings.credentials; assertThat(credentials.getAWSAccessKeyId(), is("access_key")); assertThat(credentials.getAWSSecretKey(), is("secret_key")); assertThat(credentials.getSessionToken(), is("session_token")); } + + public void testRefineWithRepoSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + secureSettings.setString("s3.client.default.session_token", "session_token"); + final S3ClientSettings baseSettings = S3ClientSettings.load( + Settings.builder().setSecureSettings(secureSettings).build()).get("default"); + + { + final S3ClientSettings refinedSettings = baseSettings.refine(new RepositoryMetaData("name", "type", Settings.EMPTY)); + assertTrue(refinedSettings == baseSettings); + } + + { + final String endpoint = "some.host"; + final S3ClientSettings refinedSettings = baseSettings.refine(new RepositoryMetaData("name", "type", + Settings.builder().put("endpoint", endpoint).build())); + assertThat(refinedSettings.endpoint, is(endpoint)); + S3BasicSessionCredentials credentials = (S3BasicSessionCredentials) refinedSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + assertThat(credentials.getSessionToken(), is("session_token")); + } + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 60ff802c4fc02..36fa8b684bbb9 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import java.util.Collections; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -49,13 +48,12 @@ public void shutdown() { private static class DummyS3Service extends S3Service { @Override - public AmazonS3Reference client(String clientName) { + public AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { return new AmazonS3Reference(new DummyS3Client()); } @Override - public Map refreshAndClearCache(Map clientsSettings) { - return Collections.emptyMap(); + public void refreshAndClearCache(Map clientsSettings) { } @Override diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index 1c64b66be3ad2..97bb36163b11d 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -29,6 +29,86 @@ setup: snapshot: snapshot-two ignore: 404 +--- +"Try to create repository with broken endpoint override and named client": + + # Register repository with broken endpoint setting + - do: + catch: /repository_verification_exception/ + snapshot.create_repository: + repository: repository_broken + body: + type: s3 + settings: + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: "${permanent_base_path}" + endpoint: 127.0.0.1:5 + canned_acl: private + storage_class: standard + + # Turn of verification to be able to create the repo with broken endpoint setting + - do: + snapshot.create_repository: + verify: false + repository: repository_broken + body: + type: s3 + settings: + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: "${permanent_base_path}" + endpoint: 127.0.0.1:5 + canned_acl: private + storage_class: standard + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Creating snapshot with broken repo should fail + - do: + catch: /repository_exception/ + snapshot.create: + repository: repository_broken + snapshot: snapshot-one + wait_for_completion: true + + # Creating snapshot with existing working repository should work + - do: + snapshot.create: + repository: repository_permanent + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state: SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed: 0 } + --- "Snapshot and Restore with repository-s3 using permanent credentials": From 99129d7786229ab2f537784ba148740a3053eb70 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Wed, 30 Jan 2019 17:51:11 +1100 Subject: [PATCH 19/35] Fix exit code for Security CLI tools (#37956) The certgen, certutil and saml-metadata tools did not correctly return their exit code to the calling shell. These commands now explicitly exit with the code that was returned from the main(args, terminal) method. --- .../packaging/test/ArchiveTestCase.java | 16 +++++++++------- .../security/cli/CertificateGenerateTool.java | 3 +-- .../xpack/security/cli/CertificateTool.java | 2 +- .../security/authc/saml/SamlMetadataCommand.java | 2 +- .../test/resources/packaging/tests/certgen.bash | 9 +++++++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 7f2299d1fc3ea..6261cd62a2b12 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -287,15 +287,17 @@ public void test90SecurityCliPackaging() { if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) { assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); - Platforms.onLinux(() -> { - final Result result = sh.run(bin.elasticsearchCertutil + " help"); + final Platforms.PlatformAction action = () -> { + Result result = sh.run(bin.elasticsearchCertutil + " --help"); assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); - }); - Platforms.onWindows(() -> { - final Result result = sh.run(bin.elasticsearchCertutil + " help"); - assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); - }); + // Ensure that the exit code from the java command is passed back up through the shell script + result = sh.runIgnoreExitCode(bin.elasticsearchCertutil + " invalid-command"); + assertThat(result.exitCode, is(64)); + assertThat(result.stdout, containsString("Unknown command [invalid-command]")); + }; + Platforms.onLinux(action); + Platforms.onWindows(action); } else if (distribution().equals(Distribution.OSS_LINUX) || distribution().equals(Distribution.OSS_WINDOWS)) { assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli"))); } diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java index 809e4a6d30524..4b30224dcd481 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateGenerateTool.java @@ -38,7 +38,6 @@ import org.elasticsearch.xpack.core.ssl.PemUtils; import javax.security.auth.x500.X500Principal; - import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -154,7 +153,7 @@ private static class InputFileParser { } public static void main(String[] args) throws Exception { - new CertificateGenerateTool().main(args, Terminal.DEFAULT); + exit(new CertificateGenerateTool().main(args, Terminal.DEFAULT)); } @Override diff --git a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java index a966cac9109d9..435305b8a6914 100644 --- a/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java +++ b/x-pack/plugin/security/cli/src/main/java/org/elasticsearch/xpack/security/cli/CertificateTool.java @@ -134,7 +134,7 @@ private static class CertificateToolParser { public static void main(String[] args) throws Exception { - new CertificateTool().main(args, Terminal.DEFAULT); + exit(new CertificateTool().main(args, Terminal.DEFAULT)); } CertificateTool() { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java index 6fa59269ac7e6..a60b2204095a5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java @@ -90,7 +90,7 @@ public class SamlMetadataCommand extends EnvironmentAwareCommand { private KeyStoreWrapper keyStoreWrapper; public static void main(String[] args) throws Exception { - new SamlMetadataCommand().main(args, Terminal.DEFAULT); + exit(new SamlMetadataCommand().main(args, Terminal.DEFAULT)); } public SamlMetadataCommand() { diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash index dd41b93ea6b28..83f967c39891b 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash +++ b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash @@ -417,3 +417,12 @@ DATA_SETTINGS echo "$testSearch" | grep '"_index":"books"' echo "$testSearch" | grep '"_id":"0"' } + +@test "[$GROUP] exit code on failure" { + run sudo -E -u $MASTER_USER "$MASTER_HOME/bin/elasticsearch-certgen" --not-a-valid-option + [ "$status" -ne 0 ] || { + echo "Expected elasticsearch-certgen tool exit code to be non-zero" + echo "$output" + false + } +} From f51bc00fcf18fa0073641df7cf1caff7e68a9104 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 07:58:26 +0100 Subject: [PATCH 20/35] Added ccr to xpack usage infrastructure (#37256) * Added ccr to xpack usage infrastructure Closes #37221 --- docs/reference/rest-api/info.asciidoc | 5 + .../elasticsearch/xpack/ccr/XPackUsageIT.java | 85 +++++++++ .../xpack/ccr/ESCCRRestTestCase.java | 16 ++ .../java/org/elasticsearch/xpack/ccr/Ccr.java | 13 ++ .../xpack/ccr/CCRFeatureSetTests.java | 123 +++++++++++++ .../xpack/ccr/CCRFeatureSetUsageTests.java | 25 +++ .../xpack/core/XPackClientPlugin.java | 2 + .../elasticsearch/xpack/core/XPackField.java | 2 + .../xpack/core/ccr/CCRFeatureSet.java | 174 ++++++++++++++++++ 9 files changed, 445 insertions(+) create mode 100644 x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index f02919cb39660..382b4ab78ff77 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -63,6 +63,11 @@ Example response: "expiry_date_in_millis" : 1542665112332 }, "features" : { + "ccr" : { + "description" : "Cross Cluster Replication", + "available" : true, + "enabled" : true + }, "graph" : { "description" : "Graph Data Exploration for the Elastic Stack", "available" : true, diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java new file mode 100644 index 0000000000000..84271ce0acaf1 --- /dev/null +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; + +public class XPackUsageIT extends ESCCRRestTestCase { + + public void testXPackCcrUsage() throws Exception { + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } + + Map previousUsage = getCcrUsage(); + putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*"); + + // This index should be auto followed: + createLeaderIndex("messages-20200101"); + // This index will be followed manually + createLeaderIndex("my_index"); + followIndex("my_index", "my_index"); + + int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count"); + int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count"); + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1)); + assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0)); + }); + + deleteAutoFollowPattern("my_pattern"); + pauseFollow("messages-20200101"); + closeIndex("messages-20200101"); + unfollow("messages-20200101"); + + pauseFollow("my_index"); + closeIndex("my_index"); + unfollow("my_index"); + + assertBusy(() -> { + Map ccrUsage = getCcrUsage(); + assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount)); + assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount)); + if (previousFollowerIndicesCount == 0) { + assertThat(ccrUsage.get("last_follow_time_in_millis"), nullValue()); + } else { + assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0)); + } + }); + } + + private void createLeaderIndex(String indexName) throws IOException { + try (RestClient leaderClient = buildLeaderClient()) { + Settings settings = Settings.builder() + .put("index.soft_deletes.enabled", true) + .build(); + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}"); + assertOK(leaderClient.performRequest(request)); + } + } + + private Map getCcrUsage() throws IOException { + Request request = new Request("GET", "/_xpack/usage"); + Map response = toMap(client().performRequest(request)); + logger.info("xpack usage response={}", response); + return (Map) response.get("ccr"); + } + +} diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 6cdb6b37961ff..656328d5ead9e 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -87,6 +87,22 @@ protected static void pauseFollow(RestClient client, String followIndex) throws assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow"))); } + protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException { + Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName); + putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" + + remoteCluster + "\"}"); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void deleteAutoFollowPattern(String patternName) throws IOException { + Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName); + assertOK(client().performRequest(putPatternRequest)); + } + + protected static void unfollow(String followIndex) throws IOException { + assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow"))); + } + protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException { verifyDocuments(index, expectedNumDocs, query, adminClient()); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 1345faaa5bcb0..a7fa69e7abd39 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -81,6 +82,7 @@ import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction; import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus; import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction; @@ -126,6 +128,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final SetOnce restoreSourceService = new SetOnce<>(); private final SetOnce ccrSettings = new SetOnce<>(); private Client client; + private final boolean transportClientMode; /** * Construct an instance of the CCR container with the specified settings. @@ -147,6 +150,7 @@ public Ccr(final Settings settings) { this.settings = settings; this.enabled = CCR_ENABLED_SETTING.get(settings); this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); + this.transportClientMode = XPackPlugin.transportClientMode(settings); } @Override @@ -314,6 +318,15 @@ public void onIndexModule(IndexModule indexModule) { } } + @Override + public Collection createGuiceModules() { + if (transportClientMode) { + return Collections.emptyList(); + } + + return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class)); + } + protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } @Override diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java new file mode 100644 index 0000000000000..b95e8fc7c4008 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class CCRFeatureSetTests extends ESTestCase { + + private XPackLicenseState licenseState; + private ClusterService clusterService; + + @Before + public void init() throws Exception { + licenseState = mock(XPackLicenseState.class); + clusterService = mock(ClusterService.class); + } + + public void testAvailable() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + + when(licenseState.isCcrAllowed()).thenReturn(false); + assertThat(featureSet.available(), equalTo(false)); + + when(licenseState.isCcrAllowed()).thenReturn(true); + assertThat(featureSet.available(), equalTo(true)); + + featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService); + assertThat(featureSet.available(), equalTo(false)); + } + + public void testEnabled() { + Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(false)); + + settings = Settings.builder().put("xpack.ccr.enabled", true); + featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService); + assertThat(featureSet.enabled(), equalTo(true)); + } + + public void testName() { + CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + assertThat(featureSet.name(), equalTo("ccr")); + } + + public void testNativeCodeInfo() { + CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService); + assertNull(featureSet.nativeCodeInfo()); + } + + public void testUsageStats() throws Exception { + MetaData.Builder metaData = MetaData.builder(); + + int numFollowerIndices = randomIntBetween(0, 32); + for (int i = 0; i < numFollowerIndices; i++) { + IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i) + .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(i) + .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>()); + metaData.put(followerIndex); + } + + // Add a regular index, to check that we do not take that one into account: + IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(numFollowerIndices); + metaData.put(regularIndex); + + int numAutoFollowPatterns = randomIntBetween(0, 32); + Map patterns = new HashMap<>(numAutoFollowPatterns); + for (int i = 0; i < numAutoFollowPatterns; i++) { + AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser", + Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null); + patterns.put("pattern" + i, pattern); + } + metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap())); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build(); + Mockito.when(clusterService.state()).thenReturn(clusterState); + + PlainActionFuture future = new PlainActionFuture<>(); + CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService); + ccrFeatureSet.usage(future); + CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get(); + assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled())); + assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available())); + + assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices)); + assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L)); + assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns)); + } + +} diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java new file mode 100644 index 0000000000000..69e41ffddab43 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; + +public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected CCRFeatureSet.Usage createTestInstance() { + return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE), + randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong()); + } + + @Override + protected Writeable.Reader instanceReader() { + return CCRFeatureSet.Usage::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 1b6f128318a2a..4e3feb3c8a2fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.ccr.CCRFeatureSet; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; @@ -412,6 +413,7 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new), new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE, in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new), // ILM new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE, IndexLifecycleFeatureSetUsage::new), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 0e6888dd80d73..0c763032e22ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -33,6 +33,8 @@ public final class XPackField { public static final String ROLLUP = "rollup"; /** Name constant for the index lifecycle feature. */ public static final String INDEX_LIFECYCLE = "ilm"; + /** Name constant for the CCR feature. */ + public static final String CCR = "ccr"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java new file mode 100644 index 0000000000000..f6e9e76d448f7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ccr; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; + +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +public class CCRFeatureSet implements XPackFeatureSet { + + private final boolean enabled; + private final XPackLicenseState licenseState; + private final ClusterService clusterService; + + @Inject + public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) { + this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings); + this.licenseState = licenseState; + this.clusterService = clusterService; + } + + @Override + public String name() { + return XPackField.CCR; + } + + @Override + public String description() { + return "Cross Cluster Replication"; + } + + @Override + public boolean available() { + return licenseState != null && licenseState.isCcrAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + MetaData metaData = clusterService.state().metaData(); + + int numberOfFollowerIndices = 0; + long lastFollowerIndexCreationDate = 0L; + for (IndexMetaData imd : metaData) { + if (imd.getCustomData("ccr") != null) { + numberOfFollowerIndices++; + if (lastFollowerIndexCreationDate < imd.getCreationDate()) { + lastFollowerIndexCreationDate = imd.getCreationDate(); + } + } + } + AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE); + int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0; + + Long lastFollowTimeInMillis; + if (numberOfFollowerIndices == 0) { + // Otherwise we would return a value that makes no sense. + lastFollowTimeInMillis = null; + } else { + lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate); + } + + Usage usage = + new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis); + listener.onResponse(usage); + } + + public static class Usage extends XPackFeatureSet.Usage { + + private final int numberOfFollowerIndices; + private final int numberOfAutoFollowPatterns; + private final Long lastFollowTimeInMillis; + + public Usage(boolean available, + boolean enabled, + int numberOfFollowerIndices, + int numberOfAutoFollowPatterns, + Long lastFollowTimeInMillis) { + super(XPackField.CCR, available, enabled); + this.numberOfFollowerIndices = numberOfFollowerIndices; + this.numberOfAutoFollowPatterns = numberOfAutoFollowPatterns; + this.lastFollowTimeInMillis = lastFollowTimeInMillis; + } + + public Usage(StreamInput in) throws IOException { + super(in); + numberOfFollowerIndices = in.readVInt(); + numberOfAutoFollowPatterns = in.readVInt(); + if (in.readBoolean()) { + lastFollowTimeInMillis = in.readVLong(); + } else { + lastFollowTimeInMillis = null; + } + } + + public int getNumberOfFollowerIndices() { + return numberOfFollowerIndices; + } + + public int getNumberOfAutoFollowPatterns() { + return numberOfAutoFollowPatterns; + } + + public Long getLastFollowTimeInMillis() { + return lastFollowTimeInMillis; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(numberOfFollowerIndices); + out.writeVInt(numberOfAutoFollowPatterns); + if (lastFollowTimeInMillis != null) { + out.writeBoolean(true); + out.writeVLong(lastFollowTimeInMillis); + } else { + out.writeBoolean(false); + } + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("follower_indices_count", numberOfFollowerIndices); + builder.field("auto_follow_patterns_count", numberOfAutoFollowPatterns); + if (lastFollowTimeInMillis != null) { + builder.field("last_follow_time_in_millis", lastFollowTimeInMillis); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Usage usage = (Usage) o; + return numberOfFollowerIndices == usage.numberOfFollowerIndices && + numberOfAutoFollowPatterns == usage.numberOfAutoFollowPatterns && + Objects.equals(lastFollowTimeInMillis, usage.lastFollowTimeInMillis); + } + + @Override + public int hashCode() { + return Objects.hash(numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis); + } + } +} From 5dcc805dc9fad2fde98610145639b5faa128ec11 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 30 Jan 2019 08:45:50 +0100 Subject: [PATCH 21/35] Restore a noop _all metadata field for 6x indices (#37808) This commit restores a noop version of the AllFieldMapper that is instanciated only for indices created in 6x. We need this metadata field mapper to be present in this version in order to allow the upgrade of indices that explicitly disable _all (enabled: false). The mapping of these indices contains a reference to the _all field that we cannot remove in 7 so we'll need to keep this metadata mapper in 7x. Since indices created in 6x will not be compatible with 8, we'll remove this noop mapper in the next major version. Closes #37429 --- .../test/mixed_cluster/10_basic.yml | 6 + .../test/old_cluster/10_basic.yml | 18 ++ .../test/upgraded_cluster/10_basic.yml | 14 ++ .../TransportGetFieldMappingsIndexAction.java | 4 +- ...TransportFieldCapabilitiesIndexAction.java | 3 +- .../index/mapper/AllFieldMapper.java | 175 ++++++++++++++++++ .../index/mapper/DocumentMapper.java | 5 +- .../index/mapper/DocumentMapperParser.java | 4 +- .../elasticsearch/indices/IndicesService.java | 5 +- .../indices/mapper/MapperRegistry.java | 18 +- .../index/mapper/AllFieldMapperTests.java | 68 +++++++ .../index/query/ExistsQueryBuilderTests.java | 3 - .../indices/IndicesModuleTests.java | 53 ++++-- .../indices/IndicesServiceTests.java | 6 +- 14 files changed, 354 insertions(+), 28 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index f50a3fd9ea42d..2605836f8573c 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -67,3 +67,9 @@ field3: value - match: { hits.total: 1 } - match: { hits.hits.0._id: q3 } + +--- +"Index with _all is available": + - do: + indices.get: + index: all-index diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index ffa9e0ce2a6f8..9e06f767d4892 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -203,3 +203,21 @@ tasks.get: wait_for_completion: true task_id: $task + +--- +"Create an index with _all explicitly disabled": + - skip: + features: warnings + - do: + warnings: + - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field." + indices.create: + index: all-index + body: + mappings: + type: + _all: + enabled: false + properties: + field: + type: text diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 39c72dfd5334b..508a898e0cdb5 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -125,3 +125,17 @@ wait_for_completion: true task_id: $task_id - match: { task.headers.X-Opaque-Id: "Reindexing Again" } + +--- +"Index with _all is available": + - do: + indices.get: + index: all-index + + - do: + indices.get_mapping: + index: all-index + + - is_true: all-index.mappings._all + - match: { all-index.mappings._all.enabled: false} + diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 67b3fe048bea1..c7415391675fc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; @@ -91,7 +92,8 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) { protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) { assert shardId != null; IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - Predicate metadataFieldPredicate = indicesService::isMetaDataField; + Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated(); + Predicate metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f); Predicate fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName())); DocumentMapper mapper = indexService.mapperService().documentMapper(); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index c46933578f163..01c21544047ed 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -83,7 +83,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI for (String field : fieldNames) { MappedFieldType ft = mapperService.fullName(field); if (ft != null) { - if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) { + if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field) + || fieldPredicate.test(ft.name())) { FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable()); responseMap.put(field, fieldCap); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java new file mode 100644 index 0000000000000..1427e67c5389b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -0,0 +1,175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Noop mapper that ensures that mappings created in 6x that explicitly disable the _all field + * can be restored in this version. + * + * TODO: Remove in 8 + */ +public class AllFieldMapper extends MetadataFieldMapper { + public static final String NAME = "_all"; + public static final String CONTENT_TYPE = "_all"; + + public static class Defaults { + public static final MappedFieldType FIELD_TYPE = new AllFieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + FIELD_TYPE.setTokenized(true); + FIELD_TYPE.setName(NAME); + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends MetadataFieldMapper.Builder { + private boolean disableExplicit = false; + + public Builder(MappedFieldType existing) { + super(NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE); + builder = this; + } + + private Builder setDisableExplicit() { + this.disableExplicit = true; + return this; + } + + @Override + public AllFieldMapper build(BuilderContext context) { + return new AllFieldMapper(fieldType, context.indexSettings(), disableExplicit); + } + } + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + @Override + public MetadataFieldMapper.Builder parse(String name, Map node, + ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + if (fieldName.equals("enabled")) { + boolean enabled = XContentMapValues.nodeBooleanValue(entry.getValue(), "enabled"); + if (enabled) { + throw new IllegalArgumentException("[_all] is disabled in this version."); + } + builder.setDisableExplicit(); + iterator.remove(); + } + } + return builder; + } + + @Override + public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { + final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); + return new AllFieldMapper(indexSettings, Defaults.FIELD_TYPE, false); + } + } + + static final class AllFieldType extends StringFieldType { + AllFieldType() { + } + + protected AllFieldType(AllFieldType ref) { + super(ref); + } + + @Override + public MappedFieldType clone() { + return new AllFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchNoDocsQuery(); + } + } + + private final boolean disableExplicit; + + private AllFieldMapper(Settings indexSettings, MappedFieldType existing, boolean disableExplicit) { + this(existing.clone(), indexSettings, disableExplicit); + } + + private AllFieldMapper(MappedFieldType fieldType, Settings indexSettings, boolean disableExplicit) { + super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); + this.disableExplicit = disableExplicit; + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + super.parse(context); + } + + @Override + public void parse(ParseContext context) throws IOException { + // we parse in post parse + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + // noop mapper + return; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + if (includeDefaults || disableExplicit) { + builder.startObject(CONTENT_TYPE); + if (disableExplicit) { + builder.field("enabled", false); + } + builder.endObject(); + } + return builder; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 058cf68e8e1f4..044e65c7ec6fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -73,7 +74,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) { final String type = rootObjectMapper.name(); final DocumentMapper existingMapper = mapperService.documentMapper(type); - final Map metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers(); + final Version indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated(); + final Map metadataMapperParsers = + mapperService.mapperRegistry.getMetadataMapperParsers(indexCreatedVersion); for (Map.Entry entry : metadataMapperParsers.entrySet()) { final String name = entry.getKey(); final MetadataFieldMapper existingMetadataMapper = existingMapper == null diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index e388dd7ebcd00..db7954e9bd76a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -65,8 +65,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer this.similarityService = similarityService; this.queryShardContextSupplier = queryShardContextSupplier; this.typeParsers = mapperRegistry.getMapperParsers(); - this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(); - indexVersionCreated = indexSettings.getIndexVersionCreated(); + this.indexVersionCreated = indexSettings.getIndexVersionCreated(); + this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated); } public Mapper.TypeParser.ParserContext parserContext(String type) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index fa42776403dca..cca63c015f1c7 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; @@ -1382,8 +1383,8 @@ public Function> getFieldFilter() { /** * Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise. */ - public boolean isMetaDataField(String field) { - return mapperRegistry.isMetaDataField(field); + public boolean isMetaDataField(Version indexCreatedVersion, String field) { + return mapperRegistry.isMetaDataField(indexCreatedVersion, field); } /** diff --git a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java index 41d563c2037ea..c79da36200e4f 100644 --- a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -19,6 +19,8 @@ package org.elasticsearch.indices.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.plugins.MapperPlugin; @@ -36,6 +38,7 @@ public final class MapperRegistry { private final Map mapperParsers; private final Map metadataMapperParsers; + private final Map metadataMapperParsers6x; private final Function> fieldFilter; @@ -43,6 +46,11 @@ public MapperRegistry(Map mapperParsers, Map metadataMapperParsers, Function> fieldFilter) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); + // add the _all field mapper for indices created in 6x + Map metadata6x = new LinkedHashMap<>(); + metadata6x.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); + metadata6x.putAll(metadataMapperParsers); + this.metadataMapperParsers6x = Collections.unmodifiableMap(metadata6x); this.fieldFilter = fieldFilter; } @@ -58,15 +66,15 @@ public Map getMapperParsers() { * Return a map of the meta mappers that have been registered. The * returned map uses the name of the field as a key. */ - public Map getMetadataMapperParsers() { - return metadataMapperParsers; + public Map getMetadataMapperParsers(Version indexCreatedVersion) { + return indexCreatedVersion.onOrAfter(Version.V_7_0_0) ? metadataMapperParsers : metadataMapperParsers6x; } /** - * Returns true if the provide field is a registered metadata field, false otherwise + * Returns true if the provided field is a registered metadata field, false otherwise */ - public boolean isMetaDataField(String field) { - return getMetadataMapperParsers().containsKey(field); + public boolean isMetaDataField(Version indexCreatedVersion, String field) { + return getMetadataMapperParsers(indexCreatedVersion).containsKey(field); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java index 62f3495ee172b..34200b51cb317 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -26,9 +28,75 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; + +import static org.hamcrest.CoreMatchers.containsString; public class AllFieldMapperTests extends ESSingleNodeTestCase { + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testAllDisabled() throws Exception { + { + final Version version = VersionUtils.randomVersionBetween(random(), + Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); + IndexService indexService = createIndex("test_6x", + Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, version) + .build() + ); + String mappingDisabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", false) + .endObject().endObject() + ); + indexService.mapperService().merge("_doc", new CompressedXContent(mappingDisabled), MergeReason.MAPPING_UPDATE); + + String mappingEnabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", true) + .endObject().endObject() + ); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingEnabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("[_all] is disabled in this version.")); + } + { + IndexService indexService = createIndex("test"); + String mappingEnabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", true) + .endObject().endObject() + ); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingEnabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mappingDisabled = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all") + .field("enabled", false) + .endObject().endObject() + ); + exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingDisabled), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mappingAll = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("_all").endObject().endObject() + ); + exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("_doc", new CompressedXContent(mappingAll), MergeReason.MAPPING_UPDATE)); + assertThat(exc.getMessage(), containsString("unsupported parameters: [_all")); + + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().endObject()); + indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals("{\"_doc\":{}}", indexService.mapperService().documentMapper("_doc").mapping().toString()); + } + } + public void testUpdateDefaultSearchAnalyzer() throws Exception { IndexService indexService = createIndex("test", Settings.builder() .put("index.analysis.analyzer.default_search.type", "custom") diff --git a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java index ad02209bf5dae..a5329856630d5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java @@ -28,7 +28,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.AbstractQueryTestCase; @@ -55,8 +54,6 @@ protected ExistsQueryBuilder doCreateTestQueryBuilder() { if (randomBoolean()) { if (randomBoolean()) { fieldPattern = fieldPattern + "*"; - } else { - fieldPattern = MetaData.ALL; } } return new ExistsQueryBuilder(fieldPattern); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 9b88c6ab8f20c..f31ac0627138e 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.indices; +import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; @@ -36,6 +38,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.Arrays; @@ -87,14 +90,36 @@ public Map getMetadataMappers() { RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + private static String[] EXPECTED_METADATA_FIELDS_6x = new String[]{AllFieldMapper.NAME, IgnoredFieldMapper.NAME, + IdFieldMapper.NAME, RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, + VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + + public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); - assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); - assertFalse(module.getMapperRegistry().getMetadataMapperParsers().isEmpty()); - Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); - int i = 0; - for (String field : metadataMapperParsers.keySet()) { - assertEquals(EXPECTED_METADATA_FIELDS[i++], field); + { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); + assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); + assertFalse(module.getMapperRegistry().getMetadataMapperParsers(version).isEmpty()); + Map metadataMapperParsers = + module.getMapperRegistry().getMetadataMapperParsers(version); + assertEquals(EXPECTED_METADATA_FIELDS_6x.length, metadataMapperParsers.size()); + int i = 0; + for (String field : metadataMapperParsers.keySet()) { + assertEquals(EXPECTED_METADATA_FIELDS_6x[i++], field); + } + } + { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT); + assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty()); + assertFalse(module.getMapperRegistry().getMetadataMapperParsers(version).isEmpty()); + Map metadataMapperParsers = + module.getMapperRegistry().getMetadataMapperParsers(version); + assertEquals(EXPECTED_METADATA_FIELDS.length, metadataMapperParsers.size()); + int i = 0; + for (String field : metadataMapperParsers.keySet()) { + assertEquals(EXPECTED_METADATA_FIELDS[i++], field); + } } } @@ -102,11 +127,15 @@ public void testBuiltinWithPlugins() { IndicesModule noPluginsModule = new IndicesModule(Collections.emptyList()); IndicesModule module = new IndicesModule(fakePlugins); MapperRegistry registry = module.getMapperRegistry(); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.V_7_0_0.minimumCompatibilityVersion()); assertThat(registry.getMapperParsers().size(), greaterThan(noPluginsModule.getMapperRegistry().getMapperParsers().size())); - assertThat(registry.getMetadataMapperParsers().size(), - greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers().size())); - Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); + assertThat(registry.getMetadataMapperParsers(version).size(), + greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers(version).size())); + Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(version); Iterator iterator = metadataMapperParsers.keySet().iterator(); + if (version.before(Version.V_7_0_0)) { + assertEquals(AllFieldMapper.NAME, iterator.next()); + } assertEquals(IgnoredFieldMapper.NAME, iterator.next()); String last = null; while(iterator.hasNext()) { @@ -187,13 +216,15 @@ public Map getMetadataMappers() { public void testFieldNamesIsLast() { IndicesModule module = new IndicesModule(Collections.emptyList()); - List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet()); + Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT); + List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet()); assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); } public void testFieldNamesIsLastWithPlugins() { IndicesModule module = new IndicesModule(fakePlugins); - List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet()); + Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT); + List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers(version).keySet()); assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1)); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index b68ec6d0598e5..60dbad99795f3 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.hamcrest.RegexMatcher; import java.io.IOException; @@ -515,9 +516,10 @@ public void testStatsByShardDoesNotDieFromExpectedExceptions() { public void testIsMetaDataField() { IndicesService indicesService = getIndicesService(); - assertFalse(indicesService.isMetaDataField(randomAlphaOfLengthBetween(10, 15))); + final Version randVersion = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); + assertFalse(indicesService.isMetaDataField(randVersion, randomAlphaOfLengthBetween(10, 15))); for (String builtIn : IndicesModule.getBuiltInMetaDataFields()) { - assertTrue(indicesService.isMetaDataField(builtIn)); + assertTrue(indicesService.isMetaDataField(randVersion, builtIn)); } } From 2732bb5cf3d854ab404a5ebe44a2f116cc3f1073 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 30 Jan 2019 08:46:14 +0100 Subject: [PATCH 22/35] Fix fetch source option in expand search phase (#37908) This change fixes the copy of the fetch source option into the expand search request that is used to retrieve the documents of each collapsed group. Closes #23829 --- .../org/elasticsearch/action/search/ExpandSearchPhase.java | 3 ++- .../elasticsearch/action/search/ExpandSearchPhaseTests.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 10a85b723166c..afc81b21da4d5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -129,7 +129,8 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio options.getSorts().forEach(groupSource::sort); } if (options.getFetchSourceContext() != null) { - if (options.getFetchSourceContext().includes() == null && options.getFetchSourceContext().excludes() == null) { + if (options.getFetchSourceContext().includes().length == 0 && + options.getFetchSourceContext().excludes().length == 0) { groupSource.fetchSource(options.getFetchSourceContext().fetchSource()); } else { groupSource.fetchSource(options.getFetchSourceContext().includes(), diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 328950e4f3569..d9de69a1c6c62 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -252,6 +252,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().fetchSource() == false)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().includes().length == 0)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().excludes().length == 0)); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() @@ -259,6 +262,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL new CollapseBuilder("someField") .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm)) ) + .fetchSource(false) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") .routing("baz"); From 4da7a446489674beff665a3093037fdb25f93787 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 09:37:41 +0100 Subject: [PATCH 23/35] Documented default values for index follow request parameters. (#37917) --- .../ccr/apis/follow-request-body.asciidoc | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index 7215cc01302a1..ec384deee8b81 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -41,4 +43,62 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again \ No newline at end of file + immediately attempt to read from the leader again + +===== Default values + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +[source,js] +-------------------------------------------------- +GET /follower_index/_ccr/info?filter_path=follower_indices.parameters +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +The following output from the follow info api describes all the default +values for the above described index follow request parameters: + +[source,js] +-------------------------------------------------- +{ + "follower_indices" : [ + { + "parameters" : { + "max_read_request_operation_count" : 5120, + "max_read_request_size" : "32mb", + "max_outstanding_read_requests" : 12, + "max_write_request_operation_count" : 5120, + "max_write_request_size" : "9223372036854775807b", + "max_outstanding_write_requests" : 9, + "max_write_buffer_count" : 2147483647, + "max_write_buffer_size" : "512mb", + "max_retry_delay" : "500ms", + "read_poll_timeout" : "1m" + } + } + ] +} + +-------------------------------------------------- +// TESTRESPONSE \ No newline at end of file From 4dee3f741846ee8e7ccc59c09c550e0c88f10039 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Wed, 30 Jan 2019 20:13:48 +1100 Subject: [PATCH 24/35] Add classifier to tar.gz in docker compose (#38011) The distribution now includes a platform specific classifier that the docker build wasn't taking into account. Relates: #37881 --- distribution/docker/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 8a5cea936bbcd..e4d27da1f1fb0 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -18,8 +18,9 @@ dependencies { } ext.expansions = { oss -> + String classifier = 'linux-x86_64' return [ - 'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz", + 'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz", 'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz', 'jdkVersion' : '11.0.2', 'license': oss ? 'Apache-2.0' : 'Elastic License', From b63b50b945d4c44c70d56a9ce2ba9a25dd897147 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:28:24 +0100 Subject: [PATCH 25/35] Give precedence to index creation when mixing typed templates with typeless index creation and vice-versa. (#37871) Currently if you mix typed templates and typeless index creation or typeless templates and typed index creation then you will end up with an error because Elasticsearch tries to create an index that has multiple types: `_doc` and the explicit type name that you used. This commit proposes to give precedence to the index creation call so that the type from the template will be ignored if the index creation call is typeless while the template is typed, and the type from the index creation call will be used if there is a typeless template. This is consistent with the fact that index creation already "wins" if a field is defined differently in the index creation call and in a template: the definition from the index creation call is used in such cases. Closes #37773 --- .../mapping/removal_of_types.asciidoc | 74 ++++++++++ .../20_mix_typeless_typeful.yml | 137 ++++++++++++++++++ .../metadata/MetaDataCreateIndexService.java | 22 +++ .../metadata/IndexCreationTaskTests.java | 26 ++++ 4 files changed, 259 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index ee5ee4b4fe664..b9066a4c7af49 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -534,3 +534,77 @@ PUT index/_doc/1 The <>, <>, <> and <> APIs will continue to return a `_type` key in the response in 7.0, but it is considered deprecated and will be removed in 8.0. + +[float] +=== Index templates + +It is recommended to make index templates typeless before upgrading to 7.0 by +re-adding them with `include_type_name` set to `false`. + +In case typeless templates are used with typed index creation calls or typed +templates are used with typeless index creation calls, the template will still +be applied but the index creation call decides whether there should be a type +or not. For instance in the below example, `index-1-01` will have a type in +spite of the fact that it matches a template that is typeless, and `index-2-01` +will be typeless in spite of the fact that it matches a template that defines +a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from +the template that they match. + +[source,js] +-------------------------------------------------- +PUT _template/template1 +{ + "index_patterns":[ "index-1-*" ], + "mappings": { + "properties": { + "foo": { + "type": "keyword" + } + } + } +} + +PUT _template/template2?include_type_name=true +{ + "index_patterns":[ "index-2-*" ], + "mappings": { + "type": { + "properties": { + "foo": { + "type": "keyword" + } + } + } + } +} + +PUT index-1-01?include_type_name=true +{ + "mappings": { + "type": { + "properties": { + "bar": { + "type": "long" + } + } + } + } +} + +PUT index-2-01 +{ + "mappings": { + "properties": { + "bar": { + "type": "long" + } + } + } +} +-------------------------------------------------- +// CONSOLE + +In case of implicit index creation, because of documents that get indexed in +an index that doesn't exist yet, the template is always honored. This is +usually not a problem due to the fact that typless index calls work on typed +indices. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..d196d5e5dd8c2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml @@ -0,0 +1,137 @@ +--- +"Create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: false + index: test-1 + body: + mappings: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings._doc # the index creation call won + - is_false: test-1.mappings.my_type + - is_true: test-1.mappings._doc.properties.foo + - is_true: test-1.mappings._doc.properties.bar + +--- +"Create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: true + index: test-1 + body: + mappings: + my_type: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the index creation call won + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar + +--- +"Implicitly create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + catch: /the final mapping would have more than 1 type/ + index: + index: test-1 + type: my_type + body: { bar: 42 } + +--- +"Implicitly create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs typeless index operations to work on typed indices + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + index: + index: test-1 + type: my_type + body: { bar: 42 } + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the template is honored + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d9120342cf4cd..838b1e2547204 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -318,6 +318,28 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(xContentRegistry, mappingString)); + } else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) { + // Typeless template with typed mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map.Entry> mappingEntry = mappings.entrySet().iterator().next(); + templateMapping = Collections.singletonMap( + mappingEntry.getKey(), // reuse type name from the mapping + templateMapping.values().iterator().next()); // but actual mappings from the template + XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping); + } else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + // Typed template with typeless mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME); + templateMapping = Collections.singletonMap( + MapperService.SINGLE_MAPPING_NAME, // make template mapping typeless + templateMapping.values().iterator().next()); + XContentHelper.mergeDefaults(mapping, templateMapping); } else { mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, mappingString)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 518a60ffe38f6..f2a87d09eb1f2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -310,6 +310,32 @@ public void testWriteIndexValidationException() throws Exception { assertThat(exception.getMessage(), startsWith("alias [alias1] has more than one write index [")); } + public void testTypelessTemplateWithTypedIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + setupRequestMapping(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent("{\"_doc\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + + public void testTypedTemplateWithTypelessIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + setupRequestMapping("type", new CompressedXContent("{\"type\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypedTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypelessTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) { final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class); From c8af0f4bfafb1ec15a59b91b0e04f157a503d615 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:31:51 +0100 Subject: [PATCH 26/35] Use mappings to format doc-value fields by default. (#30831) Doc-value fields now return a value that is based on the mappings rather than the script implementation by default. This deprecates the special `use_field_mapping` docvalue format which was added in #29639 only to ease the transition to 7.x and it is not necessary anymore in 7.0. --- .../migration/migrate_7_0/search.asciidoc | 10 ++++ .../search/request/docvalue-fields.asciidoc | 23 +++++---- .../search/request/inner-hits.asciidoc | 5 +- .../sql/endpoints/translate.asciidoc | 3 +- .../upgrades/QueryBuilderBWCIT.java | 2 +- .../test/search.inner_hits/10_basic.yml | 8 ++-- .../test/search/10_source_filtering.yml | 28 +++++------ .../rest-api-spec/test/search/30_limits.yml | 13 ++--- .../fetch/subphase/DocValueFieldsContext.java | 2 - .../subphase/DocValueFieldsFetchSubPhase.java | 47 +++++++------------ .../index/query/InnerHitBuilderTests.java | 7 +-- .../search/fields/SearchFieldsIT.java | 24 ++++------ .../extractor/fields/ExtractedField.java | 3 +- .../persistence/DatafeedConfigProvider.java | 5 +- .../ml/job/persistence/JobConfigProvider.java | 13 +++-- .../extractor/fields/ExtractedFieldTests.java | 3 +- .../fields/ExtractedFieldsTests.java | 3 +- .../fields/TimeBasedExtractedFieldsTests.java | 3 +- .../sql/qa/single_node/CliExplainIT.java | 1 - .../sql/action/SqlTranslateResponseTests.java | 3 +- .../execution/search/SqlSourceBuilder.java | 4 +- .../querydsl/container/QueryContainer.java | 3 +- .../xpack/sql/action/SqlLicenseIT.java | 2 +- .../sql/action/SqlTranslateActionIT.java | 2 +- .../container/QueryContainerTests.java | 3 +- .../sql/querydsl/query/BoolQueryTests.java | 10 ++-- .../sql/querydsl/query/LeafQueryTests.java | 4 +- .../sql/querydsl/query/NestedQueryTests.java | 17 ++++--- .../rest-api-spec/test/sql/translate.yml | 6 +-- 29 files changed, 105 insertions(+), 152 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc index 67adf9363406c..0f3dcf9771c3d 100644 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ b/docs/reference/migration/migrate_7_0/search.asciidoc @@ -122,6 +122,16 @@ using the "all fields" mode ("default_field": "*") or other fieldname expansions Search requests with extra content after the main object will no longer be accepted by the `_search` endpoint. A parsing exception will be thrown instead. +[float] +==== Doc-value fields default format + +The format of doc-value fields is changing to be the same as what could be +obtained in 6.x with the special `use_field_mapping` format. This is mostly a +change for date fields, which are now formatted based on the format that is +configured in the mappings by default. This behavior can be changed by +specifying a <> within the doc-value +field. + [float] ==== Context Completion Suggester diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc index bcfcb20d1d53b..6697b5bb3e383 100644 --- a/docs/reference/search/request/docvalue-fields.asciidoc +++ b/docs/reference/search/request/docvalue-fields.asciidoc @@ -12,9 +12,9 @@ GET /_search "match_all": {} }, "docvalue_fields" : [ + "my_ip_field", <1> { - "field": "my_ip_field", <1> - "format": "use_field_mapping" <2> + "field": "my_keyword_field" <2> }, { "field": "my_date_field", @@ -25,10 +25,10 @@ GET /_search -------------------------------------------------- // CONSOLE <1> the name of the field -<2> the special `use_field_mapping` format tells Elasticsearch to use the format from the mapping -<3> date fields may use a custom format +<2> an object notation is supported as well +<3> the object notation allows to specify a custom format -Doc value fields can work on fields that are not stored. +Doc value fields can work on fields that have doc-values enabled, regardless of whether they are stored `*` can be used as a wild card, for example: @@ -41,8 +41,8 @@ GET /_search }, "docvalue_fields" : [ { - "field": "*field", <1> - "format": "use_field_mapping" <2> + "field": "*_date_field", <1> + "format": "epoch_millis" <2> } ] } @@ -62,9 +62,8 @@ While most fields do not support custom formats, some of them do: - <> fields can take any <>. - <> fields accept a https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html[DecimalFormat pattern]. -All fields support the special `use_field_mapping` format, which tells -Elasticsearch to use the mappings to figure out a default format. +By default fields are formatted based on a sensible configuration that depends +on their mappings: `long`, `double` and other numeric fields are formatted as +numbers, `keyword` fields are formatted as strings, `date` fields are formatted +with the configured `date` format, etc. -NOTE: The default is currently to return the same output as -<>. However it will change in 7.0 -to behave as if the `use_field_mapping` format was provided. diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index b287b1609703e..7774e34c2c00d 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -246,10 +246,7 @@ POST test/_search "inner_hits": { "_source" : false, "docvalue_fields" : [ - { - "field": "comments.text.keyword", - "format": "use_field_mapping" - } + "comments.text.keyword" ] } } diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index 2bb7bfae4d225..de9d8adbeab0b 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -27,8 +27,7 @@ Which returns: "size" : 10, "docvalue_fields" : [ { - "field": "page_count", - "format": "use_field_mapping" + "field": "page_count" }, { "field": "release_date", diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index 10bdcc234c656..f22b1b44c0763 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -201,7 +201,7 @@ public void testQueryBuilderBWC() throws Exception { QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1]; Request request = new Request("GET", "/" + index + "/_search"); request.setJsonEntity("{\"query\": {\"ids\": {\"values\": [\"" + Integer.toString(i) + "\"]}}, " + - "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\", \"format\":\"use_field_mapping\"}]}"); + "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\"}]}"); Response rsp = client().performRequest(request); assertEquals(200, rsp.getStatusLine().getStatusCode()); Map hitRsp = (Map) ((List) ((Map)toMap(rsp).get("hits")).get("hits")).get(0); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index c5bfca5c5b1d7..e0d61192bf754 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -46,8 +46,8 @@ setup: "Nested doc version and seqIDs": - skip: - version: " - 6.3.99" - reason: "object notation for docvalue_fields was introduced in 6.4" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: index: @@ -62,7 +62,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ]} }}, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } @@ -86,7 +86,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ]} }}, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index e5277c0edcbbb..18191c5ee3ac2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -144,12 +144,9 @@ setup: --- "docvalue_fields": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.9.99" + reason: Triggers a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]' search: body: docvalue_fields: [ "count" ] @@ -158,12 +155,9 @@ setup: --- "multiple docvalue_fields": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.9.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count, include.field1.keyword]' search: body: docvalue_fields: [ "count", "include.field1.keyword" ] @@ -172,12 +166,9 @@ setup: --- "docvalue_fields as url param": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.99.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count]' search: docvalue_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } @@ -185,9 +176,12 @@ setup: --- "docvalue_fields with default format": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 + version: " - 6.99.99" + reason: Only triggers warnings on 7.0+ + features: warnings - do: + warnings: + - "[use_field_mapping] is a special format that was only used to ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore." search: body: docvalue_fields: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 18a004077aeae..07a871d37e65f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -67,8 +67,8 @@ setup: "Docvalues_fields size limit": - skip: - version: " - 6.3.99" - reason: "The object notation for docvalue_fields is only supported on 6.4+" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: catch: /Trying to retrieve too many docvalue_fields\. Must be less than or equal to[:] \[2\] but was \[3\]\. This limit can be set by changing the \[index.max_docvalue_fields_search\] index level setting\./ search: @@ -78,12 +78,9 @@ setup: query: match_all: {} docvalue_fields: - - field: "one" - format: "use_field_mapping" - - field: "two" - format: "use_field_mapping" - - field: "three" - format: "use_field_mapping" + - "one" + - "two" + - "three" --- "Script_fields size limit": diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java index cf1596fd326b9..daafe0970290b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java @@ -38,8 +38,6 @@ */ public class DocValueFieldsContext { - public static final String USE_DEFAULT_FORMAT = "use_field_mapping"; - /** * Wrapper around a field name and the format that should be used to * display values of this field. diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index eae3188d865b8..e94ac0fdf6c99 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; @@ -46,7 +45,6 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * Query sub phase which pulls data from doc values @@ -55,7 +53,8 @@ */ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger( LogManager.getLogger(DocValueFieldsFetchSubPhase.class)); @Override @@ -66,9 +65,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept String name = context.collapse().getFieldName(); if (context.docValueFieldsContext() == null) { context.docValueFieldsContext(new DocValueFieldsContext( - Collections.singletonList(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + Collections.singletonList(new FieldAndFormat(name, null)))); } else if (context.docValueFieldsContext().fields().stream().map(ff -> ff.field).anyMatch(name::equals) == false) { - context.docValueFieldsContext().fields().add(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)); + context.docValueFieldsContext().fields().add(new FieldAndFormat(name, null)); } } @@ -79,13 +78,13 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); - List noFormatFields = context.docValueFieldsContext().fields().stream().filter(f -> f.format == null).map(f -> f.field) - .collect(Collectors.toList()); - if (noFormatFields.isEmpty() == false) { - deprecationLogger.deprecated("There are doc-value fields which are not using a format. The output will " - + "change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass " - + "[format={}] with a doc value field in order to opt in for the future behaviour and ease the migration to " - + "7.0: {}", DocValueFieldsContext.USE_DEFAULT_FORMAT, noFormatFields); + if (context.docValueFieldsContext().fields().stream() + .map(f -> f.format) + .filter(USE_DEFAULT_FORMAT::equals) + .findAny() + .isPresent()) { + DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " + + "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); } for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) { @@ -93,19 +92,14 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept MappedFieldType fieldType = context.mapperService().fullName(field); if (fieldType != null) { final IndexFieldData indexFieldData = context.getForField(fieldType); - final DocValueFormat format; - if (fieldAndFormat.format == null) { - format = null; - } else { - String formatDesc = fieldAndFormat.format; - if (Objects.equals(formatDesc, DocValueFieldsContext.USE_DEFAULT_FORMAT)) { - formatDesc = null; - } - format = fieldType.docValueFormat(formatDesc, null); + String formatDesc = fieldAndFormat.format; + if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) { + // TODO: Remove in 8.x + formatDesc = null; } + final DocValueFormat format = fieldType.docValueFormat(formatDesc, null); LeafReaderContext subReaderContext = null; AtomicFieldData data = null; - ScriptDocValues scriptValues = null; // legacy SortedBinaryDocValues binaryValues = null; // binary / string / ip fields SortedNumericDocValues longValues = null; // int / date fields SortedNumericDoubleValues doubleValues = null; // floating-point fields @@ -115,9 +109,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept int readerIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); data = indexFieldData.load(subReaderContext); - if (format == null) { - scriptValues = data.getLegacyFieldValues(); - } else if (indexFieldData instanceof IndexNumericFieldData) { + if (indexFieldData instanceof IndexNumericFieldData) { if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) { doubleValues = ((AtomicNumericFieldData) data).getDoubleValues(); } else { @@ -138,10 +130,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept final List values = hitField.getValues(); int subDocId = hit.docId() - subReaderContext.docBase; - if (scriptValues != null) { - scriptValues.setNextDocId(subDocId); - values.addAll(scriptValues); - } else if (binaryValues != null) { + if (binaryValues != null) { if (binaryValues.advanceExact(subDocId)) { for (int i = 0, count = binaryValues.docValueCount(); i < count; ++i) { values.add(format.format(binaryValues.nextValue())); diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 257ee807419b6..54d478a7f6aec 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; @@ -158,8 +157,7 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); } innerHits.setDocValueFields(randomListStuff(16, - () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT))); + () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))); // Random script fields deduped on their field name. Map scriptFields = new HashMap<>(); for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) { @@ -201,8 +199,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> { if (randomBoolean()) { copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(), - () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)))); } else { copy.addDocValueField(randomAlphaOfLengthBetween(1, 16)); } diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index fc69df5987aff..81315071273e0 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.fields; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -49,7 +48,6 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; import org.joda.time.format.DateTimeFormat; import java.time.ZoneOffset; @@ -804,13 +802,12 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - DateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -830,13 +827,12 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -1001,9 +997,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - - ReadableDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate.getMillis(), equalTo(date.toInstant().getMillis())); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { @@ -1065,9 +1059,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - - ReadableDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate.getMillis(), equalTo(date.toInstant().getMillis())); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java index 4223bff49825e..451f480b27830 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java @@ -7,7 +7,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import java.util.List; import java.util.Map; @@ -52,7 +51,7 @@ public ExtractionMethod getExtractionMethod() { public abstract Object[] value(SearchHit hit); public String getDocValueFormat() { - return DocValueFieldsContext.USE_DEFAULT_FORMAT; + return null; } public static ExtractedField newTimeField(String name, ExtractionMethod extractionMethod) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index d9ea6cb7c32e4..36e71de8bcba1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; @@ -198,7 +197,7 @@ public void onFailure(Exception e) { public void findDatafeedsForJobIds(Collection jobIds, ActionListener> listener) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdsQuery(jobIds)); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -366,7 +365,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index a21e6bc8c133c..9019dc2032ccf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -52,7 +52,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -424,7 +423,7 @@ public void jobIdMatches(List ids, ActionListener> listener SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -509,8 +508,8 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -554,8 +553,8 @@ private SearchRequest makeExpandIdsSearchRequest(String expression, boolean excl SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); return client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -638,7 +637,7 @@ public void expandGroupIds(List groupIds, ActionListener sourceBuilder.docValueField(field.field, - field.format == null ? DocValueFieldsContext.USE_DEFAULT_FORMAT : field.format)); + docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format)); scriptFields.forEach(sourceBuilder::scriptField); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 9a784b7b112ff..fee8d0e942a3b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.search.FieldExtraction; import org.elasticsearch.xpack.sql.execution.search.SourceGenerator; @@ -183,7 +182,7 @@ private Tuple nestedHitFieldRef(FieldAttribute List nestedRefs = new ArrayList<>(); String name = aliasName(attr); - String format = attr.field().getDataType() == DataType.DATETIME ? "epoch_millis" : DocValueFieldsContext.USE_DEFAULT_FORMAT; + String format = attr.field().getDataType() == DataType.DATETIME ? "epoch_millis" : null; Query q = rewriteToContainNestedField(query, attr.source(), attr.nestedParent().name(), name, format, attr.field().isAggregatable()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java index 50dda656ab4d1..9e56f46949ff6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java @@ -154,7 +154,7 @@ public void testSqlTranslateActionLicense() throws Exception { .query("SELECT * FROM test").get(); SearchSourceBuilder source = response.source(); assertThat(source.docValueFields(), Matchers.contains( - new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT))); + new DocValueFieldsContext.FieldAndFormat("count", null))); FetchSourceContext fetchSource = source.fetchSource(); assertThat(fetchSource.includes(), Matchers.arrayContaining("data")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index 3dc41ad9dd362..0528b9121d323 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -35,7 +35,7 @@ public void testSqlTranslateAction() { assertTrue(fetch.fetchSource()); assertArrayEquals(new String[] { "data" }, fetch.includes()); assertEquals( - singletonList(new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonList(new DocValueFieldsContext.FieldAndFormat("count", null)), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java index 5e675b0be5a72..424964bdbd9f3 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.container; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.querydsl.query.BoolQuery; import org.elasticsearch.xpack.sql.querydsl.query.MatchAll; @@ -24,7 +23,7 @@ public class QueryContainerTests extends ESTestCase { private Source source = SourceTests.randomSource(); private String path = randomAlphaOfLength(5); private String name = randomAlphaOfLength(5); - private String format = DocValueFieldsContext.USE_DEFAULT_FORMAT; + private String format = null; private boolean hasDocValues = randomBoolean(); public void testRewriteToContainNestedFieldNoQuery() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java index adc733a29c33b..ac6b85538805c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Source; @@ -53,15 +52,14 @@ public void testContainsNestedField() { public void testAddNestedField() { Query q = boolQueryWithoutNestedChildren(); - assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); String path = randomAlphaOfLength(5); String field = randomAlphaOfLength(5); q = boolQueryWithNestedChildren(path, field); String newField = randomAlphaOfLength(5); boolean hasDocValues = randomBoolean(); - Query rewritten = q.addNestedField(path, newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + Query rewritten = q.addNestedField(path, newField, null, hasDocValues); assertNotSame(q, rewritten); assertTrue(rewritten.containsNestedField(path, newField)); } @@ -87,7 +85,7 @@ private Query boolQueryWithoutNestedChildren() { private Query boolQueryWithNestedChildren(String path, String field) { NestedQuery match = new NestedQuery(SourceTests.randomSource(), path, - singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), null)), new MatchAll(SourceTests.randomSource())); Query matchAll = new MatchAll(SourceTests.randomSource()); Query left; @@ -108,4 +106,4 @@ public void testToString() { new ExistsQuery(new Source(1, 1, StringUtils.EMPTY), "f1"), new ExistsQuery(new Source(1, 7, StringUtils.EMPTY), "f2")).toString()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java index 05ef987480910..29ede49d98248 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Source; @@ -54,8 +53,7 @@ public void testContainsNestedField() { public void testAddNestedField() { Query query = new DummyLeafQuery(SourceTests.randomSource()); // Leaf queries don't contain nested fields. - assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); } public void testEnrichNestedSort() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java index 5fe69760a694d..818ba04fa1882 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; @@ -45,7 +44,7 @@ private static Map> randomFields() { int size = between(0, 5); Map> fields = new HashMap<>(size); while (fields.size() < size) { - fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)); + fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), null)); } return fields; } @@ -80,18 +79,18 @@ public void testAddNestedField() { NestedQuery q = randomNestedQuery(0); for (String field : q.fields().keySet()) { // add does nothing if the field is already there - assertSame(q, q.addNestedField(q.path(), field, DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(q.path(), field, null, randomBoolean())); String otherPath = randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)); // add does nothing if the path doesn't match - assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), null, randomBoolean())); } // if the field isn't in the list then add rewrites to a query with all the old fields and the new one String newField = randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5)); boolean hasDocValues = randomBoolean(); - NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, null, hasDocValues); assertNotSame(q, added); - assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, DocValueFieldsContext.USE_DEFAULT_FORMAT))); + assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, null))); assertTrue(added.containsNestedField(q.path(), newField)); for (Map.Entry> field : q.fields().entrySet()) { assertThat(added.fields(), hasEntry(field.getKey(), field.getValue())); @@ -133,8 +132,8 @@ public void testEnrichNestedSort() { public void testToString() { NestedQuery q = new NestedQuery(new Source(1, 1, StringUtils.EMPTY), "a.b", - singletonMap("f", new SimpleImmutableEntry<>(true, DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonMap("f", new SimpleImmutableEntry<>(true, null)), new MatchAll(new Source(1, 1, StringUtils.EMPTY))); - assertEquals("NestedQuery@1:2[a.b.{f=true=use_field_mapping}[MatchAll@1:2[]]]", q.toString()); + assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml index 0a22a189fa7b6..9fa8e6259f5ff 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml @@ -1,9 +1,8 @@ --- "Translate SQL": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 - features: warnings + version: " - 6.99.99" + reason: Triggers warnings before 7.0 - do: bulk: @@ -29,7 +28,6 @@ excludes: [] docvalue_fields: - field: int - format: use_field_mapping sort: - int: order: asc From 908c8def063e529ee4f1aa146d43132b9dafd66e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 30 Jan 2019 11:34:47 +0200 Subject: [PATCH 27/35] SQL: Skip the nested and object field types in case of an ODBC request (#37948) --- .../plan/logical/command/sys/SysColumns.java | 78 ++++++++++--------- .../logical/command/sys/SysColumnsTests.java | 61 +++++++++++++-- 2 files changed, 97 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index 76b58babe2832..b3730ee33405e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -15,8 +15,8 @@ import org.elasticsearch.xpack.sql.session.Rows; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlSession; -import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; import org.elasticsearch.xpack.sql.type.EsField; @@ -133,42 +133,46 @@ static void fillInRows(String clusterName, String indexName, Map> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, true); - assertEquals(16, rows.size()); + assertEquals(14, rows.size()); assertEquals(24, rows.get(0).size()); List row = rows.get(0); @@ -90,6 +90,16 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(3); + assertEquals("keyword", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(4); assertEquals("date", name(row)); assertEquals((short) Types.TIMESTAMP, sqlType(row)); @@ -101,17 +111,58 @@ public void testSysColumnsInOdbcMode() { assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(5); + assertEquals("unsupported", name(row)); + assertEquals((short) Types.OTHER, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(0, precision(row)); + assertEquals(0, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(6); + assertEquals("some.dotted.field", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + row = rows.get(7); - assertEquals("some.dotted", name(row)); - assertEquals((short) Types.STRUCT, sqlType(row)); + assertEquals("some.string", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); - assertEquals(-1, bufferLength(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); assertNull(decimalPrecision(row)); assertEquals(Short.class, nullable(row).getClass()); assertEquals(Short.class, sqlDataType(row).getClass()); assertEquals(Short.class, sqlDataTypeSub(row).getClass()); - row = rows.get(15); + row = rows.get(8); + assertEquals("some.string.normalized", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(9); + assertEquals("some.string.typical", name(row)); + assertEquals((short) Types.VARCHAR, sqlType(row)); + assertEquals(null, radix(row)); + assertEquals(Integer.MAX_VALUE, bufferLength(row)); + assertNull(decimalPrecision(row)); + assertEquals(Short.class, nullable(row).getClass()); + assertEquals(Short.class, sqlDataType(row).getClass()); + assertEquals(Short.class, sqlDataTypeSub(row).getClass()); + + row = rows.get(13); assertEquals("some.ambiguous.normalized", name(row)); assertEquals((short) Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); From 9ec4abc31eeaf7f60a4ff3597addcb154b07a42e Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Wed, 30 Jan 2019 10:47:12 +0100 Subject: [PATCH 28/35] Ensure date parsing BWC compatibility (#37929) In order to retain BWC this changes the java date formatters to be able to parse nanoseconds resolution, even if only milliseconds are supported. This used to work on joda time as well so that a user could store a date like `2018-10-03T14:42:44.613469+0000` and then just loose the precision on anything lower than millisecond level. --- .../common/time/DateFormatters.java | 52 +++++++++++-------- .../joda/JavaJodaTimeDuellingTests.java | 26 ++++++++++ 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 2e3c2953ec375..0b92955583f7f 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -108,9 +108,6 @@ public class DateFormatters { .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) .optionalStart() - .appendFraction(NANO_OF_SECOND, 3, 3, true) - .optionalEnd() - .optionalStart() .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .optionalEnd() @@ -205,7 +202,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter BASIC_TIME_PRINTER = new DateTimeFormatterBuilder() @@ -311,7 +308,7 @@ public class DateFormatters { private static final DateFormatter BASIC_ORDINAL_DATE_TIME = new JavaDateFormatter("basic_ordinal_date_time", new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_PRINTER) .appendZoneOrOffsetId().toFormatter(Locale.ROOT), - new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_PRINTER) + new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_FORMATTER) .appendZoneOrOffsetId().toFormatter(Locale.ROOT), new DateTimeFormatterBuilder().appendPattern("yyyyDDD").append(BASIC_T_TIME_FORMATTER) .append(TIME_ZONE_FORMATTER_NO_COLON).toFormatter(Locale.ROOT) @@ -419,7 +416,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .appendZoneOrOffsetId() .toFormatter(Locale.ROOT), new DateTimeFormatterBuilder() @@ -428,7 +425,7 @@ public class DateFormatters { .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .append(TIME_ZONE_FORMATTER_NO_COLON) .toFormatter(Locale.ROOT) ); @@ -485,7 +482,7 @@ public class DateFormatters { .appendLiteral('T') .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) .optionalStart() - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -542,7 +539,7 @@ public class DateFormatters { // NOTE: this is not a strict formatter to retain the joda time based behaviour, even though it's named like this private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_FORMATTER = new DateTimeFormatterBuilder() .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER = new DateTimeFormatterBuilder() @@ -582,8 +579,8 @@ public class DateFormatters { .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .appendLiteral("T") .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) - // this one here is lenient as well to retain joda time based bwc compatibility - .appendFraction(NANO_OF_SECOND, 1, 3, true) + // this one here is lenient as well to retain joda time based bwc compatibility + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT) ); @@ -599,7 +596,7 @@ public class DateFormatters { .appendLiteral("T") .append(STRICT_HOUR_MINUTE_SECOND_FORMATTER) // this one here is lenient as well to retain joda time based bwc compatibility - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT) ); @@ -625,7 +622,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 3, 3, true) + .appendFraction(NANO_OF_SECOND, 3, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -649,7 +646,7 @@ public class DateFormatters { .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NOT_NEGATIVE) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter STRICT_TIME_PRINTER = new DateTimeFormatterBuilder() @@ -880,7 +877,7 @@ public class DateFormatters { .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) .optionalEnd() .optionalStart() - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .optionalStart().appendZoneOrOffsetId().optionalEnd() .optionalStart().appendOffset("+HHmm", "Z").optionalEnd() @@ -904,6 +901,15 @@ public class DateFormatters { .appendFraction(NANO_OF_SECOND, 1, 3, true) .toFormatter(Locale.ROOT); + private static final DateTimeFormatter HOUR_MINUTE_SECOND_FRACTION_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE) + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE) + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) + .appendFraction(NANO_OF_SECOND, 1, 9, true) + .toFormatter(Locale.ROOT); + private static final DateTimeFormatter ORDINAL_DATE_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral('-') @@ -936,7 +942,7 @@ public class DateFormatters { private static final DateTimeFormatter TIME_PREFIX = new DateTimeFormatterBuilder() .append(TIME_NO_MILLIS_FORMATTER) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .toFormatter(Locale.ROOT); private static final DateTimeFormatter WEEK_DATE_FORMATTER = new DateTimeFormatterBuilder() @@ -974,8 +980,7 @@ public class DateFormatters { /* * Returns a formatter that combines a full date, two digit hour of day, * two digit minute of hour, two digit second of minute, and three digit - * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS). Parsing will parse up - * to 3 fractional second digits. + * fraction of second (yyyy-MM-dd'T'HH:mm:ss.SSS). */ private static final DateFormatter DATE_HOUR_MINUTE_SECOND_MILLIS = new JavaDateFormatter("date_hour_minute_second_millis", @@ -990,7 +995,8 @@ public class DateFormatters { .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER) .toFormatter(Locale.ROOT)); - private static final DateFormatter DATE_HOUR_MINUTE_SECOND_FRACTION = new JavaDateFormatter("date_hour_minute_second_fraction", + private static final DateFormatter DATE_HOUR_MINUTE_SECOND_FRACTION = + new JavaDateFormatter("date_hour_minute_second_fraction", new DateTimeFormatterBuilder() .append(STRICT_YEAR_MONTH_DAY_FORMATTER) .appendLiteral("T") @@ -999,7 +1005,7 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(DATE_FORMATTER) .appendLiteral("T") - .append(HOUR_MINUTE_SECOND_MILLIS_FORMATTER) + .append(HOUR_MINUTE_SECOND_FRACTION_FORMATTER) .toFormatter(Locale.ROOT)); /* @@ -1034,7 +1040,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); @@ -1106,7 +1112,7 @@ public class DateFormatters { STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_MILLIS_FORMATTER); private static final DateFormatter HOUR_MINUTE_SECOND_FRACTION = new JavaDateFormatter("hour_minute_second_fraction", - STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_MILLIS_FORMATTER); + STRICT_HOUR_MINUTE_SECOND_MILLIS_PRINTER, HOUR_MINUTE_SECOND_FRACTION_FORMATTER); /* * Returns a formatter for a two digit hour of day and two digit minute of @@ -1142,7 +1148,7 @@ public class DateFormatters { .optionalStart() .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE) - .appendFraction(NANO_OF_SECOND, 1, 3, true) + .appendFraction(NANO_OF_SECOND, 1, 9, true) .optionalEnd() .toFormatter(Locale.ROOT); diff --git a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java index c7abea63be081..423592d6d18d7 100644 --- a/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java +++ b/server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java @@ -97,18 +97,21 @@ public void testDuellingFormatsValidParsing() { assertSameDate("20181126T121212+0100", "basic_date_time_no_millis"); assertSameDate("2018363", "basic_ordinal_date"); assertSameDate("2018363T121212.123Z", "basic_ordinal_date_time"); + assertSameDate("2018363T121212.123456789Z", "basic_ordinal_date_time"); assertSameDate("2018363T121212.123+0100", "basic_ordinal_date_time"); assertSameDate("2018363T121212.123+01:00", "basic_ordinal_date_time"); assertSameDate("2018363T121212Z", "basic_ordinal_date_time_no_millis"); assertSameDate("2018363T121212+0100", "basic_ordinal_date_time_no_millis"); assertSameDate("2018363T121212+01:00", "basic_ordinal_date_time_no_millis"); assertSameDate("121212.123Z", "basic_time"); + assertSameDate("121212.123456789Z", "basic_time"); assertSameDate("121212.123+0100", "basic_time"); assertSameDate("121212.123+01:00", "basic_time"); assertSameDate("121212Z", "basic_time_no_millis"); assertSameDate("121212+0100", "basic_time_no_millis"); assertSameDate("121212+01:00", "basic_time_no_millis"); assertSameDate("T121212.123Z", "basic_t_time"); + assertSameDate("T121212.123456789Z", "basic_t_time"); assertSameDate("T121212.123+0100", "basic_t_time"); assertSameDate("T121212.123+01:00", "basic_t_time"); assertSameDate("T121212Z", "basic_t_time_no_millis"); @@ -118,6 +121,7 @@ public void testDuellingFormatsValidParsing() { assertSameDate("1W313", "basic_week_date"); assertSameDate("18W313", "basic_week_date"); assertSameDate("2018W313T121212.123Z", "basic_week_date_time"); + assertSameDate("2018W313T121212.123456789Z", "basic_week_date_time"); assertSameDate("2018W313T121212.123+0100", "basic_week_date_time"); assertSameDate("2018W313T121212.123+01:00", "basic_week_date_time"); assertSameDate("2018W313T121212Z", "basic_week_date_time_no_millis"); @@ -138,7 +142,9 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-12-31T12:12:1", "date_hour_minute_second"); assertSameDate("2018-12-31T12:12:12.123", "date_hour_minute_second_fraction"); + assertSameDate("2018-12-31T12:12:12.123456789", "date_hour_minute_second_fraction"); assertSameDate("2018-12-31T12:12:12.123", "date_hour_minute_second_millis"); + assertParseException("2018-12-31T12:12:12.123456789", "date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "date_hour_minute_second_fraction"); @@ -148,7 +154,9 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-05-30T20:21", "date_optional_time"); assertSameDate("2018-05-30T20:21:23", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123", "date_optional_time"); + assertSameDate("2018-05-30T20:21:23.123456789", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123Z", "date_optional_time"); + assertSameDate("2018-05-30T20:21:23.123456789Z", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123+0100", "date_optional_time"); assertSameDate("2018-05-30T20:21:23.123+01:00", "date_optional_time"); assertSameDate("2018-12-1", "date_optional_time"); @@ -158,12 +166,14 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-12-31T1:15:30", "date_optional_time"); assertSameDate("2018-12-31T10:15:30.123Z", "date_time"); + assertSameDate("2018-12-31T10:15:30.123456789Z", "date_time"); assertSameDate("2018-12-31T10:15:30.123+0100", "date_time"); assertSameDate("2018-12-31T10:15:30.123+01:00", "date_time"); assertSameDate("2018-12-31T10:15:30.11Z", "date_time"); assertSameDate("2018-12-31T10:15:30.11+0100", "date_time"); assertSameDate("2018-12-31T10:15:30.11+01:00", "date_time"); assertSameDate("2018-12-31T10:15:3.123Z", "date_time"); + assertSameDate("2018-12-31T10:15:3.123456789Z", "date_time"); assertSameDate("2018-12-31T10:15:3.123+0100", "date_time"); assertSameDate("2018-12-31T10:15:3.123+01:00", "date_time"); @@ -193,9 +203,11 @@ public void testDuellingFormatsValidParsing() { assertSameDate("12:12:1", "hour_minute_second"); assertSameDate("12:12:12.123", "hour_minute_second_fraction"); + assertSameDate("12:12:12.123456789", "hour_minute_second_fraction"); assertSameDate("12:12:12.1", "hour_minute_second_fraction"); assertParseException("12:12:12", "hour_minute_second_fraction"); assertSameDate("12:12:12.123", "hour_minute_second_millis"); + assertParseException("12:12:12.123456789", "hour_minute_second_millis"); assertSameDate("12:12:12.1", "hour_minute_second_millis"); assertParseException("12:12:12", "hour_minute_second_millis"); @@ -203,9 +215,11 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-1", "ordinal_date"); assertSameDate("2018-128T10:15:30.123Z", "ordinal_date_time"); + assertSameDate("2018-128T10:15:30.123456789Z", "ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+0100", "ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+01:00", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123Z", "ordinal_date_time"); + assertSameDate("2018-1T10:15:30.123456789Z", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123+0100", "ordinal_date_time"); assertSameDate("2018-1T10:15:30.123+01:00", "ordinal_date_time"); @@ -217,6 +231,7 @@ public void testDuellingFormatsValidParsing() { assertSameDate("2018-1T10:15:30+01:00", "ordinal_date_time_no_millis"); assertSameDate("10:15:30.123Z", "time"); + assertSameDate("10:15:30.123456789Z", "time"); assertSameDate("10:15:30.123+0100", "time"); assertSameDate("10:15:30.123+01:00", "time"); assertSameDate("1:15:30.123Z", "time"); @@ -249,6 +264,7 @@ public void testDuellingFormatsValidParsing() { assertParseException("10:15:3", "time_no_millis"); assertSameDate("T10:15:30.123Z", "t_time"); + assertSameDate("T10:15:30.123456789Z", "t_time"); assertSameDate("T10:15:30.123+0100", "t_time"); assertSameDate("T10:15:30.123+01:00", "t_time"); assertSameDate("T1:15:30.123Z", "t_time"); @@ -286,6 +302,7 @@ public void testDuellingFormatsValidParsing() { assertJavaTimeParseException("2012-W1-8", "week_date"); assertSameDate("2012-W48-6T10:15:30.123Z", "week_date_time"); + assertSameDate("2012-W48-6T10:15:30.123456789Z", "week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+0100", "week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+01:00", "week_date_time"); assertSameDate("2012-W1-6T10:15:30.123Z", "week_date_time"); @@ -326,9 +343,11 @@ public void testDuelingStrictParsing() { assertSameDate("2018W313", "strict_basic_week_date"); assertParseException("18W313", "strict_basic_week_date"); assertSameDate("2018W313T121212.123Z", "strict_basic_week_date_time"); + assertSameDate("2018W313T121212.123456789Z", "strict_basic_week_date_time"); assertSameDate("2018W313T121212.123+0100", "strict_basic_week_date_time"); assertSameDate("2018W313T121212.123+01:00", "strict_basic_week_date_time"); assertParseException("2018W313T12128.123Z", "strict_basic_week_date_time"); + assertParseException("2018W313T12128.123456789Z", "strict_basic_week_date_time"); assertParseException("2018W313T81212.123Z", "strict_basic_week_date_time"); assertParseException("2018W313T12812.123Z", "strict_basic_week_date_time"); assertParseException("2018W313T12812.1Z", "strict_basic_week_date_time"); @@ -354,6 +373,7 @@ public void testDuelingStrictParsing() { assertSameDate("2018-12-31T12:12:12", "strict_date_hour_minute_second"); assertParseException("2018-12-31T12:12:1", "strict_date_hour_minute_second"); assertSameDate("2018-12-31T12:12:12.123", "strict_date_hour_minute_second_fraction"); + assertSameDate("2018-12-31T12:12:12.123456789", "strict_date_hour_minute_second_fraction"); assertSameDate("2018-12-31T12:12:12.123", "strict_date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "strict_date_hour_minute_second_millis"); assertSameDate("2018-12-31T12:12:12.1", "strict_date_hour_minute_second_fraction"); @@ -373,6 +393,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-12-31T9:15:30", "strict_date_optional_time"); assertSameDate("2015-01-04T00:00Z", "strict_date_optional_time"); assertSameDate("2018-12-31T10:15:30.123Z", "strict_date_time"); + assertSameDate("2018-12-31T10:15:30.123456789Z", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.123+0100", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.123+01:00", "strict_date_time"); assertSameDate("2018-12-31T10:15:30.11Z", "strict_date_time"); @@ -397,6 +418,7 @@ public void testDuelingStrictParsing() { assertSameDate("12:12:01", "strict_hour_minute_second"); assertParseException("12:12:1", "strict_hour_minute_second"); assertSameDate("12:12:12.123", "strict_hour_minute_second_fraction"); + assertSameDate("12:12:12.123456789", "strict_hour_minute_second_fraction"); assertSameDate("12:12:12.1", "strict_hour_minute_second_fraction"); assertParseException("12:12:12", "strict_hour_minute_second_fraction"); assertSameDate("12:12:12.123", "strict_hour_minute_second_millis"); @@ -406,6 +428,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-1", "strict_ordinal_date"); assertSameDate("2018-128T10:15:30.123Z", "strict_ordinal_date_time"); + assertSameDate("2018-128T10:15:30.123456789Z", "strict_ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+0100", "strict_ordinal_date_time"); assertSameDate("2018-128T10:15:30.123+01:00", "strict_ordinal_date_time"); assertParseException("2018-1T10:15:30.123Z", "strict_ordinal_date_time"); @@ -416,6 +439,7 @@ public void testDuelingStrictParsing() { assertParseException("2018-1T10:15:30Z", "strict_ordinal_date_time_no_millis"); assertSameDate("10:15:30.123Z", "strict_time"); + assertSameDate("10:15:30.123456789Z", "strict_time"); assertSameDate("10:15:30.123+0100", "strict_time"); assertSameDate("10:15:30.123+01:00", "strict_time"); assertParseException("1:15:30.123Z", "strict_time"); @@ -436,6 +460,7 @@ public void testDuelingStrictParsing() { assertParseException("10:15:3", "strict_time_no_millis"); assertSameDate("T10:15:30.123Z", "strict_t_time"); + assertSameDate("T10:15:30.123456789Z", "strict_t_time"); assertSameDate("T10:15:30.123+0100", "strict_t_time"); assertSameDate("T10:15:30.123+01:00", "strict_t_time"); assertParseException("T1:15:30.123Z", "strict_t_time"); @@ -466,6 +491,7 @@ public void testDuelingStrictParsing() { assertJavaTimeParseException("2012-W01-8", "strict_week_date"); assertSameDate("2012-W48-6T10:15:30.123Z", "strict_week_date_time"); + assertSameDate("2012-W48-6T10:15:30.123456789Z", "strict_week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+0100", "strict_week_date_time"); assertSameDate("2012-W48-6T10:15:30.123+01:00", "strict_week_date_time"); assertParseException("2012-W1-6T10:15:30.123Z", "strict_week_date_time"); From 7205833f92df93bfdb09296aafade865c57af0d4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:43:57 +0100 Subject: [PATCH 29/35] Revert "Documented default values for index follow request parameters. (#37917)" This reverts commit 4da7a446489674beff665a3093037fdb25f93787. --- .../ccr/apis/follow-request-body.asciidoc | 62 +------------------ 1 file changed, 1 insertion(+), 61 deletions(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index ec384deee8b81..7215cc01302a1 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,5 +1,3 @@ -[role="xpack"] -[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -43,62 +41,4 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again - -===== Default values - -////////////////////////// - -[source,js] --------------------------------------------------- -PUT /follower_index/_ccr/follow -{ - "remote_cluster" : "remote_cluster", - "leader_index" : "leader_index" -} --------------------------------------------------- -// CONSOLE -// TESTSETUP -// TEST[setup:remote_cluster_and_leader_index] - -[source,js] --------------------------------------------------- -POST /follower_index/_ccr/pause_follow --------------------------------------------------- -// CONSOLE -// TEARDOWN - -[source,js] --------------------------------------------------- -GET /follower_index/_ccr/info?filter_path=follower_indices.parameters --------------------------------------------------- -// CONSOLE - -////////////////////////// - -The following output from the follow info api describes all the default -values for the above described index follow request parameters: - -[source,js] --------------------------------------------------- -{ - "follower_indices" : [ - { - "parameters" : { - "max_read_request_operation_count" : 5120, - "max_read_request_size" : "32mb", - "max_outstanding_read_requests" : 12, - "max_write_request_operation_count" : 5120, - "max_write_request_size" : "9223372036854775807b", - "max_outstanding_write_requests" : 9, - "max_write_buffer_count" : 2147483647, - "max_write_buffer_size" : "512mb", - "max_retry_delay" : "500ms", - "read_poll_timeout" : "1m" - } - } - ] -} - --------------------------------------------------- -// TESTRESPONSE \ No newline at end of file + immediately attempt to read from the leader again \ No newline at end of file From e959dbaa998482bee009f3064b6e3edea60301e5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 10:49:42 +0100 Subject: [PATCH 30/35] Revert "Revert "Documented default values for index follow request parameters. (#37917)"" This reverts commit 7205833f92df93bfdb09296aafade865c57af0d4. --- .../ccr/apis/follow-request-body.asciidoc | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index 7215cc01302a1..ec384deee8b81 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="platinum"] `max_read_request_operation_count`:: (integer) the maximum number of operations to pull per read from the remote cluster @@ -41,4 +43,62 @@ remote cluster when the follower index is synchronized with the leader index; when the timeout has elapsed, the poll for operations will return to the follower so that it can update some statistics, and then the follower will - immediately attempt to read from the leader again \ No newline at end of file + immediately attempt to read from the leader again + +===== Default values + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +[source,js] +-------------------------------------------------- +GET /follower_index/_ccr/info?filter_path=follower_indices.parameters +-------------------------------------------------- +// CONSOLE + +////////////////////////// + +The following output from the follow info api describes all the default +values for the above described index follow request parameters: + +[source,js] +-------------------------------------------------- +{ + "follower_indices" : [ + { + "parameters" : { + "max_read_request_operation_count" : 5120, + "max_read_request_size" : "32mb", + "max_outstanding_read_requests" : 12, + "max_write_request_operation_count" : 5120, + "max_write_request_size" : "9223372036854775807b", + "max_outstanding_write_requests" : 9, + "max_write_buffer_count" : 2147483647, + "max_write_buffer_size" : "512mb", + "max_retry_delay" : "500ms", + "read_poll_timeout" : "1m" + } + } + ] +} + +-------------------------------------------------- +// TESTRESPONSE \ No newline at end of file From 3865435a01a5b60465f37f036953c306e950a75b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 30 Jan 2019 11:02:16 +0100 Subject: [PATCH 31/35] Docs test fix, wait for shards active. (a restore needs to be complete, which happens in the background and by default the ccr put follow api doesn't wait for this) (this was a recent change and the pr that added this docs test, did not include this change) Relates to #37917 --- docs/reference/ccr/apis/follow-request-body.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index ec384deee8b81..e7e6ae2e26a05 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -51,7 +51,7 @@ [source,js] -------------------------------------------------- -PUT /follower_index/_ccr/follow +PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index" From ba285a56a72655d189ee11aeed8c9eb8d2043ff9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 30 Jan 2019 05:25:17 -0500 Subject: [PATCH 32/35] Fix limit on retaining sequence number (#37992) We only assign non-negative sequence numbers to operations, so the lower limit on retaining sequence numbers should be that it is non-negative only. --- .../java/org/elasticsearch/index/seqno/RetentionLease.java | 2 +- .../elasticsearch/index/engine/InternalEngineTests.java | 2 +- .../elasticsearch/index/engine/SoftDeletesPolicyTests.java | 2 +- .../index/seqno/ReplicationTrackerRetentionLeaseTests.java | 2 +- .../elasticsearch/index/seqno/RetentionLeaseSyncIT.java | 4 ++-- .../org/elasticsearch/index/seqno/RetentionLeaseTests.java | 7 +++---- .../index/shard/IndexShardRetentionLeaseTests.java | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java index 13e3381b11553..24d144d810d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java @@ -101,7 +101,7 @@ public RetentionLease(final String id, final long retainingSequenceNumber, final // retention lease IDs can not contain these characters because they are used in encoding retention leases throw new IllegalArgumentException("retention lease ID can not contain any of [:;,] but was [" + id + "]"); } - if (retainingSequenceNumber < SequenceNumbers.UNASSIGNED_SEQ_NO) { + if (retainingSequenceNumber < 0) { throw new IllegalArgumentException("retention lease retaining sequence number [" + retainingSequenceNumber + "] out of range"); } if (timestamp < 0) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index f57bff72fc57c..d984d1702f257 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -5313,7 +5313,7 @@ public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException { final List leases = new ArrayList<>(length); for (int i = 0; i < length; i++) { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(0L, Math.max(0L, globalCheckpoint.get())); + final long retainingSequenceNumber = randomLongBetween(0, Math.max(0, globalCheckpoint.get())); final long timestamp = randomLongBetween(0L, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source)); diff --git a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java index 310e83e9d2cef..e15372d687e55 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java @@ -49,7 +49,7 @@ public void testSoftDeletesRetentionLock() { AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong[] retainingSequenceNumbers = new AtomicLong[randomIntBetween(0, 8)]; for (int i = 0; i < retainingSequenceNumbers.length; i++) { - retainingSequenceNumbers[i] = new AtomicLong(SequenceNumbers.UNASSIGNED_SEQ_NO); + retainingSequenceNumbers[i] = new AtomicLong(); } final Supplier> retentionLeasesSupplier = () -> { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index 7a867027412e1..90eb162374469 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -156,7 +156,7 @@ private void runExpirationTest(final boolean primaryMode) { replicationTracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED); } final long[] retainingSequenceNumbers = new long[1]; - retainingSequenceNumbers[0] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primaryMode) { replicationTracker.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); } else { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index 7d6e5fa2dc5a6..a99d0caea8e6c 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -67,7 +67,7 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { final Map currentRetentionLeases = new HashMap<>(); for (int i = 0; i < length; i++) { final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8)); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); final CountDownLatch latch = new CountDownLatch(1); final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); @@ -119,7 +119,7 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { final int length = randomIntBetween(1, 8); for (int i = 0; i < length; i++) { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); final CountDownLatch latch = new CountDownLatch(1); final ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString())); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java index 500393f2cfac2..1a8d159c18757 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseTests.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -53,7 +52,7 @@ public void testEmptyId() { } public void testRetainingSequenceNumberOutOfRange() { - final long retainingSequenceNumber = randomLongBetween(Long.MIN_VALUE, UNASSIGNED_SEQ_NO - 1); + final long retainingSequenceNumber = randomLongBetween(Long.MIN_VALUE, -1); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> new RetentionLease("id", retainingSequenceNumber, randomNonNegativeLong(), "source")); @@ -66,7 +65,7 @@ public void testTimestampOutOfRange() { final long timestamp = randomLongBetween(Long.MIN_VALUE, -1); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> new RetentionLease("id", randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), timestamp, "source")); + () -> new RetentionLease("id", randomNonNegativeLong(), timestamp, "source")); assertThat(e, hasToString(containsString("retention lease timestamp [" + timestamp + "] out of range"))); } @@ -87,7 +86,7 @@ public void testEmptySource() { public void testRetentionLeaseSerialization() throws IOException { final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final long timestamp = randomNonNegativeLong(); final String source = randomAlphaOfLength(8); final RetentionLease retentionLease = new RetentionLease(id, retainingSequenceNumber, timestamp, source); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index cd7d2a2c12cb8..26e67fb6dd264 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -113,7 +113,7 @@ private void runExpirationTest(final boolean primary) throws IOException { final IndexShard indexShard = newStartedShard(primary, settings, new InternalEngineFactory()); try { final long[] retainingSequenceNumbers = new long[1]; - retainingSequenceNumbers[0] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); + retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primary) { indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); } else { From b91d587275cbce37142d46d0f2ef95cdc94ae87e Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 30 Jan 2019 12:05:54 +0100 Subject: [PATCH 33/35] Move SearchHit and SearchHits to Writeable (#37931) This allowed to make SearchHits immutable, while quite a few fields in SearchHit have to stay mutable unfortunately. Relates to #34389 --- .../index/rankeval/RatedSearchHit.java | 3 +- .../org/elasticsearch/search/SearchHit.java | 279 +++++++++--------- .../org/elasticsearch/search/SearchHits.java | 123 ++++---- .../aggregations/metrics/InternalTopHits.java | 2 +- .../search/fetch/FetchSearchResult.java | 4 +- .../internal/InternalSearchResponse.java | 2 +- .../completion/CompletionSuggestion.java | 2 +- .../elasticsearch/search/SearchHitTests.java | 11 +- .../elasticsearch/search/SearchHitsTests.java | 12 +- 9 files changed, 208 insertions(+), 230 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index 4b76d837b9515..f28e71134bd85 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -49,8 +49,7 @@ public RatedSearchHit(SearchHit searchHit, OptionalInt rating) { } RatedSearchHit(StreamInput in) throws IOException { - this(SearchHit.readSearchHit(in), - in.readBoolean() == true ? OptionalInt.of(in.readVInt()) : OptionalInt.empty()); + this(new SearchHit(in), in.readBoolean() == true ? OptionalInt.of(in.readVInt()) : OptionalInt.empty()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index df82bbec59900..4cf3bda83530b 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -80,17 +79,17 @@ * * @see SearchHits */ -public final class SearchHit implements Streamable, ToXContentObject, Iterable { +public final class SearchHit implements Writeable, ToXContentObject, Iterable { - private transient int docId; + private final transient int docId; private static final float DEFAULT_SCORE = Float.NaN; private float score = DEFAULT_SCORE; - private Text id; - private Text type; + private final Text id; + private final Text type; - private NestedIdentity nestedIdentity; + private final NestedIdentity nestedIdentity; private long version = -1; private long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -98,7 +97,7 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable fields = emptyMap(); + private Map fields; private Map highlightFields = null; @@ -121,10 +120,6 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable innerHits; - SearchHit() { - - } - //used only in tests public SearchHit(int docId) { this(docId, null, null, null); @@ -146,6 +141,134 @@ public SearchHit(int nestedTopDocId, String id, Text type, NestedIdentity nested this.fields = fields; } + public SearchHit(StreamInput in) throws IOException { + docId = -1; + score = in.readFloat(); + id = in.readOptionalText(); + type = in.readOptionalText(); + nestedIdentity = in.readOptionalWriteable(NestedIdentity::new); + version = in.readLong(); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + seqNo = in.readZLong(); + primaryTerm = in.readVLong(); + } + source = in.readBytesReference(); + if (source.length() == 0) { + source = null; + } + if (in.readBoolean()) { + explanation = readExplanation(in); + } + int size = in.readVInt(); + if (size == 0) { + fields = emptyMap(); + } else if (size == 1) { + DocumentField hitField = DocumentField.readDocumentField(in); + fields = singletonMap(hitField.getName(), hitField); + } else { + Map fields = new HashMap<>(); + for (int i = 0; i < size; i++) { + DocumentField hitField = DocumentField.readDocumentField(in); + fields.put(hitField.getName(), hitField); + } + this.fields = unmodifiableMap(fields); + } + + size = in.readVInt(); + if (size == 0) { + highlightFields = emptyMap(); + } else if (size == 1) { + HighlightField field = readHighlightField(in); + highlightFields = singletonMap(field.name(), field); + } else { + Map highlightFields = new HashMap<>(); + for (int i = 0; i < size; i++) { + HighlightField field = readHighlightField(in); + highlightFields.put(field.name(), field); + } + this.highlightFields = unmodifiableMap(highlightFields); + } + + sortValues = new SearchSortValues(in); + + size = in.readVInt(); + if (size > 0) { + matchedQueries = new String[size]; + for (int i = 0; i < size; i++) { + matchedQueries[i] = in.readString(); + } + } + // we call the setter here because that also sets the local index parameter + shard(in.readOptionalWriteable(SearchShardTarget::new)); + size = in.readVInt(); + if (size > 0) { + innerHits = new HashMap<>(size); + for (int i = 0; i < size; i++) { + String key = in.readString(); + SearchHits value = new SearchHits(in); + innerHits.put(key, value); + } + } else { + innerHits = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeFloat(score); + out.writeOptionalText(id); + out.writeOptionalText(type); + out.writeOptionalWriteable(nestedIdentity); + out.writeLong(version); + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + out.writeZLong(seqNo); + out.writeVLong(primaryTerm); + } + out.writeBytesReference(source); + if (explanation == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + writeExplanation(out, explanation); + } + if (fields == null) { + out.writeVInt(0); + } else { + out.writeVInt(fields.size()); + for (DocumentField hitField : getFields().values()) { + hitField.writeTo(out); + } + } + if (highlightFields == null) { + out.writeVInt(0); + } else { + out.writeVInt(highlightFields.size()); + for (HighlightField highlightField : highlightFields.values()) { + highlightField.writeTo(out); + } + } + sortValues.writeTo(out); + + if (matchedQueries.length == 0) { + out.writeVInt(0); + } else { + out.writeVInt(matchedQueries.length); + for (String matchedFilter : matchedQueries) { + out.writeString(matchedFilter); + } + } + out.writeOptionalWriteable(shard); + if (innerHits == null) { + out.writeVInt(0); + } else { + out.writeVInt(innerHits.size()); + for (Map.Entry entry : innerHits.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + } + public int docId() { return this.docId; } @@ -771,140 +894,6 @@ private void buildExplanation(XContentBuilder builder, Explanation explanation) builder.endObject(); } - public static SearchHit readSearchHit(StreamInput in) throws IOException { - SearchHit hit = new SearchHit(); - hit.readFrom(in); - return hit; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - score = in.readFloat(); - id = in.readOptionalText(); - type = in.readOptionalText(); - nestedIdentity = in.readOptionalWriteable(NestedIdentity::new); - version = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_6_7_0)) { - seqNo = in.readZLong(); - primaryTerm = in.readVLong(); - } - source = in.readBytesReference(); - if (source.length() == 0) { - source = null; - } - if (in.readBoolean()) { - explanation = readExplanation(in); - } - int size = in.readVInt(); - if (size == 0) { - fields = emptyMap(); - } else if (size == 1) { - DocumentField hitField = DocumentField.readDocumentField(in); - fields = singletonMap(hitField.getName(), hitField); - } else { - Map fields = new HashMap<>(); - for (int i = 0; i < size; i++) { - DocumentField hitField = DocumentField.readDocumentField(in); - fields.put(hitField.getName(), hitField); - } - this.fields = unmodifiableMap(fields); - } - - size = in.readVInt(); - if (size == 0) { - highlightFields = emptyMap(); - } else if (size == 1) { - HighlightField field = readHighlightField(in); - highlightFields = singletonMap(field.name(), field); - } else { - Map highlightFields = new HashMap<>(); - for (int i = 0; i < size; i++) { - HighlightField field = readHighlightField(in); - highlightFields.put(field.name(), field); - } - this.highlightFields = unmodifiableMap(highlightFields); - } - - sortValues = new SearchSortValues(in); - - size = in.readVInt(); - if (size > 0) { - matchedQueries = new String[size]; - for (int i = 0; i < size; i++) { - matchedQueries[i] = in.readString(); - } - } - // we call the setter here because that also sets the local index parameter - shard(in.readOptionalWriteable(SearchShardTarget::new)); - size = in.readVInt(); - if (size > 0) { - innerHits = new HashMap<>(size); - for (int i = 0; i < size; i++) { - String key = in.readString(); - SearchHits value = SearchHits.readSearchHits(in); - innerHits.put(key, value); - } - } else { - innerHits = null; - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeFloat(score); - out.writeOptionalText(id); - out.writeOptionalText(type); - out.writeOptionalWriteable(nestedIdentity); - out.writeLong(version); - if (out.getVersion().onOrAfter(Version.V_6_7_0)) { - out.writeZLong(seqNo); - out.writeVLong(primaryTerm); - } - out.writeBytesReference(source); - if (explanation == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - writeExplanation(out, explanation); - } - if (fields == null) { - out.writeVInt(0); - } else { - out.writeVInt(fields.size()); - for (DocumentField hitField : getFields().values()) { - hitField.writeTo(out); - } - } - if (highlightFields == null) { - out.writeVInt(0); - } else { - out.writeVInt(highlightFields.size()); - for (HighlightField highlightField : highlightFields.values()) { - highlightField.writeTo(out); - } - } - sortValues.writeTo(out); - - if (matchedQueries.length == 0) { - out.writeVInt(0); - } else { - out.writeVInt(matchedQueries.length); - for (String matchedFilter : matchedQueries) { - out.writeString(matchedFilter); - } - } - out.writeOptionalWriteable(shard); - if (innerHits == null) { - out.writeVInt(0); - } else { - out.writeVInt(innerHits.size()); - for (Map.Entry entry : innerHits.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } - } - } - @Override public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java index f04183ffde700..93478c94048a2 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHits.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -43,7 +42,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -public final class SearchHits implements Streamable, ToXContentFragment, Iterable { +public final class SearchHits implements Writeable, ToXContentFragment, Iterable { public static SearchHits empty() { return empty(true); } @@ -55,22 +54,15 @@ public static SearchHits empty(boolean withTotalHits) { public static final SearchHit[] EMPTY = new SearchHit[0]; - private SearchHit[] hits; - - private Total totalHits; - - private float maxScore; - + private final SearchHit[] hits; + private final Total totalHits; + private final float maxScore; @Nullable - private SortField[] sortFields; + private final SortField[] sortFields; @Nullable - private String collapseField; + private final String collapseField; @Nullable - private Object[] collapseValues; - - SearchHits() { - - } + private final Object[] collapseValues; public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScore) { this(hits, totalHits, maxScore, null, null, null); @@ -86,6 +78,55 @@ public SearchHits(SearchHit[] hits, @Nullable TotalHits totalHits, float maxScor this.collapseValues = collapseValues; } + public SearchHits(StreamInput in) throws IOException { + if (in.readBoolean()) { + totalHits = new Total(in); + } else { + // track_total_hits is false + totalHits = null; + } + maxScore = in.readFloat(); + int size = in.readVInt(); + if (size == 0) { + hits = EMPTY; + } else { + hits = new SearchHit[size]; + for (int i = 0; i < hits.length; i++) { + hits[i] = new SearchHit(in); + } + } + if (in.getVersion().onOrAfter(Version.V_6_6_0)) { + sortFields = in.readOptionalArray(Lucene::readSortField, SortField[]::new); + collapseField = in.readOptionalString(); + collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); + } else { + sortFields = null; + collapseField = null; + collapseValues = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + final boolean hasTotalHits = totalHits != null; + out.writeBoolean(hasTotalHits); + if (hasTotalHits) { + totalHits.writeTo(out); + } + out.writeFloat(maxScore); + out.writeVInt(hits.length); + if (hits.length > 0) { + for (SearchHit hit : hits) { + hit.writeTo(out); + } + } + if (out.getVersion().onOrAfter(Version.V_6_6_0)) { + out.writeOptionalArray(Lucene::writeSortField, sortFields); + out.writeOptionalString(collapseField); + out.writeOptionalArray(Lucene::writeSortValue, collapseValues); + } + } + /** * The total number of hits for the query or null if the tracking of total hits * is disabled in the request. @@ -222,58 +263,6 @@ public static SearchHits fromXContent(XContentParser parser) throws IOException return new SearchHits(hits.toArray(new SearchHit[0]), totalHits, maxScore); } - public static SearchHits readSearchHits(StreamInput in) throws IOException { - SearchHits hits = new SearchHits(); - hits.readFrom(in); - return hits; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - if (in.readBoolean()) { - totalHits = new Total(in); - } else { - // track_total_hits is false - totalHits = null; - } - maxScore = in.readFloat(); - int size = in.readVInt(); - if (size == 0) { - hits = EMPTY; - } else { - hits = new SearchHit[size]; - for (int i = 0; i < hits.length; i++) { - hits[i] = SearchHit.readSearchHit(in); - } - } - if (in.getVersion().onOrAfter(Version.V_6_6_0)) { - sortFields = in.readOptionalArray(Lucene::readSortField, SortField[]::new); - collapseField = in.readOptionalString(); - collapseValues = in.readOptionalArray(Lucene::readSortValue, Object[]::new); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - final boolean hasTotalHits = totalHits != null; - out.writeBoolean(hasTotalHits); - if (hasTotalHits) { - totalHits.writeTo(out); - } - out.writeFloat(maxScore); - out.writeVInt(hits.length); - if (hits.length > 0) { - for (SearchHit hit : hits) { - hit.writeTo(out); - } - } - if (out.getVersion().onOrAfter(Version.V_6_6_0)) { - out.writeOptionalArray(Lucene::writeSortField, sortFields); - out.writeOptionalString(collapseField); - out.writeOptionalArray(Lucene::writeSortValue, collapseValues); - } - } - @Override public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java index 60ae4c78edc7e..4a266ee0703bd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java @@ -65,7 +65,7 @@ public InternalTopHits(StreamInput in) throws IOException { from = in.readVInt(); size = in.readVInt(); topDocs = Lucene.readTopDocs(in); - searchHits = SearchHits.readSearchHits(in); + searchHits = new SearchHits(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java index 12391151861d0..400ab3623c0e8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java @@ -22,9 +22,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.query.QuerySearchResult; import java.io.IOException; @@ -92,7 +92,7 @@ public static FetchSearchResult readFetchSearchResult(StreamInput in) throws IOE public void readFrom(StreamInput in) throws IOException { super.readFrom(in); requestId = in.readLong(); - hits = SearchHits.readSearchHits(in); + hits = new SearchHits(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java index e78ce7f3fb194..bac7b6486bb88 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java +++ b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java @@ -51,7 +51,7 @@ public InternalSearchResponse(SearchHits hits, InternalAggregations aggregations public InternalSearchResponse(StreamInput in) throws IOException { super( - SearchHits.readSearchHits(in), + new SearchHits(in), in.readBoolean() ? InternalAggregations.readAggregations(in) : null, in.readBoolean() ? new Suggest(in) : null, in.readBoolean(), diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java index 34c191df3b36a..bc2e2abdb7d03 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java @@ -292,7 +292,7 @@ public Option(StreamInput in) throws IOException { super(in); this.doc = Lucene.readScoreDoc(in); if (in.readBoolean()) { - this.hit = SearchHit.readSearchHit(in); + this.hit = new SearchHit(in); } int contextSize = in.readInt(); this.contexts = new LinkedHashMap<>(contextSize); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 4831729201183..bbd5b12ec45ee 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -39,7 +40,7 @@ import org.elasticsearch.search.SearchHit.NestedIdentity; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests; -import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.RandomObjects; import org.elasticsearch.test.VersionUtils; @@ -59,7 +60,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class SearchHitTests extends AbstractStreamableTestCase { +public class SearchHitTests extends AbstractWireSerializingTestCase { public static SearchHit createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); } @@ -139,8 +140,8 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp } @Override - protected SearchHit createBlankInstance() { - return new SearchHit(); + protected Writeable.Reader instanceReader() { + return SearchHit::new; } @Override @@ -246,7 +247,7 @@ public void testSerializeShardTarget() throws Exception { SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2}, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1f); Version version = VersionUtils.randomVersion(random()); - SearchHits results = copyStreamable(hits, getNamedWriteableRegistry(), SearchHits::new, version); + SearchHits results = copyWriteable(hits, getNamedWriteableRegistry(), SearchHits::new, version); SearchShardTarget deserializedTarget = results.getAt(0).getShard(); assertThat(deserializedTarget, equalTo(target)); assertThat(results.getAt(0).getInnerHits().get("1").getAt(0).getShard(), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 396879e8f65bd..9e87628d35d1c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.LuceneTests; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -37,7 +38,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.VersionUtils; import java.io.IOException; @@ -45,7 +46,7 @@ import java.util.Collections; import java.util.function.Predicate; -public class SearchHitsTests extends AbstractStreamableXContentTestCase { +public class SearchHitsTests extends AbstractSerializingTestCase { public static SearchHits createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); @@ -171,8 +172,8 @@ protected String[] getShuffleFieldsExceptions() { } @Override - protected SearchHits createBlankInstance() { - return new SearchHits(); + protected Writeable.Reader instanceReader() { + return SearchHits::new; } @Override @@ -274,8 +275,7 @@ public void testFromXContentWithShards() throws IOException { public void testReadFromPre6_6_0() throws IOException { try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode("AQC/gAAAAAA="))) { in.setVersion(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_6_6_0))); - SearchHits searchHits = new SearchHits(); - searchHits.readFrom(in); + SearchHits searchHits = new SearchHits(in); assertEquals(0, searchHits.getHits().length); assertNotNull(searchHits.getTotalHits()); assertEquals(0L, searchHits.getTotalHits().value); From f3379940c6ceb74a3bcb553c90151e1650a2718d Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 30 Jan 2019 14:41:35 +0200 Subject: [PATCH 34/35] Fix the packer cache script (#38023) The script is used to create a cache on ephemeral CI workers. Changes: - create and use a `pullFixture` task that always exists regardless of docker support - wire dependencies correctly so any pre fixture setup runs for pull as well - set up java env vars so bwc versions can build --- .ci/packer_cache.sh | 7 ++++++- .../gradle/testfixtures/TestFixturesPlugin.java | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 6a7e2e69faebf..ce78ebcc1cae7 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -16,4 +16,9 @@ while [ -h "$SCRIPT" ] ; do done source $(dirname "${SCRIPT}")/java-versions.properties -JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel resolveAllDependencies composePull +export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} +# We are caching BWC versions too, need these so we can build those +export JAVA8_HOME="${HOME}"/.java/java8 +export JAVA11_HOME="${HOME}"/.java/java11 +export JAVA12_HOME="${HOME}"/.java/java12 +./gradlew --parallel clean pullFixture --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java index 35a7eacf1fde1..3dfccaf435031 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java @@ -59,14 +59,17 @@ public void apply(Project project) { disableTaskByType(tasks, JarHellTask.class); Task buildFixture = project.getTasks().create("buildFixture"); + Task pullFixture = project.getTasks().create("pullFixture"); Task preProcessFixture = project.getTasks().create("preProcessFixture"); buildFixture.dependsOn(preProcessFixture); + pullFixture.dependsOn(preProcessFixture); Task postProcessFixture = project.getTasks().create("postProcessFixture"); if (dockerComposeSupported(project) == false) { preProcessFixture.setEnabled(false); postProcessFixture.setEnabled(false); buildFixture.setEnabled(false); + pullFixture.setEnabled(false); return; } @@ -81,7 +84,9 @@ public void apply(Project project) { ); buildFixture.dependsOn(tasks.getByName("composeUp")); + pullFixture.dependsOn(tasks.getByName("composePull")); tasks.getByName("composeUp").mustRunAfter(preProcessFixture); + tasks.getByName("composePull").mustRunAfter(preProcessFixture); postProcessFixture.dependsOn(buildFixture); configureServiceInfoForTask( From 8280a2066444391979a7a35a73c39a0d19c3186e Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 30 Jan 2019 06:51:11 -0600 Subject: [PATCH 35/35] ML: Add upgrade mode docs, hlrc, and fix bug (#37942) * ML: Add upgrade mode docs, hlrc, and fix bug * [DOCS] Fixes build error and edits text * adjusting docs * Update docs/reference/ml/apis/set-upgrade-mode.asciidoc Co-Authored-By: benwtrent * Update set-upgrade-mode.asciidoc * Update set-upgrade-mode.asciidoc --- .../client/MLRequestConverters.java | 12 ++ .../client/MachineLearningClient.java | 39 +++++++ .../client/ml/SetUpgradeModeRequest.java | 93 ++++++++++++++++ .../client/MLRequestConvertersTests.java | 17 +++ .../client/MachineLearningIT.java | 27 +++++ .../MlClientDocumentationIT.java | 52 +++++++++ .../high-level/ml/set-upgrade-mode.asciidoc | 40 +++++++ .../high-level/supported-apis.asciidoc | 2 + docs/reference/ml/apis/ml-api.asciidoc | 13 ++- .../ml/apis/set-upgrade-mode.asciidoc | 103 ++++++++++++++++++ .../xpack/core/ml/MlMetadata.java | 8 +- .../action/TransportSetUpgradeModeAction.java | 7 ++ .../api/ml.set_upgrade_mode.json | 2 +- 13 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java create mode 100644 docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc create mode 100644 docs/reference/ml/apis/set-upgrade-mode.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 6b0a5d2642f02..073b92f84a3a3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -64,6 +64,7 @@ import org.elasticsearch.client.ml.PutFilterRequest; import org.elasticsearch.client.ml.PutJobRequest; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StopDatafeedRequest; import org.elasticsearch.client.ml.UpdateDatafeedRequest; @@ -624,6 +625,17 @@ static Request deleteFilter(DeleteFilterRequest deleteFilterRequest) { return request; } + static Request setUpgradeMode(SetUpgradeModeRequest setUpgradeModeRequest) { + String endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "set_upgrade_mode").build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + RequestConverters.Params params = new RequestConverters.Params(request); + params.putParam(SetUpgradeModeRequest.ENABLED.getPreferredName(), Boolean.toString(setUpgradeModeRequest.isEnabled())); + if (setUpgradeModeRequest.getTimeout() != null) { + params.putParam(SetUpgradeModeRequest.TIMEOUT.getPreferredName(), setUpgradeModeRequest.getTimeout().toString()); + } + return request; + } + static Request mlInfo(MlInfoRequest infoRequest) { String endpoint = new EndpointBuilder() .addPathPartAsIs("_ml", "info") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index aaff35a238998..2e359931c1025 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -86,6 +86,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -1838,4 +1839,42 @@ public void findFileStructureAsync(FindFileStructureRequest request, RequestOpti listener, Collections.emptySet()); } + + /** + * Sets the ML cluster setting upgrade_mode + *

+ * For additional info + * see Set Upgrade Mode + * + * @param request The request to set upgrade mode + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return response + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public AcknowledgedResponse setUpgradeMode(SetUpgradeModeRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::setUpgradeMode, + options, + AcknowledgedResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Sets the ML cluster setting upgrade_mode asynchronously + *

+ * For additional info + * see Set Upgrade Mode + * + * @param request The request of Machine Learning info + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void setUpgradeModeAsync(SetUpgradeModeRequest request, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::setUpgradeMode, + options, + AcknowledgedResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java new file mode 100644 index 0000000000000..64e94f0251785 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/SetUpgradeModeRequest.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; + +import java.util.Objects; + +/** + * Sets ML into upgrade_mode + */ +public class SetUpgradeModeRequest extends ActionRequest { + + + public static final ParseField ENABLED = new ParseField("enabled"); + public static final ParseField TIMEOUT = new ParseField("timeout"); + + private boolean enabled; + private TimeValue timeout; + + /** + * Create a new request + * + * @param enabled whether to enable `upgrade_mode` or not + */ + public SetUpgradeModeRequest(boolean enabled) { + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public TimeValue getTimeout() { + return timeout; + } + + /** + * How long to wait for the request to be completed + * + * @param timeout default value of 30 seconds + */ + public void setTimeout(TimeValue timeout) { + this.timeout = timeout; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public int hashCode() { + return Objects.hash(enabled, timeout); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + SetUpgradeModeRequest that = (SetUpgradeModeRequest) other; + return Objects.equals(enabled, that.enabled) && Objects.equals(timeout, that.timeout); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index b472837f9c25d..11faaf879729d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -61,6 +61,7 @@ import org.elasticsearch.client.ml.PutFilterRequest; import org.elasticsearch.client.ml.PutJobRequest; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedRequestTests; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -818,6 +819,22 @@ public void testFindFileStructure() throws Exception { assertEquals(sample, requestEntityToString(request)); } + public void testSetUpgradeMode() { + SetUpgradeModeRequest setUpgradeModeRequest = new SetUpgradeModeRequest(true); + + Request request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest); + assertThat(request.getEndpoint(), equalTo("/_ml/set_upgrade_mode")); + assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME)); + assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(true))); + assertThat(request.getParameters().containsKey(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is(false)); + + setUpgradeModeRequest.setTimeout(TimeValue.timeValueHours(1)); + setUpgradeModeRequest.setEnabled(false); + request = MLRequestConverters.setUpgradeMode(setUpgradeModeRequest); + assertThat(request.getParameters().get(SetUpgradeModeRequest.ENABLED.getPreferredName()), equalTo(Boolean.toString(false))); + assertThat(request.getParameters().get(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is("1h")); + } + private static Job createValidJob(String jobId) { AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList( Detector.builder().setFunction("count").build())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 162fceed9e572..07d7187fd1d70 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -84,6 +84,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -1614,4 +1615,30 @@ public void testFindFileStructure() throws IOException { assertEquals("timestamp", structure.getTimestampField()); assertFalse(structure.needClientTimezone()); } + + public void testEnableUpgradeMode() throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + MlInfoResponse mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false)); + + AcknowledgedResponse setUpgrademodeResponse = execute(new SetUpgradeModeRequest(true), + machineLearningClient::setUpgradeMode, + machineLearningClient::setUpgradeModeAsync); + + assertThat(setUpgrademodeResponse.isAcknowledged(), is(true)); + + + mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(true)); + + setUpgrademodeResponse = execute(new SetUpgradeModeRequest(false), + machineLearningClient::setUpgradeMode, + machineLearningClient::setUpgradeModeAsync); + + assertThat(setUpgrademodeResponse.isAcknowledged(), is(true)); + + mlInfoResponse = machineLearningClient.getMlInfo(new MlInfoRequest(), RequestOptions.DEFAULT); + assertThat(mlInfoResponse.getInfo().get("upgrade_mode"), equalTo(false)); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 68881206b487f..f4e3f86196f29 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -99,6 +99,7 @@ import org.elasticsearch.client.ml.PutJobResponse; import org.elasticsearch.client.ml.RevertModelSnapshotRequest; import org.elasticsearch.client.ml.RevertModelSnapshotResponse; +import org.elasticsearch.client.ml.SetUpgradeModeRequest; import org.elasticsearch.client.ml.StartDatafeedRequest; import org.elasticsearch.client.ml.StartDatafeedResponse; import org.elasticsearch.client.ml.StopDatafeedRequest; @@ -3078,6 +3079,57 @@ public void onFailure(Exception e) { } } + public void testSetUpgradeMode() throws Exception { + RestHighLevelClient client = highLevelClient(); + { + // tag::set-upgrade-mode-request + SetUpgradeModeRequest request = new SetUpgradeModeRequest(true); // <1> + request.setTimeout(TimeValue.timeValueMinutes(10)); // <2> + // end::set-upgrade-mode-request + + // Set to false so that the cluster setting does not have to be unset at the end of the test. + request.setEnabled(false); + + // tag::set-upgrade-mode-execute + AcknowledgedResponse acknowledgedResponse = client.machineLearning().setUpgradeMode(request, RequestOptions.DEFAULT); + // end::set-upgrade-mode-execute + + // tag::set-upgrade-mode-response + boolean acknowledged = acknowledgedResponse.isAcknowledged(); // <1> + // end::set-upgrade-mode-response + assertThat(acknowledged, is(true)); + } + { + SetUpgradeModeRequest request = new SetUpgradeModeRequest(false); + + // tag::set-upgrade-mode-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::set-upgrade-mode-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::set-upgrade-mode-execute-async + client.machineLearning() + .setUpgradeModeAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::set-upgrade-mode-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + + } + } + private String createFilter(RestHighLevelClient client) throws IOException { MlFilter.Builder filterBuilder = MlFilter.builder("my_safe_domains") .setDescription("A list of safe domains") diff --git a/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc b/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc new file mode 100644 index 0000000000000..80bb1874e4a63 --- /dev/null +++ b/docs/java-rest/high-level/ml/set-upgrade-mode.asciidoc @@ -0,0 +1,40 @@ +-- +:api: set-upgrade-mode +:request: SetUpgradeModeRequest +:response: AcknowledgedResponse +-- +[id="{upid}-{api}"] +=== Set Upgrade Mode API + +The Set Upgrade Mode API temporarily halts all {ml} job and {dfeed} tasks when `enabled=true`. Their +reported states remain unchanged. Consequently, when exiting upgrade mode the halted {ml} jobs and +{dfeeds} will return to their previous state. + +It accepts a +{request}+ object and responds with a +{response}+ object. + +When `enabled=true`, no new jobs can be opened, and no job or {dfeed} tasks will +be running. Be sure to set `enabled=false` once upgrade actions are completed. + +[id="{upid}-{api}-request"] +==== Set Upgrade Mode Request + +A +{request}+ object gets created setting the desired `enabled` state. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Constructing a new request referencing enabling upgrade mode +<2> Optionally setting the `timeout` value for how long the +execution should wait. + +[id="{upid}-{api}-response"] +==== Set Upgrade Mode Response + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> `isAcknowledged()` from the +{response}+ indicates if the setting was set successfully. + +include::../execution.asciidoc[] \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 0b4a2570c896d..70f06e457e9b5 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -295,6 +295,7 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <<{upid}-update-model-snapshot>> * <<{upid}-get-ml-info>> * <<{upid}-delete-expired-data>> +* <<{upid}-set-upgrade-mode>> include::ml/put-job.asciidoc[] include::ml/get-job.asciidoc[] @@ -338,6 +339,7 @@ include::ml/revert-model-snapshot.asciidoc[] include::ml/update-model-snapshot.asciidoc[] include::ml/get-info.asciidoc[] include::ml/delete-expired-data.asciidoc[] +include::ml/set-upgrade-mode.asciidoc[] == Migration APIs diff --git a/docs/reference/ml/apis/ml-api.asciidoc b/docs/reference/ml/apis/ml-api.asciidoc index 6cb0dc6ba4093..7933dea85ce0a 100644 --- a/docs/reference/ml/apis/ml-api.asciidoc +++ b/docs/reference/ml/apis/ml-api.asciidoc @@ -72,7 +72,7 @@ machine learning APIs and in advanced job configuration options in Kibana. [float] [[ml-api-file-structure-endpoint]] -=== File Structure +=== File structure * <> @@ -84,10 +84,16 @@ machine learning APIs and in advanced job configuration options in Kibana. [float] [[ml-api-delete-expired-data-endpoint]] -=== Delete Expired Data +=== Delete expired data * <> +[float] +[[ml-set-upgrade-mode-endpoint]] +=== Set upgrade mode + +* <> + //ADD include::post-calendar-event.asciidoc[] include::put-calendar-job.asciidoc[] @@ -137,7 +143,8 @@ include::post-data.asciidoc[] include::preview-datafeed.asciidoc[] //REVERT include::revert-snapshot.asciidoc[] -//START/STOP +//SET/START/STOP +include::set-upgrade-mode.asciidoc[] include::start-datafeed.asciidoc[] include::stop-datafeed.asciidoc[] //UPDATE diff --git a/docs/reference/ml/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/apis/set-upgrade-mode.asciidoc new file mode 100644 index 0000000000000..5434d70d4e61e --- /dev/null +++ b/docs/reference/ml/apis/set-upgrade-mode.asciidoc @@ -0,0 +1,103 @@ +[role="xpack"] +[testenv="platinum"] +[[ml-set-upgrade-mode]] +=== Set upgrade mode API +++++ +Set upgrade mode +++++ + +Sets a cluster wide upgrade_mode setting that prepares {ml} indices for an +upgrade. + +==== Request +////////////////////////// + +[source,js] +-------------------------------------------------- +POST /_ml/set_upgrade_mode?enabled=false&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST +// TEARDOWN + +////////////////////////// + + +`POST _ml/set_upgrade_mode` + +==== Description + +When upgrading your cluster, in some circumstances you must restart your nodes and +reindex your {ml} indices. In those circumstances, there must be no {ml} jobs running. +You can close the {ml} jobs, do the upgrade, then open all the jobs again. +Alternatively, you can use this API to temporarily halt tasks associated +with the jobs and {dfeeds} and prevent new jobs from opening. You can also use this +API during upgrades that do not require you to reindex your {ml} indices, +though stopping jobs is not a requirement in that case. + +For more information, see {stack-ref}/upgrading-elastic-stack.html[Upgrading the {stack}]. + + +When `enabled=true` this API temporarily halts all job and {dfeed} tasks and +prohibits new job and {dfeed} tasks from starting. + +Subsequently, you can call the API with the enabled parameter set to false, +which causes {ml} jobs and {dfeeds} to return to their desired states. + +You can see the current value for the `upgrade_mode` setting by using the +<>. + +IMPORTANT: No new {ml} jobs can be opened while the `upgrade_mode` setting is +`true`. + +==== Query Parameters + +`enabled`:: + (boolean) When `true`, this enables `upgrade_mode`. Defaults to `false` + +`timeout`:: + (time) The time to wait for the request to be completed. + The default value is 30 seconds. + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see +{stack-ov}/security-privileges.html[Security privileges]. + + +==== Examples + +The following example enables `upgrade_mode` for the cluster: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=true&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST + +When the call is successful, an acknowledged response is returned. For example: + +[source,js] +---- +{ + "acknowledged": true +} +---- +// TESTRESPONSE + +The acknowledged response will only be returned once all {ml} jobs and {dfeeds} have +finished writing to the {ml} internal indices. This means it is safe to reindex those +internal indices without causing failures. You must wait for the acknowledged +response before reindexing to ensure that all writes are completed. + +When the upgrade is complete, you must set `upgrade_mode` to `false` for +{ml} jobs to start running again. For example: + +[source,js] +-------------------------------------------------- +POST _ml/set_upgrade_mode?enabled=false&timeout=10m +-------------------------------------------------- +// CONSOLE +// TEST diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java index 43462c552da63..54c83e9a88a75 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java @@ -137,7 +137,7 @@ public MlMetadata(StreamInput in) throws IOException { } this.datafeeds = datafeeds; this.groupOrJobLookup = new GroupOrJobLookup(jobs.values()); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { this.upgradeMode = in.readBoolean(); } else { this.upgradeMode = false; @@ -148,7 +148,7 @@ public MlMetadata(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { writeMap(jobs, out); writeMap(datafeeds, out); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeBoolean(upgradeMode); } } @@ -201,7 +201,7 @@ public MlMetadataDiff(StreamInput in) throws IOException { MlMetadataDiff::readJobDiffFrom); this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new, MlMetadataDiff::readDatafeedDiffFrom); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { upgradeMode = in.readBoolean(); } else { upgradeMode = false; @@ -224,7 +224,7 @@ public MetaData.Custom apply(MetaData.Custom part) { public void writeTo(StreamOutput out) throws IOException { jobs.writeTo(out); datafeeds.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { out.writeBoolean(upgradeMode); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java index 1834b0b3c0616..6ec77a34c55b6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java @@ -164,6 +164,13 @@ protected void masterOperation(SetUpgradeModeAction.Request request, ClusterStat wrappedListener.onFailure(new ElasticsearchTimeoutException("Unknown error occurred while updating cluster state")); return; } + + // There are no tasks to worry about starting/stopping + if (tasksCustomMetaData == null || tasksCustomMetaData.tasks().isEmpty()) { + wrappedListener.onResponse(new AcknowledgedResponse(true)); + return; + } + // Did we change from disabled -> enabled? if (request.isEnabled()) { isolateDatafeeds(tasksCustomMetaData, isolateDatafeedListener); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json index bb3220ece6b13..5406d2d06eceb 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json @@ -1,6 +1,6 @@ { "ml.set_upgrade_mode": { - "documentation": "TODO", + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-set-upgrade-mode.html", "methods": [ "POST" ], "url": { "path": "/_ml/set_upgrade_mode",