From 454e4967f14f0a8e5377170be73510882764f0fd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 12 Apr 2019 10:43:10 -0600 Subject: [PATCH 01/15] Add ILM poll_interval limit to breaking changes (#41095) ILM's poll_interval setting now has a lower limit of 1 second, which is technically a breaking change. --- docs/reference/migration/migrate_8_0.asciidoc | 2 ++ .../migration/migrate_8_0/ilm.asciidoc | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 docs/reference/migration/migrate_8_0/ilm.asciidoc diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc index 56a9f582142b7..48c883c6197d9 100644 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -17,6 +17,7 @@ coming[8.0.0] * <> * <> * <> +* <> * <> //NOTE: The notable-breaking-changes tagged regions are re-used in the @@ -46,4 +47,5 @@ include::migrate_8_0/mappings.asciidoc[] include::migrate_8_0/packaging.asciidoc[] include::migrate_8_0/snapshots.asciidoc[] include::migrate_8_0/security.asciidoc[] +include::migrate_8_0/ilm.asciidoc[] include::migrate_8_0/java.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/ilm.asciidoc b/docs/reference/migration/migrate_8_0/ilm.asciidoc new file mode 100644 index 0000000000000..e18b39a0d7d9e --- /dev/null +++ b/docs/reference/migration/migrate_8_0/ilm.asciidoc @@ -0,0 +1,17 @@ +[float] +[[breaking_80_ilm_changes]] +=== Index Lifecycle Management changes + +//NOTE: The notable-breaking-changes tagged regions are re-used in the +//Installation and Upgrade Guide + +//tag::notable-breaking-changes[] + +// end::notable-breaking-changes[] + +[float] +[[ilm-poll-interval-limit]] +==== `indices.lifecycle.poll_interval` must be greater than 1 second + +The setting `indices.lifecycle.poll_interval`, if set too low, can cause +excessive load on a cluster. This setting must now be set to 1 second or higher. From 07e0f0df21b67d62c18e00a9f4f124df76c2fc15 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 12 Apr 2019 15:42:05 -0600 Subject: [PATCH 02/15] Remove deprecated transport settings (#40918) This is related to #36652. There are a number of transport settings that were deprecated and replaced with new versions in 7.x. This commit removes them from 8.0. --- docs/reference/migration/migrate_8_0.asciidoc | 6 ++ .../migration/migrate_8_0/http.asciidoc | 15 +++++ .../migration/migrate_8_0/network.asciidoc | 16 ++++++ .../migration/migrate_8_0/transport.asciidoc | 22 ++++++++ ...Netty4TransportMultiPortIntegrationIT.java | 2 +- .../common/network/NetworkService.java | 5 -- .../common/settings/ClusterSettings.java | 11 ---- .../http/HttpTransportSettings.java | 4 +- .../transport/TransportSettings.java | 55 ++++--------------- .../resources/packaging/tests/certgen.bash | 2 +- 10 files changed, 72 insertions(+), 66 deletions(-) create mode 100644 docs/reference/migration/migrate_8_0/http.asciidoc create mode 100644 docs/reference/migration/migrate_8_0/network.asciidoc create mode 100644 docs/reference/migration/migrate_8_0/transport.asciidoc diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc index 48c883c6197d9..11502be569076 100644 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -19,6 +19,9 @@ coming[8.0.0] * <> * <> * <> +* <> +* <> +* <> //NOTE: The notable-breaking-changes tagged regions are re-used in the //Installation and Upgrade Guide @@ -49,3 +52,6 @@ include::migrate_8_0/snapshots.asciidoc[] include::migrate_8_0/security.asciidoc[] include::migrate_8_0/ilm.asciidoc[] include::migrate_8_0/java.asciidoc[] +include::migrate_8_0/network.asciidoc[] +include::migrate_8_0/transport.asciidoc[] +include::migrate_8_0/http.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/http.asciidoc b/docs/reference/migration/migrate_8_0/http.asciidoc new file mode 100644 index 0000000000000..6209beeaac4e2 --- /dev/null +++ b/docs/reference/migration/migrate_8_0/http.asciidoc @@ -0,0 +1,15 @@ +[float] +[[breaking_80_http_changes]] +=== HTTP changes + +//NOTE: The notable-breaking-changes tagged regions are re-used in the +//Installation and Upgrade Guide +//tag::notable-breaking-changes[] + +// end::notable-breaking-changes[] + +[float] +==== Removal of old HTTP settings + +The `http.tcp_no_delay` setting was deprecated in 7.x and has been removed in 8.0. It has been replaced by +`http.tcp.no_delay`. \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/network.asciidoc b/docs/reference/migration/migrate_8_0/network.asciidoc new file mode 100644 index 0000000000000..ac77f6e9c0cb8 --- /dev/null +++ b/docs/reference/migration/migrate_8_0/network.asciidoc @@ -0,0 +1,16 @@ +[float] +[[breaking_80_network_changes]] +=== Network changes + +//NOTE: The notable-breaking-changes tagged regions are re-used in the +//Installation and Upgrade Guide +//tag::notable-breaking-changes[] + +// end::notable-breaking-changes[] + +[float] +==== Removal of old network settings + +The `network.tcp.connect_timeout` setting was deprecated in 7.x and has been removed in 8.0. This setting +was a fallback setting for `transport.connect_timeout`. To change the default connection timeout for client +connections `transport.connect_timeout` should be modified. \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/transport.asciidoc b/docs/reference/migration/migrate_8_0/transport.asciidoc new file mode 100644 index 0000000000000..f55117ae81c22 --- /dev/null +++ b/docs/reference/migration/migrate_8_0/transport.asciidoc @@ -0,0 +1,22 @@ +[float] +[[breaking_80_transport_changes]] +=== Transport changes + +//tag::notable-breaking-changes[] +[float] +==== Removal of old transport settings + +The following settings have been deprecated in 7.x and removed in 8.0. Each setting has a replacement +setting that was introduced in 6.7. + +- `transport.tcp.port` replaced by `transport.port` +- `transport.tcp.compress` replaced by `transport.compress` +- `transport.tcp.connect_timeout` replaced by `transport.connect_timeout` +- `transport.tcp_no_delay` replaced by `transport.tcp.no_delay` +- `transport.profiles.profile_name.tcp_no_delay` replaced by `transport.profiles.profile_name.tcp.no_delay` +- `transport.profiles.profile_name.tcp_keep_alive` replaced by `transport.profiles.profile_name.tcp.keep_alive` +- `transport.profiles.profile_name.reuse_address` replaced by `transport.profiles.profile_name.tcp.reuse_address` +- `transport.profiles.profile_name.send_buffer_size` replaced by `transport.profiles.profile_name.tcp.send_buffer_size` +- `transport.profiles.profile_name.receive_buffer_size` replaced by `transport.profiles.profile_name.tcp.receive_buffer_size` + +// end::notable-breaking-changes[] \ No newline at end of file diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java index 52ad32efb5645..881fe73112c5a 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -65,7 +65,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put("transport.profiles.client1.port", randomPortRange) .put("transport.profiles.client1.publish_host", "127.0.0.7") .put("transport.profiles.client1.publish_port", "4321") - .put("transport.profiles.client1.reuse_address", true); + .put("transport.profiles.client1.tcp.reuse_address", true); return builder.build(); } diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkService.java b/server/src/main/java/org/elasticsearch/common/network/NetworkService.java index babc83a17722c..cdae56a4e59c9 100644 --- a/server/src/main/java/org/elasticsearch/common/network/NetworkService.java +++ b/server/src/main/java/org/elasticsearch/common/network/NetworkService.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.net.InetAddress; @@ -32,7 +31,6 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.concurrent.TimeUnit; import java.util.function.Function; public final class NetworkService { @@ -58,9 +56,6 @@ public final class NetworkService { Setting.byteSizeSetting("network.tcp.send_buffer_size", new ByteSizeValue(-1), Property.NodeScope); public static final Setting TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("network.tcp.receive_buffer_size", new ByteSizeValue(-1), Property.NodeScope); - public static final Setting TCP_CONNECT_TIMEOUT = - Setting.timeSetting("network.tcp.connect_timeout", new TimeValue(30, TimeUnit.SECONDS), Property.NodeScope, - Setting.Property.Deprecated); /** * A custom name resolver can support custom lookup keys (my_net_key:ipv4) and also change diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 315e6175a3a36..82ae117c3e24b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -261,7 +261,6 @@ public void apply(Settings value, Settings current, Settings previous) { HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH, HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT, HttpTransportSettings.SETTING_HTTP_RESET_COOKIES, - HttpTransportSettings.OLD_SETTING_HTTP_TCP_NO_DELAY, HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY, HttpTransportSettings.SETTING_HTTP_TCP_KEEP_ALIVE, HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS, @@ -309,32 +308,23 @@ public void apply(Settings value, Settings current, Settings previous) { TransportSettings.PUBLISH_HOST_PROFILE, TransportSettings.BIND_HOST, TransportSettings.BIND_HOST_PROFILE, - TransportSettings.OLD_PORT, TransportSettings.PORT, TransportSettings.PORT_PROFILE, TransportSettings.PUBLISH_PORT, TransportSettings.PUBLISH_PORT_PROFILE, - TransportSettings.OLD_TRANSPORT_COMPRESS, TransportSettings.TRANSPORT_COMPRESS, TransportSettings.PING_SCHEDULE, - TransportSettings.TCP_CONNECT_TIMEOUT, TransportSettings.CONNECT_TIMEOUT, TransportSettings.DEFAULT_FEATURES_SETTING, - TransportSettings.OLD_TCP_NO_DELAY, TransportSettings.TCP_NO_DELAY, - TransportSettings.OLD_TCP_NO_DELAY_PROFILE, TransportSettings.TCP_NO_DELAY_PROFILE, TransportSettings.TCP_KEEP_ALIVE, - TransportSettings.OLD_TCP_KEEP_ALIVE_PROFILE, TransportSettings.TCP_KEEP_ALIVE_PROFILE, TransportSettings.TCP_REUSE_ADDRESS, - TransportSettings.OLD_TCP_REUSE_ADDRESS_PROFILE, TransportSettings.TCP_REUSE_ADDRESS_PROFILE, TransportSettings.TCP_SEND_BUFFER_SIZE, - TransportSettings.OLD_TCP_SEND_BUFFER_SIZE_PROFILE, TransportSettings.TCP_SEND_BUFFER_SIZE_PROFILE, TransportSettings.TCP_RECEIVE_BUFFER_SIZE, - TransportSettings.OLD_TCP_RECEIVE_BUFFER_SIZE_PROFILE, TransportSettings.TCP_RECEIVE_BUFFER_SIZE_PROFILE, TransportSettings.CONNECTIONS_PER_NODE_RECOVERY, TransportSettings.CONNECTIONS_PER_NODE_BULK, @@ -352,7 +342,6 @@ public void apply(Settings value, Settings current, Settings previous) { NetworkService.TCP_REUSE_ADDRESS, NetworkService.TCP_SEND_BUFFER_SIZE, NetworkService.TCP_RECEIVE_BUFFER_SIZE, - NetworkService.TCP_CONNECT_TIMEOUT, IndexSettings.QUERY_STRING_ANALYZE_WILDCARD, IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD, ScriptService.SCRIPT_CACHE_SIZE_SETTING, diff --git a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 2a5639f2e72ce..58d30edf79944 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -107,10 +107,8 @@ public final class HttpTransportSettings { // Tcp socket settings - public static final Setting OLD_SETTING_HTTP_TCP_NO_DELAY = - boolSetting("http.tcp_no_delay", NetworkService.TCP_NO_DELAY, Setting.Property.NodeScope, Setting.Property.Deprecated); public static final Setting SETTING_HTTP_TCP_NO_DELAY = - boolSetting("http.tcp.no_delay", OLD_SETTING_HTTP_TCP_NO_DELAY, Setting.Property.NodeScope); + boolSetting("http.tcp.no_delay", NetworkService.TCP_NO_DELAY, Setting.Property.NodeScope); public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE = boolSetting("http.tcp.keep_alive", NetworkService.TCP_KEEP_ALIVE, Setting.Property.NodeScope); public static final Setting SETTING_HTTP_TCP_REUSE_ADDRESS = diff --git a/server/src/main/java/org/elasticsearch/transport/TransportSettings.java b/server/src/main/java/org/elasticsearch/transport/TransportSettings.java index 1dd5541b0e2e4..b89c06de84a19 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportSettings.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportSettings.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import static java.util.Collections.emptyList; @@ -52,84 +53,48 @@ public final class TransportSettings { listSetting("transport.bind_host", HOST, Function.identity(), Setting.Property.NodeScope); public static final Setting.AffixSetting> BIND_HOST_PROFILE = affixKeySetting("transport.profiles.", "bind_host", key -> listSetting(key, BIND_HOST, Function.identity(), Setting.Property.NodeScope)); - public static final Setting OLD_PORT = - new Setting<>("transport.tcp.port", "9300-9400", Function.identity(), Setting.Property.NodeScope, Setting.Property.Deprecated); public static final Setting PORT = - new Setting<>("transport.port", OLD_PORT, Function.identity(), Setting.Property.NodeScope); + new Setting<>("transport.port", "9300-9400", Function.identity(), Setting.Property.NodeScope); public static final Setting.AffixSetting PORT_PROFILE = affixKeySetting("transport.profiles.", "port", key -> new Setting<>(key, PORT, Function.identity(), Setting.Property.NodeScope)); public static final Setting PUBLISH_PORT = intSetting("transport.publish_port", -1, -1, Setting.Property.NodeScope); public static final Setting.AffixSetting PUBLISH_PORT_PROFILE = affixKeySetting("transport.profiles.", "publish_port", key -> intSetting(key, -1, -1, Setting.Property.NodeScope)); - public static final Setting OLD_TRANSPORT_COMPRESS = - boolSetting("transport.tcp.compress", false, Setting.Property.NodeScope, Setting.Property.Deprecated); public static final Setting TRANSPORT_COMPRESS = - boolSetting("transport.compress", OLD_TRANSPORT_COMPRESS, Setting.Property.NodeScope); + boolSetting("transport.compress", false, Setting.Property.NodeScope); // the scheduled internal ping interval setting, defaults to disabled (-1) public static final Setting PING_SCHEDULE = timeSetting("transport.ping_schedule", TimeValue.timeValueSeconds(-1), Setting.Property.NodeScope); - public static final Setting TCP_CONNECT_TIMEOUT = - timeSetting("transport.tcp.connect_timeout", NetworkService.TCP_CONNECT_TIMEOUT, Setting.Property.NodeScope, - Setting.Property.Deprecated); public static final Setting CONNECT_TIMEOUT = - timeSetting("transport.connect_timeout", TCP_CONNECT_TIMEOUT, Setting.Property.NodeScope); + timeSetting("transport.connect_timeout", new TimeValue(30, TimeUnit.SECONDS), Setting.Property.NodeScope); public static final Setting DEFAULT_FEATURES_SETTING = Setting.groupSetting(FEATURE_PREFIX + ".", Setting.Property.NodeScope); // Tcp socket settings - public static final Setting OLD_TCP_NO_DELAY = - boolSetting("transport.tcp_no_delay", NetworkService.TCP_NO_DELAY, Setting.Property.NodeScope, Setting.Property.Deprecated); public static final Setting TCP_NO_DELAY = - boolSetting("transport.tcp.no_delay", OLD_TCP_NO_DELAY, Setting.Property.NodeScope); - public static final Setting.AffixSetting OLD_TCP_NO_DELAY_PROFILE = - affixKeySetting("transport.profiles.", "tcp_no_delay", key -> boolSetting(key, TCP_NO_DELAY, Setting.Property.NodeScope, - Setting.Property.Deprecated)); + boolSetting("transport.tcp.no_delay", NetworkService.TCP_NO_DELAY, Setting.Property.NodeScope); public static final Setting.AffixSetting TCP_NO_DELAY_PROFILE = - affixKeySetting("transport.profiles.", "tcp.no_delay", - key -> boolSetting(key, - fallback(key, OLD_TCP_NO_DELAY_PROFILE, "tcp\\.no_delay$", "tcp_no_delay"), - Setting.Property.NodeScope)); + affixKeySetting("transport.profiles.", "tcp.no_delay", key -> boolSetting(key, TCP_NO_DELAY, Setting.Property.NodeScope)); public static final Setting TCP_KEEP_ALIVE = boolSetting("transport.tcp.keep_alive", NetworkService.TCP_KEEP_ALIVE, Setting.Property.NodeScope); - public static final Setting.AffixSetting OLD_TCP_KEEP_ALIVE_PROFILE = - affixKeySetting("transport.profiles.", "tcp_keep_alive", - key -> boolSetting(key, TCP_KEEP_ALIVE, Setting.Property.NodeScope, Setting.Property.Deprecated)); public static final Setting.AffixSetting TCP_KEEP_ALIVE_PROFILE = affixKeySetting("transport.profiles.", "tcp.keep_alive", - key -> boolSetting(key, - fallback(key, OLD_TCP_KEEP_ALIVE_PROFILE, "tcp\\.keep_alive$", "tcp_keep_alive"), - Setting.Property.NodeScope)); + key -> boolSetting(key, TCP_KEEP_ALIVE, Setting.Property.NodeScope)); public static final Setting TCP_REUSE_ADDRESS = boolSetting("transport.tcp.reuse_address", NetworkService.TCP_REUSE_ADDRESS, Setting.Property.NodeScope); - public static final Setting.AffixSetting OLD_TCP_REUSE_ADDRESS_PROFILE = - affixKeySetting("transport.profiles.", "reuse_address", key -> boolSetting(key, TCP_REUSE_ADDRESS, Setting.Property.NodeScope, - Setting.Property.Deprecated)); public static final Setting.AffixSetting TCP_REUSE_ADDRESS_PROFILE = - affixKeySetting("transport.profiles.", "tcp.reuse_address", - key -> boolSetting(key, - fallback(key, OLD_TCP_REUSE_ADDRESS_PROFILE, "tcp\\.reuse_address$", "reuse_address"), - Setting.Property.NodeScope)); + affixKeySetting("transport.profiles.", "tcp.reuse_address", key -> boolSetting(key, TCP_REUSE_ADDRESS, Setting.Property.NodeScope)); public static final Setting TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting("transport.tcp.send_buffer_size", NetworkService.TCP_SEND_BUFFER_SIZE, Setting.Property.NodeScope); - public static final Setting.AffixSetting OLD_TCP_SEND_BUFFER_SIZE_PROFILE = - affixKeySetting("transport.profiles.", "send_buffer_size", - key -> Setting.byteSizeSetting(key, TCP_SEND_BUFFER_SIZE, Setting.Property.NodeScope, Setting.Property.Deprecated)); public static final Setting.AffixSetting TCP_SEND_BUFFER_SIZE_PROFILE = affixKeySetting("transport.profiles.", "tcp.send_buffer_size", - key -> Setting.byteSizeSetting(key, - fallback(key, OLD_TCP_SEND_BUFFER_SIZE_PROFILE, "tcp\\.send_buffer_size$", "send_buffer_size"), - Setting.Property.NodeScope)); + key -> Setting.byteSizeSetting(key, TCP_SEND_BUFFER_SIZE, Setting.Property.NodeScope)); public static final Setting TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting("transport.tcp.receive_buffer_size", NetworkService.TCP_RECEIVE_BUFFER_SIZE, Setting.Property.NodeScope); - public static final Setting.AffixSetting OLD_TCP_RECEIVE_BUFFER_SIZE_PROFILE = - affixKeySetting("transport.profiles.", "receive_buffer_size", - key -> Setting.byteSizeSetting(key, TCP_RECEIVE_BUFFER_SIZE, Setting.Property.NodeScope, Setting.Property.Deprecated)); public static final Setting.AffixSetting TCP_RECEIVE_BUFFER_SIZE_PROFILE = affixKeySetting("transport.profiles.", "tcp.receive_buffer_size", - key -> Setting.byteSizeSetting(key, - fallback(key, OLD_TCP_RECEIVE_BUFFER_SIZE_PROFILE, "tcp\\.receive_buffer_size$", "receive_buffer_size"), - Setting.Property.NodeScope)); + key -> Setting.byteSizeSetting(key, TCP_RECEIVE_BUFFER_SIZE, Setting.Property.NodeScope)); // Connections per node settings diff --git a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash index 13aed28b4c1ba..6cc6fab3d4d1e 100644 --- a/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash +++ b/x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash @@ -345,7 +345,7 @@ xpack.security.http.ssl.certificate: $ESCONFIG/certs//node-data/node-data.crt xpack.security.http.ssl.certificate_authorities: ["$ESCONFIG/certs/ca/ca.crt"] xpack.security.transport.ssl.enabled: true -transport.tcp.port: 9301 +transport.port: 9301 xpack.security.http.ssl.enabled: true http.port: 9201 From c86e7f17e27624bb5f76f2c1061e99500d55d7bb Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 14 Apr 2019 11:29:51 +0200 Subject: [PATCH 03/15] Simplify TransportShardBulkAction#performOnReplica (#41065) * Simplify TransortShardBulkAction#performOnReplica * Resolve TODO since 8.0 doesn't have to worry about pre 6.x nodes * Remove test for removed method since the logic is now completely internal to `performOnReplica` --- .../action/bulk/TransportShardBulkAction.java | 86 ++++--------------- .../bulk/TransportShardBulkActionTests.java | 43 ---------- 2 files changed, 17 insertions(+), 112 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index da30dedfe5e60..f49521a37463e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -338,9 +338,8 @@ private static boolean isConflictException(final Exception e) { /** * Creates a new bulk item result from the given requests and result of performing the update operation on the shard. */ - static BulkItemResponse processUpdateResponse(final UpdateRequest updateRequest, final String concreteIndex, - BulkItemResponse operationResponse, - final UpdateHelper.Result translate) { + private static BulkItemResponse processUpdateResponse(final UpdateRequest updateRequest, final String concreteIndex, + BulkItemResponse operationResponse, final UpdateHelper.Result translate) { final BulkItemResponse response; DocWriteResponse.Result translatedResult = translate.getResponseResult(); if (operationResponse.isFailed()) { @@ -382,54 +381,6 @@ static BulkItemResponse processUpdateResponse(final UpdateRequest updateRequest, return response; } - - /** Modes for executing item request on replica depending on corresponding primary execution result */ - public enum ReplicaItemExecutionMode { - - /** - * When primary execution succeeded - */ - NORMAL, - - /** - * When primary execution failed before sequence no was generated - * or primary execution was a noop (only possible when request is originating from pre-6.0 nodes) - */ - NOOP, - - /** - * When primary execution failed after sequence no was generated - */ - FAILURE - } - - /** - * Determines whether a bulk item request should be executed on the replica. - * - * @return {@link ReplicaItemExecutionMode#NORMAL} upon normal primary execution with no failures - * {@link ReplicaItemExecutionMode#FAILURE} upon primary execution failure after sequence no generation - * {@link ReplicaItemExecutionMode#NOOP} upon primary execution failure before sequence no generation or - * when primary execution resulted in noop (only possible for write requests from pre-6.0 nodes) - */ - static ReplicaItemExecutionMode replicaItemExecutionMode(final BulkItemRequest request, final int index) { - final BulkItemResponse primaryResponse = request.getPrimaryResponse(); - assert primaryResponse != null : "expected primary response to be set for item [" + index + "] request [" + request.request() + "]"; - if (primaryResponse.isFailed()) { - return primaryResponse.getFailure().getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO - ? ReplicaItemExecutionMode.FAILURE // we have a seq no generated with the failure, replicate as no-op - : ReplicaItemExecutionMode.NOOP; // no seq no generated, ignore replication - } else { - // TODO: once we know for sure that every operation that has been processed on the primary is assigned a seq# - // (i.e., all nodes on the cluster are on v6.0.0 or higher) we can use the existence of a seq# to indicate whether - // an operation should be processed or be treated as a noop. This means we could remove this method and the - // ReplicaItemExecutionMode enum and have a simple boolean check for seq != UNASSIGNED_SEQ_NO which will work for - // both failures and indexing operations. - return primaryResponse.getResponse().getResult() != DocWriteResponse.Result.NOOP - ? ReplicaItemExecutionMode.NORMAL // execution successful on primary - : ReplicaItemExecutionMode.NOOP; // ignore replication - } - } - @Override public WriteReplicaResult shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception { final Translog.Location location = performOnReplica(request, replica); @@ -442,25 +393,22 @@ public static Translog.Location performOnReplica(BulkShardRequest request, Index BulkItemRequest item = request.items()[i]; final Engine.Result operationResult; DocWriteRequest docWriteRequest = item.request(); - switch (replicaItemExecutionMode(item, i)) { - case NORMAL: - final DocWriteResponse primaryResponse = item.getPrimaryResponse().getResponse(); - operationResult = performOpOnReplica(primaryResponse, docWriteRequest, replica); - assert operationResult != null : "operation result must never be null when primary response has no failure"; - location = syncOperationResultOrThrow(operationResult, location); - break; - case NOOP: - break; - case FAILURE: - final BulkItemResponse.Failure failure = item.getPrimaryResponse().getFailure(); - assert failure.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "seq no must be assigned"; - operationResult = replica.markSeqNoAsNoop(failure.getSeqNo(), failure.getMessage()); - assert operationResult != null : "operation result must never be null when primary response has no failure"; - location = syncOperationResultOrThrow(operationResult, location); - break; - default: - throw new IllegalStateException("illegal replica item execution mode for: " + docWriteRequest); + final BulkItemResponse response = item.getPrimaryResponse(); + final BulkItemResponse.Failure failure = response.getFailure(); + final DocWriteResponse writeResponse = response.getResponse(); + final long seqNum = failure == null ? writeResponse.getSeqNo() : failure.getSeqNo(); + if (seqNum == SequenceNumbers.UNASSIGNED_SEQ_NO) { + assert failure != null || writeResponse.getResult() == DocWriteResponse.Result.NOOP + || writeResponse.getResult() == DocWriteResponse.Result.NOT_FOUND; + continue; + } + if (failure == null) { + operationResult = performOpOnReplica(writeResponse, docWriteRequest, replica); + } else { + operationResult = replica.markSeqNoAsNoop(seqNum, failure.getMessage()); } + assert operationResult != null : "operation result must never be null when primary response has no failure"; + location = syncOperationResultOrThrow(operationResult, location); } return location; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java index 62217f7873138..55078840153f2 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.LatchedActionListener; -import org.elasticsearch.action.bulk.TransportShardBulkAction.ReplicaItemExecutionMode; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; @@ -59,7 +58,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.action.bulk.TransportShardBulkAction.replicaItemExecutionMode; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.notNullValue; @@ -96,47 +94,6 @@ private IndexMetaData indexMetaData() throws IOException { .primaryTerm(0, 1).build(); } - public void testShouldExecuteReplicaItem() throws Exception { - // Successful index request should be replicated - DocWriteRequest writeRequest = new IndexRequest("index", "_doc", "id") - .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); - DocWriteResponse response = new IndexResponse(shardId, "type", "id", 1, 17, 1, randomBoolean()); - BulkItemRequest request = new BulkItemRequest(0, writeRequest); - request.setPrimaryResponse(new BulkItemResponse(0, DocWriteRequest.OpType.INDEX, response)); - assertThat(replicaItemExecutionMode(request, 0), - equalTo(ReplicaItemExecutionMode.NORMAL)); - - // Failed index requests without sequence no should not be replicated - writeRequest = new IndexRequest("index", "_doc", "id") - .source(Requests.INDEX_CONTENT_TYPE, "foo", "bar"); - request = new BulkItemRequest(0, writeRequest); - request.setPrimaryResponse( - new BulkItemResponse(0, DocWriteRequest.OpType.INDEX, - new BulkItemResponse.Failure("index", "type", "id", - new IllegalArgumentException("i died")))); - assertThat(replicaItemExecutionMode(request, 0), - equalTo(ReplicaItemExecutionMode.NOOP)); - - // Failed index requests with sequence no should be replicated - request = new BulkItemRequest(0, writeRequest); - request.setPrimaryResponse( - new BulkItemResponse(0, DocWriteRequest.OpType.INDEX, - new BulkItemResponse.Failure("index", "type", "id", - new IllegalArgumentException( - "i died after sequence no was generated"), - 1))); - assertThat(replicaItemExecutionMode(request, 0), - equalTo(ReplicaItemExecutionMode.FAILURE)); - // NOOP requests should not be replicated - DocWriteRequest updateRequest = new UpdateRequest("index", "type", "id"); - response = new UpdateResponse(shardId, "type", "id", 1, DocWriteResponse.Result.NOOP); - request = new BulkItemRequest(0, updateRequest); - request.setPrimaryResponse(new BulkItemResponse(0, DocWriteRequest.OpType.UPDATE, - response)); - assertThat(replicaItemExecutionMode(request, 0), - equalTo(ReplicaItemExecutionMode.NOOP)); - } - public void testExecuteBulkIndexRequest() throws Exception { IndexShard shard = newStartedShard(true); From f0fac9f56b2877bd1f9663172e92d3dfc371d292 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 14 Apr 2019 10:39:50 +0100 Subject: [PATCH 04/15] Further clarify cluster.initial_master_nodes (#41179) The following phrase causes confusion: > Alternatively the IP addresses or hostnames (if node name defaults to the > host name) can be used. This change clarifies the conditions under which you can use a hostname, and adds an anchor to the note introduced in (#41137) so we can link directly to it in conversations with users. --- .../modules/discovery/bootstrapping.asciidoc | 62 ++++++++++--------- .../discovery-settings.asciidoc | 14 +++-- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index f4a775e12df99..0ba7d4b17cef7 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -9,16 +9,21 @@ up: nodes that have already joined a cluster store this information in their data folder and freshly-started nodes that are joining an existing cluster obtain this information from the cluster's elected master. -The initial set of master-eligible nodes is defined in the -<>. When you -start a master-eligible node, you can provide this setting on the command line -or in the `elasticsearch.yml` file. After the cluster has formed, this setting -is no longer required and is ignored. It need not be set -on master-ineligible nodes, nor on master-eligible nodes that are started to -join an existing cluster. Note that master-eligible nodes should use storage -that persists across restarts. If they do not, and -`cluster.initial_master_nodes` is set, and a full cluster restart occurs, then -another brand-new cluster will form and this may result in data loss. +The initial set of master-eligible nodes is defined in the +<>. This is a list +of the <> or IP addresses of the master-eligible nodes in +the new cluster. If you do not configure `node.name` then it is set to the +node's hostname, so in this case you can use hostnames in +`cluster.initial_master_nodes` too. + +When you start a master-eligible node, you can provide this setting on the +command line or in the `elasticsearch.yml` file. After the cluster has formed, +this setting is no longer required and is ignored. It need not be set on +master-ineligible nodes, nor on master-eligible nodes that are started to join +an existing cluster. Note that master-eligible nodes should use storage that +persists across restarts. If they do not, and `cluster.initial_master_nodes` is +set, and a full cluster restart occurs, then another brand-new cluster will +form and this may result in data loss. It is technically sufficient to set `cluster.initial_master_nodes` on a single master-eligible node in the cluster, and only to mention that single node in the @@ -42,10 +47,9 @@ cluster.initial_master_nodes: - master-c -------------------------------------------------- -Alternatively the IP addresses or hostnames (<>) can be used. If there is more than one Elasticsearch node -with the same IP address or hostname then the transport ports must also be given -to specify exactly which node is meant: +You can use a mix of IP addresses and node names too. If there is more than one +Elasticsearch node with the same IP address then the transport port must also +be given to specify exactly which node is meant: [source,yaml] -------------------------------------------------- @@ -56,14 +60,23 @@ cluster.initial_master_nodes: - master-node-hostname -------------------------------------------------- +Like all node settings, it is also possible to specify the initial set of master +nodes on the command-line that is used to start Elasticsearch: + +[source,bash] +-------------------------------------------------- +$ bin/elasticsearch -Ecluster.initial_master_nodes=master-a,master-b,master-c +-------------------------------------------------- + [NOTE] ================================================== -The node names used in this list must exactly match the `node.name` properties -of the nodes. By default the node name is set to the machine's hostname which -may or may not be fully-qualified depending on your system configuration. If -each node name is a fully-qualified domain name such as `master-a.example.com` -then you must use fully-qualified domain names in the +[[modules-discovery-bootstrap-cluster-fqdns]] The node names used in the +`cluster.initial_master_nodes` list must exactly match the `node.name` +properties of the nodes. By default the node name is set to the machine's +hostname which may or may not be fully-qualified depending on your system +configuration. If each node name is a fully-qualified domain name such as +`master-a.example.com` then you must use fully-qualified domain names in the `cluster.initial_master_nodes` list too; conversely if your node names are bare hostnames (without the `.example.com` suffix) then you must use bare hostnames in the `cluster.initial_master_nodes` list. If you use a mix of fully-qualifed @@ -81,18 +94,11 @@ bootstrap a cluster: have discovered [{master-b.example.com}{... This message shows the node names `master-a.example.com` and `master-b.example.com` as well as the `cluster.initial_master_nodes` entries -`master-a` and `master-b`, and it is apparent that they do not match exactly. +`master-a` and `master-b`, and it is clear from this message that they do not +match exactly. ================================================== -Like all node settings, it is also possible to specify the initial set of master -nodes on the command-line that is used to start Elasticsearch: - -[source,bash] --------------------------------------------------- -$ bin/elasticsearch -Ecluster.initial_master_nodes=master-a,master-b,master-c --------------------------------------------------- - [float] ==== Choosing a cluster name diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc index 4edf5cfbab5e9..5709ae3bb9345 100644 --- a/docs/reference/setup/important-settings/discovery-settings.asciidoc +++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc @@ -58,15 +58,17 @@ cluster.initial_master_nodes: `transport.port` if not specified. <2> If a hostname resolves to multiple IP addresses then the node will attempt to discover other nodes at all resolved addresses. -<3> Initial master nodes can be identified by their <>. - Make sure that the value here matches the `node.name` exactly. If you use a - fully-qualified domain name such as `master-node-a.example.com` for your +<3> Initial master nodes can be identified by their <>, + which defaults to the hostname. Make sure that the value in + `cluster.initial_master_nodes` matches the `node.name` exactly. If you use + a fully-qualified domain name such as `master-node-a.example.com` for your node names then you must use the fully-qualified name in this list; conversely if `node.name` is a bare hostname without any trailing qualifiers then you must also omit the trailing qualifiers in `cluster.initial_master_nodes`. <4> Initial master nodes can also be identified by their IP address. -<5> If multiple master nodes share an IP address then the port must be used to - disambiguate them. +<5> If multiple master nodes share an IP address then the transport port must + be used to distinguish between them. -For more information, see <>. +For more information, see <> and +<>. From 7543478fafe075c6b5bd7f676b78968811563929 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 15 Apr 2019 09:28:58 +0200 Subject: [PATCH 05/15] Remove xpack dependencies from qa rest modules (#41134) This commit removes xpack dependencies of many xpack qa modules. (for some qa modules this will require some more work) The reason behind this change is that qa rest modules should not depend on the x-pack plugins, because the plugins are an implementation detail and the tests should only know about the rest interface and qa cluster that is being tested. Also some qa modules rely on xpack plugins and hlrc (which is a valid dependency for rest qa tests) creates a cyclic dependency and this is something that we should avoid. Also Eclipse can't handle gradle cyclic dependencies (see #41064). * don't copy xpack-core's plugin property into the test resource of qa modules. Otherwise installing security manager fails, because it tries to find the XPackPlugin class. --- .../support}/WatcherTemplateTests.java | 2 +- x-pack/qa/build.gradle | 7 ++++ .../build.gradle | 3 +- ...CoreWithSecurityClientYamlTestSuiteIT.java | 2 +- x-pack/qa/full-cluster-restart/build.gradle | 27 +++--------- .../xpack/restart/FullClusterRestartIT.java | 35 +++++++--------- .../MlMigrationFullClusterRestartIT.java | 3 +- .../build.gradle | 4 +- ...sterSearchWithSecurityYamlTestSuiteIT.java | 2 +- x-pack/qa/multi-node/build.gradle | 2 +- .../GlobalCheckpointSyncActionIT.java | 2 +- .../elasticsearch/multi_node/RollupIT.java | 2 +- x-pack/qa/rolling-upgrade-basic/build.gradle | 13 +----- .../build.gradle | 4 +- x-pack/qa/rolling-upgrade/build.gradle | 14 +------ .../upgrades/AbstractUpgradeTestCase.java | 6 +-- .../upgrades/MlMappingsUpgradeIT.java | 6 +-- .../UpgradeClusterClientYamlTestSuiteIT.java | 3 +- x-pack/qa/saml-idp-tests/build.gradle | 7 +--- .../build.gradle | 4 +- .../MonitoringWithWatcherRestIT.java | 33 ++++++++------- x-pack/qa/smoke-test-plugins/build.gradle | 2 +- ...SmokeTestPluginsClientYamlTestSuiteIT.java | 2 +- .../build.gradle | 5 +-- ...rityWithMustacheClientYamlTestSuiteIT.java | 2 +- .../build.gradle | 2 +- ...cherWithSecurityClientYamlTestSuiteIT.java | 6 +-- .../SmokeTestWatcherWithSecurityIT.java | 6 +-- x-pack/qa/smoke-test-watcher/build.gradle | 5 +-- .../SmokeTestWatcherTestSuiteIT.java | 6 +-- .../smoketest/WatcherRestIT.java | 4 +- .../test/SecuritySettingsSourceField.java | 42 +++++++++++++++++++ .../test/rest/XPackRestTestConstants.java | 38 +++++++++++++++++ .../xpack/test/rest/XPackRestTestHelper.java | 19 +++------ 34 files changed, 173 insertions(+), 147 deletions(-) rename x-pack/{qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest => plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support}/WatcherTemplateTests.java (99%) create mode 100644 x-pack/qa/src/main/java/org/elasticsearch/xpack/test/SecuritySettingsSourceField.java create mode 100644 x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java rename x-pack/{plugin/core/src/test => qa/src/main}/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java (81%) diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java similarity index 99% rename from x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java rename to x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java index df98e73118711..8c45bc15b4c9e 100644 --- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.smoketest; +package org.elasticsearch.xpack.watcher.support; import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.elasticsearch.common.Nullable; diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle index 7d705d5b0dc9f..2555b0ef729dc 100644 --- a/x-pack/qa/build.gradle +++ b/x-pack/qa/build.gradle @@ -3,6 +3,13 @@ import org.elasticsearch.gradle.test.RestIntegTestTask +apply plugin: 'elasticsearch.build' +test.enabled = false + +dependencies { + compile project(':test:framework') +} + subprojects { // HACK: please fix this // we want to add the rest api specs for xpack to qa tests, but we diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index dbf5aa6fc227c..72b62f94b5542 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -2,8 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('security'), configuration: 'testArtifacts') + testCompile project(':x-pack:qa') } integTest { diff --git a/x-pack/qa/core-rest-tests-with-security/src/test/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/test/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java index 212a342479d3f..1d98c4fd59c76 100644 --- a/x-pack/qa/core-rest-tests-with-security/src/test/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/core-rest-tests-with-security/src/test/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java @@ -18,7 +18,7 @@ import java.util.Objects; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 964cc2fb43cc7..7f0e14d2a53bf 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -10,22 +10,12 @@ apply plugin: 'elasticsearch.standalone-test' test.enabled = false dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile (project(path: xpackModule('security'), configuration: 'runtime')) { - // Need to drop the guava dependency here or we get a conflict with watcher's guava dependency. - // This is total #$%, but the solution is to get the SAML realm (which uses guava) out of security proper - exclude group: "com.google.guava", module: "guava" - } - testCompile project(path: xpackModule('watcher'), configuration: 'runtime') - + // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core. + // Currently needed for ml tests are using the building for datafeed and job config) testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile (project(path: xpackModule('security'), configuration: 'testArtifacts')) { - // Need to drop the guava dependency here or we get a conflict with watcher's guava dependency. - // This is total #$%, but the solution is to get the SAML realm (which uses guava) out of security proper - exclude group: "com.google.guava", module: "guava" - } + testCompile project(path: ':qa:full-cluster-restart', configuration: 'testArtifacts') + testCompile project(':x-pack:qa') } Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> @@ -225,11 +215,4 @@ task copyXPackRestSpec(type: Copy) { include 'rest-api-spec/api/**' into project.sourceSets.test.output.resourcesDir } - -task copyXPackPluginProps(type: Copy) { - dependsOn(copyXPackRestSpec) - from project(xpackModule('core')).file('src/main/plugin-metadata') - from project(xpackModule('core')).tasks.pluginProperties - into outputDir -} -project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps) +project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackRestSpec) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 29f3237dcedb5..7ed204e152b64 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -13,22 +13,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.ObjectPath; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.test.StreamsUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; -import org.elasticsearch.xpack.core.upgrade.UpgradeField; -import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; -import org.elasticsearch.xpack.watcher.actions.index.IndexAction; -import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction; -import org.elasticsearch.xpack.watcher.common.text.TextTemplate; -import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition; -import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; -import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger; import org.hamcrest.Matcher; import java.io.IOException; @@ -54,6 +44,12 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { + public static final String INDEX_ACTION_TYPES_DEPRECATION_MESSAGE = + "[types removal] Specifying types in a watcher index action is deprecated."; + + public static final int UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION = 6; + public static final int SECURITY_EXPECTED_INDEX_FORMAT_VERSION = 6; + @Override protected Settings restClientSettings() { String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); @@ -106,7 +102,7 @@ public void testSecurityNativeRealm() throws Exception { if (settingsMap.containsKey("index")) { @SuppressWarnings("unchecked") int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The security index needs to be upgraded", SecurityIndexManager.INTERNAL_INDEX_FORMAT, format); + assertEquals("The security index needs to be upgraded", SECURITY_EXPECTED_INDEX_FORMAT_VERSION, format); } } @@ -127,8 +123,8 @@ public void testWatcher() throws Exception { Request createBwcWatch = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_watch"); Request createBwcThrottlePeriod = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_throttle_period"); if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) { - createBwcWatch.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE)); - createBwcThrottlePeriod.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE)); + createBwcWatch.setOptions(expectWarnings(INDEX_ACTION_TYPES_DEPRECATION_MESSAGE)); + createBwcThrottlePeriod.setOptions(expectWarnings(INDEX_ACTION_TYPES_DEPRECATION_MESSAGE)); } createBwcWatch.setJsonEntity(loadWatch("simple-watch.json")); client().performRequest(createBwcWatch); @@ -166,7 +162,7 @@ public void testWatcher() throws Exception { logger.info("settings map {}", settingsMap); if (settingsMap.containsKey("index")) { int format = Integer.parseInt(String.valueOf(((Map)settingsMap.get("index")).get("format"))); - assertEquals("The watches index needs to be upgraded", UpgradeField.EXPECTED_INDEX_FORMAT_VERSION, format); + assertEquals("The watches index needs to be upgraded", UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION, format); } } @@ -289,7 +285,7 @@ private void assertWatchIndexContentsWork() throws Exception { Request getRequest = new Request("GET", "_watcher/watch/bwc_watch"); getRequest.setOptions( expectWarnings( - IndexAction.TYPES_DEPRECATION_MESSAGE + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE ) ); @@ -310,7 +306,7 @@ private void assertWatchIndexContentsWork() throws Exception { getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period"); getRequest.setOptions( expectWarnings( - IndexAction.TYPES_DEPRECATION_MESSAGE + INDEX_ACTION_TYPES_DEPRECATION_MESSAGE ) ); @@ -352,10 +348,9 @@ private void assertWatchIndexContentsWork() throws Exception { private void assertBasicWatchInteractions() throws Exception { - String watch = new WatchSourceBuilder() - .condition(InternalAlwaysCondition.INSTANCE) - .trigger(ScheduleTrigger.builder(new IntervalSchedule(IntervalSchedule.Interval.seconds(1)))) - .addAction("awesome", LoggingAction.builder(new TextTemplate("test"))).buildAsBytes(XContentType.JSON).utf8ToString(); + String watch = "{\"trigger\":{\"schedule\":{\"interval\":\"1s\"}},\"input\":{\"none\":{}}," + + "\"condition\":{\"always\":{}}," + + "\"actions\":{\"awesome\":{\"logging\":{\"level\":\"info\",\"text\":\"test\"}}}}"; Request createWatchRequest = new Request("PUT", "_watcher/watch/new_watch"); createWatchRequest.setJsonEntity(watch); Map createWatch = entityAsMap(client().performRequest(createWatchRequest)); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java index 917d73d5af2c4..3c849811f5620 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.junit.Before; @@ -56,7 +57,7 @@ protected Settings restClientSettings() { @Before public void waitForMlTemplates() throws Exception { - List templatesToWaitFor = XPackRestTestHelper.ML_POST_V660_TEMPLATES; + List templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES; XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor); } diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle index c31b2c0ad1d5e..8406345575dfd 100644 --- a/x-pack/qa/multi-cluster-search-security/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/build.gradle @@ -3,9 +3,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(':x-pack:qa') } task remoteClusterTest(type: RestIntegTestTask) { diff --git a/x-pack/qa/multi-cluster-search-security/src/test/java/org/elasticsearch/xpack/security/MultiClusterSearchWithSecurityYamlTestSuiteIT.java b/x-pack/qa/multi-cluster-search-security/src/test/java/org/elasticsearch/xpack/security/MultiClusterSearchWithSecurityYamlTestSuiteIT.java index e61ff9062d171..011fe5be92803 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/java/org/elasticsearch/xpack/security/MultiClusterSearchWithSecurityYamlTestSuiteIT.java +++ b/x-pack/qa/multi-cluster-search-security/src/test/java/org/elasticsearch/xpack/security/MultiClusterSearchWithSecurityYamlTestSuiteIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; public class MultiClusterSearchWithSecurityYamlTestSuiteIT extends ESClientYamlSuiteTestCase { diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle index 243a6f40438cc..28de4a4c114a9 100644 --- a/x-pack/qa/multi-node/build.gradle +++ b/x-pack/qa/multi-node/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" + testCompile project(':x-pack:qa') } integTestCluster { diff --git a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java index 9d3e88cbc5c8d..02dd7ecb40cc1 100644 --- a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java +++ b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.test.rest.yaml.ObjectPath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; import static org.hamcrest.Matchers.equalTo; public class GlobalCheckpointSyncActionIT extends ESRestTestCase { diff --git a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java index feddc57f6e014..a5579ad0aa5d8 100644 --- a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java +++ b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.isOneOf; diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index bb28821b96fc0..28b6428146e45 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -4,9 +4,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit + testCompile project(':x-pack:qa') } // This is a top level task which we will add dependencies to below. @@ -141,14 +139,7 @@ task copyXPackRestSpec(type: Copy) { include 'rest-api-spec/api/**' into project.sourceSets.test.output.resourcesDir } - -task copyXPackPluginProps(type: Copy) { - dependsOn(copyXPackRestSpec) - from project(xpackModule('core')).file('src/main/plugin-metadata') - from project(xpackModule('core')).tasks.pluginProperties - into outputDir -} -project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps) +project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackRestSpec) repositories { maven { diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index 19e1f5a716c08..ade76e361e6c4 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -4,9 +4,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit + testCompile project(':x-pack:qa') } // This is a top level task which we will add dependencies to below. diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index b3d6bbd221d56..53810b89386f1 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -10,10 +10,7 @@ apply plugin: 'elasticsearch.standalone-test' test.enabled = false dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('security'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') // to be moved in a later commit + testCompile project(':x-pack:qa') testCompile ("org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}") } @@ -307,11 +304,4 @@ task copyXPackRestSpec(type: Copy) { include 'rest-api-spec/api/**' into project.sourceSets.test.output.resourcesDir } - -task copyXPackPluginProps(type: Copy) { - dependsOn(copyXPackRestSpec) - from project(xpackModule('core')).file('src/main/plugin-metadata') - from project(xpackModule('core')).tasks.pluginProperties - into outputDir -} -project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps) +project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackRestSpec) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index 0d0f05bcf9c6d..64c3a785d14e4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -8,7 +8,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.xpack.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; @@ -16,12 +16,12 @@ import java.util.Collection; import java.util.Collections; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; public abstract class AbstractUpgradeTestCase extends ESRestTestCase { private static final String BASIC_AUTH_VALUE = - basicAuthHeaderValue("test_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); + basicAuthHeaderValue("test_user", SecuritySettingsSourceField.TEST_PASSWORD); @Override protected boolean preserveIndicesUponCompletion() { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 4bded9a25c56c..13ed2dafc5f31 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.client.ml.job.config.Job; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import java.io.IOException; @@ -30,7 +30,7 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase { @Override protected Collection templatesToWaitFor() { - return Stream.concat(XPackRestTestHelper.ML_POST_V660_TEMPLATES.stream(), + return Stream.concat(XPackRestTestConstants.ML_POST_V660_TEMPLATES.stream(), super.templatesToWaitFor().stream()).collect(Collectors.toSet()); } @@ -81,7 +81,7 @@ private void createAndOpenTestJob() throws IOException { private void assertUpgradedMappings() throws Exception { assertBusy(() -> { - Request getMappings = new Request("GET", AnomalyDetectorsIndex.resultsWriteAlias(JOB_ID) + "/_mappings"); + Request getMappings = new Request("GET", XPackRestTestHelper.resultsWriteAlias(JOB_ID) + "/_mappings"); Response response = client().performRequest(getMappings); Map responseLevel = entityAsMap(response); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 9374346449c95..33f256ac1e198 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.junit.Before; @@ -28,7 +29,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa */ @Before public void waitForTemplates() throws Exception { - XPackRestTestHelper.waitForTemplates(client(), XPackRestTestHelper.ML_POST_V660_TEMPLATES); + XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V660_TEMPLATES); } @Override diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle index 7b76321fe9d4f..4355ac0b5b825 100644 --- a/x-pack/qa/saml-idp-tests/build.gradle +++ b/x-pack/qa/saml-idp-tests/build.gradle @@ -6,10 +6,7 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.test.fixtures' dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('security'), configuration: 'testArtifacts') testCompile 'com.google.jimfs:jimfs:1.1' } testFixtures.useFixture ":x-pack:test:idp-fixture" @@ -103,9 +100,7 @@ thirdPartyAudit { 'com.google.common.cache.Striped64$1', 'com.google.common.cache.Striped64$Cell', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', - 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', - 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', - 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1' + 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1' ) ignoreMissingClasses ( diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle index 57be337f634f2..18bf5656f1991 100644 --- a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle +++ b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle @@ -2,9 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackModule('watcher')) - testCompile project(path: xpackModule('monitoring')) + testCompile project(':x-pack:qa') } integTestCluster { diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java index 66b52afed1652..40ed71df842f3 100644 --- a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java +++ b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java @@ -9,28 +9,32 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; -import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; -import org.elasticsearch.xpack.watcher.actions.ActionBuilders; -import org.elasticsearch.xpack.watcher.client.WatchSourceBuilders; -import org.elasticsearch.xpack.watcher.trigger.TriggerBuilders; -import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule; import org.junit.After; import java.io.IOException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput; -import static org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule.Interval.Unit.MINUTES; import static org.hamcrest.Matchers.is; @TestLogging("org.elasticsearch.client:TRACE,tracer:TRACE") @AwaitsFix(bugUrl = "flaky tests") public class MonitoringWithWatcherRestIT extends ESRestTestCase { + /** + * An unsorted list of Watch IDs representing resource files for Monitoring Cluster Alerts. + */ + public static final String[] WATCH_IDS = { + "elasticsearch_cluster_status", + "elasticsearch_version_mismatch", + "kibana_version_mismatch", + "logstash_version_mismatch", + "xpack_license_expiration", + "elasticsearch_nodes", + }; + @After public void cleanExporters() throws Exception { Request request = new Request("PUT", "/_cluster/settings"); @@ -53,7 +57,7 @@ public void testThatLocalExporterAddsWatches() throws Exception { .endObject().endObject())); adminClient().performRequest(request); - assertTotalWatchCount(ClusterAlertsUtil.WATCH_IDS.length); + assertTotalWatchCount(WATCH_IDS.length); assertMonitoringWatchHasBeenOverWritten(watchId); } @@ -71,7 +75,7 @@ public void testThatHttpExporterAddsWatches() throws Exception { .endObject().endObject())); adminClient().performRequest(request); - assertTotalWatchCount(ClusterAlertsUtil.WATCH_IDS.length); + assertTotalWatchCount(WATCH_IDS.length); assertMonitoringWatchHasBeenOverWritten(watchId); } @@ -95,11 +99,10 @@ private String createMonitoringWatch() throws Exception { String clusterUUID = getClusterUUID(); String watchId = clusterUUID + "_kibana_version_mismatch"; Request request = new Request("PUT", "/_watcher/watch/" + watchId); - request.setJsonEntity(WatchSourceBuilders.watchBuilder() - .trigger(TriggerBuilders.schedule(new IntervalSchedule(new IntervalSchedule.Interval(1000, MINUTES)))) - .input(simpleInput()) - .addAction("logme", ActionBuilders.loggingAction("foo")) - .buildAsBytes(XContentType.JSON).utf8ToString()); + String watch = "{\"trigger\":{\"schedule\":{\"interval\":\"1000m\"}},\"input\":{\"simple\":{}}," + + "\"condition\":{\"always\":{}}," + + "\"actions\":{\"logme\":{\"logging\":{\"level\":\"info\",\"text\":\"foo\"}}}}"; + request.setJsonEntity(watch); client().performRequest(request); return watchId; } diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle index 3b7661eeeb05a..5aa3adbdf09f5 100644 --- a/x-pack/qa/smoke-test-plugins/build.gradle +++ b/x-pack/qa/smoke-test-plugins/build.gradle @@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" + testCompile project(':x-pack:qa') } ext.pluginsCount = 0 diff --git a/x-pack/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/XSmokeTestPluginsClientYamlTestSuiteIT.java b/x-pack/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/XSmokeTestPluginsClientYamlTestSuiteIT.java index 29671386f5ba0..6a676be7430df 100644 --- a/x-pack/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/XSmokeTestPluginsClientYamlTestSuiteIT.java +++ b/x-pack/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/XSmokeTestPluginsClientYamlTestSuiteIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; public class XSmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle index 48b525ba3dae9..bf2e6c325730e 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle +++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle @@ -2,10 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here - testCompile project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') + testCompile project(':x-pack:qa') } integTestCluster { diff --git a/x-pack/qa/smoke-test-security-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestSecurityWithMustacheClientYamlTestSuiteIT.java b/x-pack/qa/smoke-test-security-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestSecurityWithMustacheClientYamlTestSuiteIT.java index 46daddae62b0d..f65fd64be29f5 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestSecurityWithMustacheClientYamlTestSuiteIT.java +++ b/x-pack/qa/smoke-test-security-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestSecurityWithMustacheClientYamlTestSuiteIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; public class SmokeTestSecurityWithMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle index 2ff2ff9272fa5..0b622fc446b38 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle +++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" + testCompile project(':x-pack:qa') } // bring in watcher rest test suite diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java index 879be233fa180..679bc08f01f38 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java +++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityClientYamlTestSuiteIT.java @@ -14,7 +14,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.junit.After; import org.junit.Before; @@ -23,7 +23,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; import static org.hamcrest.Matchers.is; public class SmokeTestWatcherWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -76,7 +76,7 @@ public void startWatcher() throws Exception { }); assertBusy(() -> { - for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES_NO_ILM) { + for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) { ClientYamlTestResponse templateExistsResponse = getAdminExecutionContext().callApi("indices.exists_template", singletonMap("name", template), emptyList(), emptyMap()); assertThat(templateExistsResponse.getStatusCode(), is(200)); diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java index da8eca37d5476..902115e82925d 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java +++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; -import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.junit.After; import org.junit.Before; @@ -26,7 +26,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; @@ -83,7 +83,7 @@ public void startWatcher() throws Exception { }); assertBusy(() -> { - for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES) { + for (String template : XPackRestTestConstants.TEMPLATE_NAMES) { assertOK(adminClient().performRequest(new Request("HEAD", "_template/" + template))); } }); diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle index fb2e4c06ced88..9194c46daed01 100644 --- a/x-pack/qa/smoke-test-watcher/build.gradle +++ b/x-pack/qa/smoke-test-watcher/build.gradle @@ -2,10 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' dependencies { - testCompile "org.elasticsearch.plugin:x-pack-core:${version}" - testCompile project(path: xpackModule('watcher'), configuration: 'runtime') - testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') - testCompile project(path: ':modules:lang-painless', configuration: 'runtime') + testCompile project(':x-pack:qa') } integTestCluster { diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java index e08b2a20278f0..ec5015345fcb2 100644 --- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java +++ b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java @@ -15,7 +15,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ObjectPath; -import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.junit.After; import org.junit.Before; @@ -25,7 +25,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; @@ -64,7 +64,7 @@ public void startWatcher() throws Exception { }); assertBusy(() -> { - for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES_NO_ILM) { + for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) { Response templateExistsResponse = adminClient().performRequest(new Request("HEAD", "/_template/" + template)); assertThat(templateExistsResponse.getStatusLine().getStatusCode(), is(200)); } diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherRestIT.java b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherRestIT.java index 19c82c8cef799..2dd5cc86a89c6 100644 --- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherRestIT.java +++ b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherRestIT.java @@ -10,7 +10,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.junit.After; import org.junit.Before; @@ -58,7 +58,7 @@ public void startWatcher() throws Exception { }); assertBusy(() -> { - for (String template : WatcherIndexTemplateRegistryField.TEMPLATE_NAMES_NO_ILM) { + for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) { ClientYamlTestResponse templateExistsResponse = getAdminExecutionContext().callApi("indices.exists_template", singletonMap("name", template), emptyList(), emptyMap()); assertThat(templateExistsResponse.getStatusCode(), is(200)); diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/SecuritySettingsSourceField.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/SecuritySettingsSourceField.java new file mode 100644 index 0000000000000..8747d7809896b --- /dev/null +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/SecuritySettingsSourceField.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.test; + +import org.elasticsearch.common.CharArrays; +import org.elasticsearch.common.settings.SecureString; + +import java.nio.CharBuffer; +import java.util.Arrays; +import java.util.Base64; + +public final class SecuritySettingsSourceField { + public static final SecureString TEST_PASSWORD_SECURE_STRING = new SecureString("x-pack-test-password".toCharArray()); + public static final String TEST_PASSWORD = "x-pack-test-password"; + + private SecuritySettingsSourceField() {} + + public static String basicAuthHeaderValue(String username, String passwd) { + return basicAuthHeaderValue(username, new SecureString(passwd.toCharArray())); + } + + public static String basicAuthHeaderValue(String username, SecureString passwd) { + CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1); + byte[] charBytes = null; + try { + chars.put(username).put(':').put(passwd.getChars()); + charBytes = CharArrays.toUtf8Bytes(chars.array()); + + //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence? + String basicToken = Base64.getEncoder().encodeToString(charBytes); + return "Basic " + basicToken; + } finally { + Arrays.fill(chars.array(), (char) 0); + if (charBytes != null) { + Arrays.fill(charBytes, (byte) 0); + } + } + } +} diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java new file mode 100644 index 0000000000000..1a6a59fbc696b --- /dev/null +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.test.rest; + +import java.util.List; + +public final class XPackRestTestConstants { + + // Watcher constants: + public static final String INDEX_TEMPLATE_VERSION = "9"; + public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; + public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; + public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; + public static final String WATCHES_TEMPLATE_NAME = ".watches"; + public static final String[] TEMPLATE_NAMES = new String[] { + HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME + }; + public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] { + HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME + }; + + // ML constants: + public static final String ML_META_INDEX_NAME = ".ml-meta"; + public static final String AUDITOR_NOTIFICATIONS_INDEX = ".ml-notifications"; + public static final String CONFIG_INDEX = ".ml-config"; + public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; + public static final String STATE_INDEX_PREFIX = ".ml-state"; + public static final String RESULTS_INDEX_DEFAULT = "shared"; + + public static final List ML_POST_V660_TEMPLATES = + List.of(AUDITOR_NOTIFICATIONS_INDEX, ML_META_INDEX_NAME, STATE_INDEX_PREFIX, RESULTS_INDEX_PREFIX, CONFIG_INDEX); + + private XPackRestTestConstants() { + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java similarity index 81% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java rename to x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java index 24804129bde6c..6ad16d512ef0b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java +++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java @@ -14,14 +14,8 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.MlMetaIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; -import org.elasticsearch.xpack.core.ml.notifications.AuditorField; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -30,13 +24,6 @@ public final class XPackRestTestHelper { - public static final List ML_POST_V660_TEMPLATES = Collections.unmodifiableList( - Arrays.asList(AuditorField.NOTIFICATIONS_INDEX, - MlMetaIndex.INDEX_NAME, - AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, - AnomalyDetectorsIndex.jobResultsIndexPrefix(), - AnomalyDetectorsIndex.configIndexName())); - private XPackRestTestHelper() { } @@ -89,4 +76,10 @@ public static void waitForTemplates(RestClient client, List templateName }); } } + + public static String resultsWriteAlias(String jobId) { + // ".write" rather than simply "write" to avoid the danger of clashing + // with the read alias of a job whose name begins with "write-" + return XPackRestTestConstants.RESULTS_INDEX_PREFIX + ".write-" + jobId; + } } From cb3b21308636edccbcc529f603699ef2646bef48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 Apr 2019 09:41:34 +0200 Subject: [PATCH 06/15] Clarify some ToXContent implementations behaviour (#41000) This change adds either ToXContentObject or ToXContentFragment to classes directly implementing ToXContent currently. This helps in reasoning about whether those implementations output full xcontent object or just fragments. Relates to #16347 --- .../client/graph/GraphExploreResponse.java | 2 - .../indices/PutIndexTemplateRequest.java | 34 ++++++------- .../client/ml/FindFileStructureRequest.java | 4 +- .../client/RestHighLevelClientTests.java | 14 +++--- .../action/PainlessExecuteAction.java | 3 +- .../action/ShardOperationFailedException.java | 4 +- .../storedscripts/PutStoredScriptRequest.java | 4 +- .../shards/IndicesShardStoresResponse.java | 7 +-- .../template/put/PutIndexTemplateRequest.java | 49 ++++++++++--------- .../search/SearchPhaseExecutionException.java | 2 - .../action/search/SearchResponse.java | 4 +- .../action/search/ShardSearchFailure.java | 18 ++++--- .../DefaultShardOperationFailedException.java | 7 +++ .../routing/allocation/decider/Decision.java | 6 ++- .../index/query/IntervalsSourceProvider.java | 4 +- .../index/seqno/RetentionLease.java | 4 +- .../index/seqno/RetentionLeases.java | 3 +- .../rest/action/RestActions.java | 2 - .../elasticsearch/snapshots/SnapshotInfo.java | 4 -- .../snapshots/SnapshotShardFailure.java | 2 + .../common/xcontent/BaseXContentTestCase.java | 5 +- .../xpack/graph/GraphExploreResponse.java | 2 - .../rollup/action/GetRollupCapsAction.java | 5 +- .../action/GetRollupIndexCapsAction.java | 5 +- .../rollup/action/GetRollupJobsAction.java | 5 +- .../rollup/action/StartRollupJobAction.java | 5 +- .../rollup/action/StopRollupJobAction.java | 5 +- .../action/token/InvalidateTokenResponse.java | 3 +- .../support/mapper/TemplateRoleName.java | 4 +- .../actions/get/GetWatchResponse.java | 6 ++- .../rest/action/RestGetWatchAction.java | 2 - 31 files changed, 120 insertions(+), 104 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java index dddc4bedfe466..2171731290151 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java @@ -133,9 +133,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(FAILURES.getPreferredName()); if (shardFailures != null) { for (ShardOperationFailedException shardFailure : shardFailures) { - builder.startObject(); shardFailure.toXContent(builder, params); - builder.endObject(); } } builder.endArray(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java index 5f22691b046eb..7008a719b7b36 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -56,7 +56,7 @@ /** * A request to create an index template. */ -public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContent { +public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContentFragment { private String name; @@ -191,7 +191,7 @@ public PutIndexTemplateRequest settings(Map source) { public Settings settings() { return this.settings; } - + /** * Adds mapping that will be added when the index gets created. * @@ -201,7 +201,7 @@ public Settings settings() { public PutIndexTemplateRequest mapping(String source, XContentType xContentType) { internalMapping(XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2()); return this; - } + } /** * The cause for this index template creation. @@ -221,11 +221,11 @@ public String cause() { * @param source The mapping source */ public PutIndexTemplateRequest mapping(XContentBuilder source) { - internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), + internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), true, source.contentType()).v2()); - return this; - } - + return this; + } + /** * Adds mapping that will be added when the index gets created. * @@ -235,8 +235,8 @@ public PutIndexTemplateRequest mapping(XContentBuilder source) { public PutIndexTemplateRequest mapping(BytesReference source, XContentType xContentType) { internalMapping(XContentHelper.convertToMap(source, true, xContentType).v2()); return this; - } - + } + /** * Adds mapping that will be added when the index gets created. * @@ -244,7 +244,7 @@ public PutIndexTemplateRequest mapping(BytesReference source, XContentType xCont */ public PutIndexTemplateRequest mapping(Map source) { return internalMapping(source); - } + } private PutIndexTemplateRequest internalMapping(Map source) { try { @@ -257,12 +257,12 @@ private PutIndexTemplateRequest internalMapping(Map source) { return this; } catch (IOException e) { throw new UncheckedIOException("failed to convert source to json", e); - } + } } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); } - } - + } + public BytesReference mappings() { return this.mappings; } @@ -349,8 +349,8 @@ public PutIndexTemplateRequest source(byte[] source, int offset, int length, XCo */ public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { return source(XContentHelper.convertToMap(source, true, xContentType).v2()); - } - + } + public Set aliases() { return this.aliases; @@ -441,7 +441,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.copyCurrentStructure(parser); } } - + builder.startObject("aliases"); for (Alias alias : aliases) { alias.toXContent(builder, params); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java index 90e0c720e8811..adfee92bd6171 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.Optional; -public class FindFileStructureRequest implements Validatable, ToXContent { +public class FindFileStructureRequest implements Validatable, ToXContentFragment { public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample"); public static final ParseField TIMEOUT = new ParseField("timeout"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 90d440fe723bc..ed5d7b66d80c1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.client; import com.fasterxml.jackson.core.JsonParseException; + import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -61,6 +62,7 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.cbor.CborXContent; @@ -176,7 +178,7 @@ public void testInfo() throws IOException { MainResponse testInfo = new MainResponse("nodeName", new MainResponse.Version("number", "buildFlavor", "buildType", "buildHash", "buildDate", true, "luceneVersion", "minimumWireCompatibilityVersion", "minimumIndexCompatibilityVersion"), "clusterName", "clusterUuid", "You Know, for Search"); - mockResponse((builder, params) -> { + mockResponse((ToXContentFragment) (builder, params) -> { // taken from the server side MainResponse builder.field("name", testInfo.getNodeName()); builder.field("cluster_name", testInfo.getClusterName()); @@ -762,12 +764,12 @@ public void testApiNamingConventions() throws Exception { Collectors.mapping(Tuple::v2, Collectors.toSet()))); // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate - // impl will replace the existing getTemplate method. + // impl will replace the existing getTemplate method. // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` - // looking like it doesn't have a valid implementatation when it does. + // looking like it doesn't have a valid implementatation when it does. apiUnsupported.remove("indices.get_template"); - - + + for (Map.Entry> entry : methods.entrySet()) { String apiName = entry.getKey(); @@ -830,7 +832,7 @@ private static void assertSyncMethod(Method method, String apiName, List assertThat("the return type for method [" + method + "] is incorrect", method.getReturnType().getSimpleName(), equalTo("boolean")); } else { - // It's acceptable for 404s to be represented as empty Optionals + // It's acceptable for 404s to be represented as empty Optionals if (!method.getReturnType().isAssignableFrom(Optional.class)) { assertThat("the return type for method [" + method + "] is incorrect", method.getReturnType().getSimpleName(), endsWith("Response")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 81e3bdcd5c4c3..e8d93b8ef772e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -55,7 +55,6 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -107,7 +106,7 @@ public Response newResponse() { return new Response(); } - public static class Request extends SingleShardRequest implements ToXContent { + public static class Request extends SingleShardRequest implements ToXContentObject { private static final ParseField SCRIPT_FIELD = new ParseField("script"); private static final ParseField CONTEXT_FIELD = new ParseField("context"); diff --git a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java index 490a1760abeaa..34a8ccd7ad148 100644 --- a/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java +++ b/server/src/main/java/org/elasticsearch/action/ShardOperationFailedException.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.rest.RestStatus; import java.util.Objects; @@ -30,7 +30,7 @@ * An exception indicating that a failure occurred performing an operation on the shard. * */ -public abstract class ShardOperationFailedException implements Streamable, ToXContent { +public abstract class ShardOperationFailedException implements Streamable, ToXContentObject { protected String index; protected int shardId = -1; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 29171f12a1859..9904fc5dcde2b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; @@ -35,7 +35,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContent { +public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContentFragment { private String id; private String context; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java index a200bc82476c3..86198cb74391d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; @@ -258,8 +259,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field("node", nodeId()); - super.toXContent(builder, params); + super.innerToXContent(builder, params); + builder.endObject(); return builder; } } @@ -352,9 +355,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (failures.size() > 0) { builder.startArray(Fields.FAILURES); for (Failure failure : failures) { - builder.startObject(); failure.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index 7e3c549cfcf76..a23fac0ac12b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -38,7 +38,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -59,14 +59,14 @@ import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; /** * A request to create an index template. */ -public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContent { +public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContentObject { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(PutIndexTemplateRequest.class)); @@ -498,32 +498,35 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("index_patterns", indexPatterns); - builder.field("order", order); - if (version != null) { - builder.field("version", version); - } + builder.startObject(); + { + builder.field("index_patterns", indexPatterns); + builder.field("order", order); + if (version != null) { + builder.field("version", version); + } - builder.startObject("settings"); - settings.toXContent(builder, params); - builder.endObject(); + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); - builder.startObject("mappings"); - for (Map.Entry entry : mappings.entrySet()) { - builder.field(entry.getKey()); - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, entry.getValue())) { - builder.copyCurrentStructure(parser); + builder.startObject("mappings"); + for (Map.Entry entry : mappings.entrySet()) { + builder.field(entry.getKey()); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, entry.getValue())) { + builder.copyCurrentStructure(parser); + } } - } - builder.endObject(); + builder.endObject(); - builder.startObject("aliases"); - for (Alias alias : aliases) { - alias.toXContent(builder, params); + builder.startObject("aliases"); + for (Alias alias : aliases) { + alias.toXContent(builder, params); + } + builder.endObject(); } builder.endObject(); - return builder; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java index e3247c4f55199..8b4187c7cf0e9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java @@ -140,9 +140,7 @@ protected void metadataToXContent(XContentBuilder builder, Params params) throws builder.startArray(); ShardOperationFailedException[] failures = ExceptionsHelper.groupBy(shardFailures); for (ShardOperationFailedException failure : failures) { - builder.startObject(); failure.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 0e6773f5fbb43..b9b1887be2da8 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -29,7 +29,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.StatusToXContentObject; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -401,7 +401,7 @@ public String toString() { * Holds info about the clusters that the search was executed on: how many in total, how many of them were successful * and how many of them were skipped. */ - public static class Clusters implements ToXContent, Writeable { + public static class Clusters implements ToXContentFragment, Writeable { public static final Clusters EMPTY = new Clusters(0, 0, 0); diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 451ceda70fdc4..cfd23e3c77368 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -118,14 +118,18 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(SHARD_FIELD, shardId()); - builder.field(INDEX_FIELD, index()); - if (shardTarget != null) { - builder.field(NODE_FIELD, shardTarget.getNodeId()); - } - builder.field(REASON_FIELD); builder.startObject(); - ElasticsearchException.generateThrowableXContent(builder, params, cause); + { + builder.field(SHARD_FIELD, shardId()); + builder.field(INDEX_FIELD, index()); + if (shardTarget != null) { + builder.field(NODE_FIELD, shardTarget.getNodeId()); + } + builder.field(REASON_FIELD); + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, params, cause); + builder.endObject(); + } builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java index 85d8a2c1a38db..7aa7dfb62a63c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java +++ b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java @@ -91,6 +91,13 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + innerToXContent(builder, params); + builder.endObject(); + return builder; + } + + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { builder.field("shard", shardId()); builder.field("index", index()); builder.field("status", status.name()); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java index 725df82e51599..5311cd9c4a3fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; @@ -150,7 +152,7 @@ public boolean higherThan(Type other) { /** * Simple class representing a single decision */ - public static class Single extends Decision { + public static class Single extends Decision implements ToXContentObject { private Type type; private String label; private String explanation; @@ -269,7 +271,7 @@ public void writeTo(StreamOutput out) throws IOException { /** * Simple class representing a list of decisions */ - public static class Multi extends Decision { + public static class Multi extends Decision implements ToXContentFragment { private final List decisions = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java index 8aef53bc20e31..6aa8f2d700e7e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java @@ -31,8 +31,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -457,7 +457,7 @@ protected boolean accept(IntervalIterator it) { } } - public static class IntervalFilter implements ToXContent, Writeable { + public static class IntervalFilter implements ToXContentObject, Writeable { public static final String NAME = "filter"; diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java index e6d6ed3fe825f..9cfad7c36ea06 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -37,7 +37,7 @@ * otherwise merge away operations that have been soft deleted). Each retention lease contains a unique identifier, the retaining sequence * number, the timestamp of when the lease was created or renewed, and the source of the retention lease (e.g., "ccr"). */ -public final class RetentionLease implements ToXContent, Writeable { +public final class RetentionLease implements ToXContentObject, Writeable { private final String id; diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java index 3bad887282502..7c3b9e3c7b9c9 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeases.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.gateway.MetaDataStateFormat; @@ -42,7 +43,7 @@ * Represents a versioned collection of retention leases. We version the collection of retention leases to ensure that sync requests that * arrive out of order on the replica, using the version to ensure that older sync requests are rejected. */ -public class RetentionLeases implements ToXContent, Writeable { +public class RetentionLeases implements ToXContentFragment, Writeable { private final long primaryTerm; diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestActions.java b/server/src/main/java/org/elasticsearch/rest/action/RestActions.java index f25fd107e51db..f6ad30706f757 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestActions.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestActions.java @@ -91,9 +91,7 @@ public static void buildBroadcastShardsHeader(XContentBuilder builder, Params pa if (shardFailures != null && shardFailures.length > 0) { builder.startArray(FAILURES_FIELD.getPreferredName()); for (ShardOperationFailedException shardFailure : ExceptionsHelper.groupBy(shardFailures)) { - builder.startObject(); shardFailure.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index a8402dfa72afd..ca743f77aa3ba 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -510,9 +510,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (verbose || !shardFailures.isEmpty()) { builder.startArray(FAILURES); for (SnapshotShardFailure shardFailure : shardFailures) { - builder.startObject(); shardFailure.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } @@ -551,9 +549,7 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final builder.field(SUCCESSFUL_SHARDS, successfulShards); builder.startArray(FAILURES); for (SnapshotShardFailure shardFailure : shardFailures) { - builder.startObject(); shardFailure.toXContent(builder, params); - builder.endObject(); } builder.endArray(); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java index a497160967287..10e92b617d353 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java @@ -187,6 +187,7 @@ public static SnapshotShardFailure fromXContent(XContentParser parser) throws IO @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field("index", shardId.getIndexName()); builder.field("index_uuid", shardId.getIndexName()); builder.field("shard_id", shardId.id()); @@ -195,6 +196,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("node_id", nodeId); } builder.field("status", status.name()); + builder.endObject(); return builder; } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index bafe3b7403d6f..7c5cc2bc80264 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParseException; + import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -752,7 +753,7 @@ public void testToXContent() throws Exception { .field("xcontent", xcontent0) .endObject()); - ToXContent xcontent1 = (builder, params) -> { + ToXContentObject xcontent1 = (builder, params) -> { builder.startObject(); builder.field("field", "value"); builder.startObject("foo"); @@ -762,7 +763,7 @@ public void testToXContent() throws Exception { return builder; }; - ToXContent xcontent2 = (builder, params) -> { + ToXContentObject xcontent2 = (builder, params) -> { builder.startObject(); builder.field("root", xcontent0); builder.array("childs", xcontent0, xcontent1); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java index b44e192f407ac..5f6a4b35a9ec1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java @@ -168,9 +168,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(FAILURES.getPreferredName()); if (shardFailures != null) { for (ShardOperationFailedException shardFailure : shardFailures) { - builder.startObject(); shardFailure.toXContent(builder, params); - builder.endObject(); } } builder.endArray(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java index d28d14a0ac02d..f544c21a15c9a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java @@ -17,7 +17,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.rollup.RollupField; @@ -43,7 +44,7 @@ public Response newResponse() { return new Response(); } - public static class Request extends ActionRequest implements ToXContent { + public static class Request extends ActionRequest implements ToXContentFragment { private String indexPattern; public Request(String indexPattern) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java index 4f95919c4986b..9dcd673c39fb2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java @@ -19,7 +19,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.rollup.RollupField; @@ -47,7 +48,7 @@ public Response newResponse() { return new Response(); } - public static class Request extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { + public static class Request extends ActionRequest implements IndicesRequest.Replaceable, ToXContentFragment { private String[] indices; private IndicesOptions options; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java index 4b33e018826d7..913e544e74190 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.Task; @@ -58,7 +57,7 @@ public Writeable.Reader getResponseReader() { return Response::new; } - public static class Request extends BaseTasksRequest implements ToXContent { + public static class Request extends BaseTasksRequest implements ToXContentObject { private String id; public Request(String id) { @@ -107,7 +106,9 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field(RollupField.ID.getPreferredName(), id); + builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java index ff803b136281a..ca2a5cd8d7264 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -44,7 +43,7 @@ public Writeable.Reader getResponseReader() { return Response::new; } - public static class Request extends BaseTasksRequest implements ToXContent { + public static class Request extends BaseTasksRequest implements ToXContentObject { private String id; public Request(String id) { @@ -75,7 +74,9 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field(RollupField.ID.getPreferredName(), id); + builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java index dadc54726b522..6fc079e0328c9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -51,7 +50,7 @@ public Writeable.Reader getResponseReader() { return Response::new; } - public static class Request extends BaseTasksRequest implements ToXContent { + public static class Request extends BaseTasksRequest implements ToXContentObject { private String id; private boolean waitForCompletion = false; private TimeValue timeout = null; @@ -106,11 +105,13 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field(RollupField.ID.getPreferredName(), id); builder.field(WAIT_FOR_COMPLETION.getPreferredName(), waitForCompletion); if (timeout != null) { builder.field(TIMEOUT.getPreferredName(), timeout); } + builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java index 9f11c48c96aba..7e25683a2a933 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponse.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.security.authc.support.TokensInvalidationResult; @@ -18,7 +19,7 @@ /** * Response for a invalidation of one or multiple tokens. */ -public final class InvalidateTokenResponse extends ActionResponse implements ToXContent { +public final class InvalidateTokenResponse extends ActionResponse implements ToXContentObject { private TokensInvalidationResult result; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java index d77882d6454d7..59f9eafec1c00 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleName.java @@ -16,7 +16,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -44,7 +44,7 @@ /** * Representation of a Mustache template for expressing one or more roles names in a {@link ExpressionRoleMapping}. */ -public class TemplateRoleName implements ToXContent, Writeable { +public class TemplateRoleName implements ToXContentObject, Writeable { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "role-mapping-template", false, arr -> new TemplateRoleName((BytesReference) arr[0], (Format) arr[1])); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java index 18ec33f5dfb0d..fe79fdc33f938 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; @@ -19,7 +19,7 @@ import java.io.IOException; import java.util.Objects; -public class GetWatchResponse extends ActionResponse implements ToXContent { +public class GetWatchResponse extends ActionResponse implements ToXContentObject { private String id; private WatchStatus status; @@ -122,6 +122,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.field("found", found); builder.field("_id", id); if (found) { @@ -131,6 +132,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("status", status, params); builder.field("watch", source, params); } + builder.endObject(); return builder; } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java index 15cb9612445e1..0d9b1ee6a1991 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java @@ -48,9 +48,7 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest request, Watche return channel -> client.getWatch(getWatchRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetWatchResponse response, XContentBuilder builder) throws Exception { - builder.startObject(); response.toXContent(builder, request); - builder.endObject(); RestStatus status = response.isFound() ? OK : NOT_FOUND; return new BytesRestResponse(status, builder); } From f70d8cb9a56148047e2e90bfeefb2d9af29549d0 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 15 Apr 2019 09:46:50 +0200 Subject: [PATCH 07/15] Full text queries should not always ignore unmapped fields (#41062) Full text queries ignore unmapped fields since https://github.com/elastic/elasticsearch/issues/41022 even if all fields in the query are unmapped. This change makes sure that we ignore unmapped fields only if they are mixed with mapped fields and returns a MatchNoDocsQuery otherwise. Closes #41022 --- .../common/lucene/search/Queries.java | 8 ++++- .../index/search/MultiMatchQuery.java | 8 ++++- .../query/QueryStringQueryBuilderTests.java | 31 +++++++++++++++++-- .../query/SimpleQueryStringBuilderTests.java | 11 +++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index dd4389c2d6b0f..56d1b5cedc33c 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -36,6 +36,8 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.regex.Pattern; @@ -52,7 +54,11 @@ public static Query newMatchNoDocsQuery(String reason) { public static Query newUnmappedFieldQuery(String field) { - return Queries.newMatchNoDocsQuery("unmapped field [" + (field != null ? field : "null") + "]"); + return newUnmappedFieldsQuery(Collections.singletonList(field)); + } + + public static Query newUnmappedFieldsQuery(Collection fields) { + return Queries.newMatchNoDocsQuery("unmapped fields " + fields); } public static Query newLenientFieldQuery(String field, RuntimeException e) { diff --git a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java index 667d3a3823db8..b9943870df727 100644 --- a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java +++ b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java @@ -59,6 +59,12 @@ public void setTieBreaker(float tieBreaker) { public Query parse(MultiMatchQueryBuilder.Type type, Map fieldNames, Object value, String minimumShouldMatch) throws IOException { + boolean hasMappedField = fieldNames.keySet().stream() + .anyMatch(k -> context.fieldMapper(k) != null); + if (hasMappedField == false) { + // all query fields are unmapped + return Queries.newUnmappedFieldsQuery(fieldNames.keySet()); + } final float tieBreaker = groupTieBreaker == null ? type.tieBreaker() : groupTieBreaker; final List queries; switch (type) { @@ -91,7 +97,7 @@ private Query combineGrouped(List groupQuery, float tieBreaker) { } private List buildFieldQueries(MultiMatchQueryBuilder.Type type, Map fieldNames, - Object value, String minimumShouldMatch) throws IOException{ + Object value, String minimumShouldMatch) throws IOException { List queries = new ArrayList<>(); for (String fieldName : fieldNames.keySet()) { if (context.fieldMapper(fieldName) == null) { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index c85ef842d6870..608eb5d9d6cd2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1210,13 +1210,13 @@ public void testUnmappedFieldRewriteToMatchNoDocs() throws IOException { .field("unmapped_field") .lenient(true) .toQuery(createShardContext()); - assertEquals(new BooleanQuery.Builder().build(), query); + assertEquals(new MatchNoDocsQuery(), query); // Unmapped prefix field query = new QueryStringQueryBuilder("unmapped_field:hello") .lenient(true) .toQuery(createShardContext()); - assertEquals(new BooleanQuery.Builder().build(), query); + assertEquals(new MatchNoDocsQuery(), query); // Unmapped fields query = new QueryStringQueryBuilder("hello") @@ -1224,7 +1224,32 @@ public void testUnmappedFieldRewriteToMatchNoDocs() throws IOException { .field("unmapped_field") .field("another_field") .toQuery(createShardContext()); - assertEquals(new BooleanQuery.Builder().build(), query); + assertEquals(new MatchNoDocsQuery(), query); + + // Multi block + query = new QueryStringQueryBuilder("first unmapped:second") + .field(STRING_FIELD_NAME) + .field("unmapped") + .field("another_unmapped") + .defaultOperator(Operator.AND) + .toQuery(createShardContext()); + BooleanQuery expected = new BooleanQuery.Builder() + .add(new TermQuery(new Term(STRING_FIELD_NAME, "first")), BooleanClause.Occur.MUST) + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .build(); + assertEquals(expected, query); + + query = new SimpleQueryStringBuilder("first unknown:second") + .field("unmapped") + .field("another_unmapped") + .defaultOperator(Operator.AND) + .toQuery(createShardContext()); + expected = new BooleanQuery.Builder() + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .build(); + assertEquals(expected, query); + } public void testDefaultField() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 3242f343379aa..2bb289ddc11fa 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -717,6 +717,17 @@ public void testUnmappedFieldNoTokenWithAndOperator() throws IOException { .add(new TermQuery(new Term(STRING_FIELD_NAME, "second")), BooleanClause.Occur.MUST) .build(); assertEquals(expected, query); + query = new SimpleQueryStringBuilder("first & second") + .field("unmapped") + .field("another_unmapped") + .defaultOperator(Operator.AND) + .toQuery(createShardContext()); + expected = new BooleanQuery.Builder() + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .add(new MatchNoDocsQuery(), BooleanClause.Occur.MUST) + .build(); + assertEquals(expected, query); } public void testNegativeFieldBoost() { From 9f74a932eb558897009754cccda41621d423128e Mon Sep 17 00:00:00 2001 From: Guilherme Ferreira Date: Mon, 15 Apr 2019 14:09:46 +0200 Subject: [PATCH 08/15] [Docs] Correct spelling the "_none_" stopwords element (#41191) --- docs/reference/analysis/analyzers/standard-analyzer.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc index 3097ece21db23..5117763b66819 100644 --- a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc @@ -132,7 +132,7 @@ The `standard` analyzer accepts the following parameters: `stopwords`:: A pre-defined stop words list like `_english_` or an array containing a - list of stop words. Defaults to `\_none_`. + list of stop words. Defaults to `_none_`. `stopwords_path`:: From 923e2c738daa640621b70d10c5f23fef8fd3eb2f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 15 Apr 2019 15:24:01 +0300 Subject: [PATCH 09/15] Mute failing test Tracked in #41172 --- .../rest-api-spec/test/mustache/50_webhook_url_escaping.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml b/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml index 01326f9764fa1..e11809a79baa5 100644 --- a/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml +++ b/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml @@ -1,5 +1,8 @@ --- "Test url escaping with url mustache function": + - skip: + version: "all" + reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/41172" - do: cluster.health: wait_for_status: yellow From 17463d2be480a92a3fe8d9b8e6563548c0b0b704 Mon Sep 17 00:00:00 2001 From: Guilherme Ferreira Date: Mon, 15 Apr 2019 15:10:57 +0200 Subject: [PATCH 10/15] [Docs] Correct spelling of "_none_" (#41192) --- docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc index 3167a4342ac2d..259bf785b5b30 100644 --- a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc @@ -78,4 +78,4 @@ Elasticsearch provides the following predefined list of languages: `_portuguese_`, `_romanian_`, `_russian_`, `_sorani_`, `_spanish_`, `_swedish_`, `_thai_`, `_turkish_`. -For the empty stopwords list (to disable stopwords) use: `\_none_`. +For the empty stopwords list (to disable stopwords) use: `_none_`. From 3d5a407e6ab1e0cdc3f83f59e18161f5ce95e672 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 15 Apr 2019 14:30:30 +0100 Subject: [PATCH 11/15] [ML] Allow xpack.ml.max_machine_memory_percent higher than 100% (#41193) Values higher than 100% are now allowed to accommodate use cases where swapping has been determined to be acceptable. Anomaly detector jobs only use their full model memory during background persistence, and this is deliberately staggered, so with large numbers of jobs few will generally be persisting state at the same time. Settings higher than available memory are only recommended for OEM type situations where a wrapper tightly controls the types of jobs that can be created, and each job alone is considerably smaller than what each node can handle. --- .../xpack/ml/MachineLearning.java | 8 ++++++- .../xpack/ml/MachineLearningTests.java | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index b69e7b786a77e..281a374b97b2f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -271,8 +271,14 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu public static final String MACHINE_MEMORY_NODE_ATTR = "ml.machine_memory"; public static final Setting CONCURRENT_JOB_ALLOCATIONS = Setting.intSetting("xpack.ml.node_concurrent_job_allocations", 2, 0, Property.Dynamic, Property.NodeScope); + // Values higher than 100% are allowed to accommodate use cases where swapping has been determined to be acceptable. + // Anomaly detector jobs only use their full model memory during background persistence, and this is deliberately + // staggered, so with large numbers of jobs few will generally be persisting state at the same time. + // Settings higher than available memory are only recommended for OEM type situations where a wrapper tightly + // controls the types of jobs that can be created, and each job alone is considerably smaller than what each node + // can handle. public static final Setting MAX_MACHINE_MEMORY_PERCENT = - Setting.intSetting("xpack.ml.max_machine_memory_percent", 30, 5, 90, Property.Dynamic, Property.NodeScope); + Setting.intSetting("xpack.ml.max_machine_memory_percent", 30, 5, 200, Property.Dynamic, Property.NodeScope); public static final Setting MAX_LAZY_ML_NODES = Setting.intSetting("xpack.ml.max_lazy_ml_nodes", 0, 0, 3, Property.Dynamic, Property.NodeScope); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java index 2c296691c249d..9504cbe7a7011 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java @@ -31,6 +31,29 @@ public void testMaxOpenWorkersSetting_givenSetting() { assertEquals(7, maxOpenWorkers); } + public void testMaxMachineMemoryPercent_givenDefault() { + int maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(Settings.EMPTY); + assertEquals(30, maxMachineMemoryPercent); + } + + public void testMaxMachineMemoryPercent_givenValidSetting() { + Settings.Builder settings = Settings.builder(); + int expectedMaxMachineMemoryPercent = randomIntBetween(5, 200); + settings.put(MachineLearning.MAX_MACHINE_MEMORY_PERCENT.getKey(), expectedMaxMachineMemoryPercent); + int maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings.build()); + assertEquals(expectedMaxMachineMemoryPercent, maxMachineMemoryPercent); + } + + public void testMaxMachineMemoryPercent_givenInvalidSetting() { + Settings.Builder settings = Settings.builder(); + int invalidMaxMachineMemoryPercent = randomFrom(4, 201); + settings.put(MachineLearning.MAX_MACHINE_MEMORY_PERCENT.getKey(), invalidMaxMachineMemoryPercent); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings.build())); + assertThat(e.getMessage(), startsWith("Failed to parse value [" + invalidMaxMachineMemoryPercent + + "] for setting [xpack.ml.max_machine_memory_percent] must be")); + } + public void testNoAttributes_givenNoClash() { Settings.Builder builder = Settings.builder(); if (randomBoolean()) { From 3ad2b295466979991f16e9542ba4ac557298b208 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 15 Apr 2019 14:33:16 +0100 Subject: [PATCH 12/15] [ML DataFrame] Data Frame stop all (#41156) Wild card support for the data frame stop API --- .../dataframe/stop_data_frame.asciidoc | 4 +- .../data-frames/apis/stop-transform.asciidoc | 12 +++- .../action/StopDataFrameTransformAction.java | 36 ++++++++++-- ...pDataFrameTransformActionRequestTests.java | 35 +++++++++++- ...TransportStopDataFrameTransformAction.java | 51 ++++++++++++++--- ...portStopDataFrameTransformActionTests.java | 55 +++++++++++++++++++ .../test/data_frame/transforms_start_stop.yml | 42 ++++++++++++++ 7 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java diff --git a/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc b/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc index c91c228e0ea90..9b05687c00875 100644 --- a/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc +++ b/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc @@ -12,7 +12,9 @@ It accepts a +{request}+ object and responds with a +{response}+ object. [id="{upid}-{api}-request"] ==== Stop Data Frame Request -A +{request}+ object requires a non-null `id`. +A +{request}+ object requires a non-null `id`. `id` can be a comma separated list of Ids +or a single Id. Wildcards, `*` and `_all` are also accepted. + ["source","java",subs="attributes,callouts,macros"] --------------------------------------------------- diff --git a/docs/reference/data-frames/apis/stop-transform.asciidoc b/docs/reference/data-frames/apis/stop-transform.asciidoc index 179646ddb47ba..2eb5a2d641bd4 100644 --- a/docs/reference/data-frames/apis/stop-transform.asciidoc +++ b/docs/reference/data-frames/apis/stop-transform.asciidoc @@ -10,9 +10,17 @@ Stops one or more {dataframe-transforms}. ==== Request -`POST _data_frame/transforms//_stop` +`POST _data_frame/transforms//_stop` + -//==== Description +`POST _data_frame/transforms/,/_stop` + + +`POST _data_frame/transforms/_all/_stop` + + +==== Description +You can stop multiple {dataframe-transforms} in a single API request by using a +comma-separated list of {dataframe-transforms} or a wildcard expression. +All {dataframe-transforms} can be stopped by using `_all` or `*` as the ``. ==== Path Parameters diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java index 7fa437bd15606..54153aab91ced 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java @@ -21,8 +21,11 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; public class StopDataFrameTransformAction extends Action { @@ -45,6 +48,7 @@ public static class Request extends BaseTasksRequest { private String id; private final boolean waitForCompletion; private final boolean force; + private Set expandedIds; public Request(String id, boolean waitForCompletion, boolean force, @Nullable TimeValue timeout) { this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName()); @@ -64,6 +68,9 @@ public Request(StreamInput in) throws IOException { id = in.readString(); waitForCompletion = in.readBoolean(); force = in.readBoolean(); + if (in.readBoolean()) { + expandedIds = new HashSet<>(Arrays.asList(in.readStringArray())); + } } public String getId() { @@ -82,12 +89,25 @@ public boolean isForce() { return force; } + public Set getExpandedIds() { + return expandedIds; + } + + public void setExpandedIds(Set expandedIds ) { + this.expandedIds = expandedIds; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBoolean(waitForCompletion); out.writeBoolean(force); + boolean hasExpandedIds = expandedIds != null; + out.writeBoolean(hasExpandedIds); + if (hasExpandedIds) { + out.writeStringArray(expandedIds.toArray(new String[0])); + } } @Override @@ -98,7 +118,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { // the base class does not implement hashCode, therefore we need to hash timeout ourselves - return Objects.hash(id, waitForCompletion, force, this.getTimeout()); + return Objects.hash(id, waitForCompletion, force, expandedIds, this.getTimeout()); } @Override @@ -118,15 +138,21 @@ public boolean equals(Object obj) { } return Objects.equals(id, other.id) && - Objects.equals(waitForCompletion, other.waitForCompletion) && - Objects.equals(force, other.force); + Objects.equals(waitForCompletion, other.waitForCompletion) && + Objects.equals(force, other.force) && + Objects.equals(expandedIds, other.expandedIds); } @Override public boolean match(Task task) { - String expectedDescription = DataFrameField.PERSISTENT_TASK_DESCRIPTION_PREFIX + id; + if (task.getDescription().startsWith(DataFrameField.PERSISTENT_TASK_DESCRIPTION_PREFIX)) { + String id = task.getDescription().substring(DataFrameField.PERSISTENT_TASK_DESCRIPTION_PREFIX.length()); + if (expandedIds != null) { + return expandedIds.contains(id); + } + } - return task.getDescription().equals(expectedDescription); + return false; } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java index c117e249aef91..81f03902980ac 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java @@ -8,15 +8,27 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction.Request; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + public class StopDataFrameTransformActionRequestTests extends AbstractWireSerializingTestCase { @Override protected Request createTestInstance() { TimeValue timeout = randomBoolean() ? TimeValue.timeValueMinutes(randomIntBetween(1, 10)) : null; - return new Request(randomAlphaOfLengthBetween(1, 10), randomBoolean(), randomBoolean(), timeout); + Request request = new Request(randomAlphaOfLengthBetween(1, 10), randomBoolean(), randomBoolean(), timeout); + if (randomBoolean()) { + request.setExpandedIds(new HashSet<>(Arrays.asList(generateRandomStringArray(5, 6, false)))); + } + return request; } @Override @@ -35,4 +47,25 @@ public void testSameButDifferentTimeout() { assertNotEquals(r1,r2); assertNotEquals(r1.hashCode(),r2.hashCode()); } + + public void testMatch() { + String dataFrameId = "dataframe-id"; + + Task dataFrameTask = new Task(1L, "persistent", "action", + DataFrameField.PERSISTENT_TASK_DESCRIPTION_PREFIX + dataFrameId, + TaskId.EMPTY_TASK_ID, Collections.emptyMap()); + + Request request = new Request("unrelated", false, false, null); + request.setExpandedIds(Set.of("foo", "bar")); + assertFalse(request.match(dataFrameTask)); + + Request matchingRequest = new Request(dataFrameId, false, false, null); + matchingRequest.setExpandedIds(Set.of(dataFrameId)); + assertTrue(matchingRequest.match(dataFrameTask)); + + Task notADataFrameTask = new Task(1L, "persistent", "action", + "some other task, say monitoring", + TaskId.EMPTY_TASK_ID, Collections.emptyMap()); + assertFalse(matchingRequest.match(notADataFrameTask)); + } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java index 2234226a50134..2092493caaf4c 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java @@ -13,20 +13,27 @@ import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.tasks.TransportTasksAction; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.util.PageParams; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager; import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformTask; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.elasticsearch.ExceptionsHelper.convertToElastic; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; @@ -52,17 +59,28 @@ public TransportStopDataFrameTransformAction(TransportService transportService, @Override protected void doExecute(Task task, StopDataFrameTransformAction.Request request, ActionListener listener) { - // Need to verify that the config actually exists - dataFrameTransformsConfigManager.getTransformConfiguration(request.getId(), ActionListener.wrap( - config -> super.doExecute(task, request, listener), - listener::onFailure + + dataFrameTransformsConfigManager.expandTransformIds(request.getId(), new PageParams(0, 10_000), ActionListener.wrap( + expandedIds -> { + request.setExpandedIds(new HashSet<>(expandedIds)); + request.setNodes(dataframeNodes(expandedIds, clusterService.state())); + super.doExecute(task, request, listener); + }, + listener::onFailure )); } @Override protected void taskOperation(StopDataFrameTransformAction.Request request, DataFrameTransformTask transformTask, ActionListener listener) { - if (transformTask.getTransformId().equals(request.getId())) { + + Set ids = request.getExpandedIds(); + if (ids == null) { + listener.onFailure(new IllegalStateException("Request does not have expandedIds set")); + return; + } + + if (ids.contains(transformTask.getTransformId())) { if (transformTask.getState().getTaskState() == DataFrameTransformTaskState.FAILED && request.isForce() == false) { listener.onFailure( new ElasticsearchStatusException("Unable to stop data frame transform [" + request.getId() @@ -138,9 +156,28 @@ protected StopDataFrameTransformAction.Response newResponse(StopDataFrameTransfo } } - assert tasks.size() == 1; - boolean allStopped = tasks.stream().allMatch(StopDataFrameTransformAction.Response::isStopped); return new StopDataFrameTransformAction.Response(allStopped); } + + static String[] dataframeNodes(List dataFrameIds, ClusterState clusterState) { + + Set executorNodes = new HashSet<>(); + + PersistentTasksCustomMetaData tasksMetaData = + PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(clusterState); + + if (tasksMetaData != null) { + Set dataFrameIdsSet = new HashSet<>(dataFrameIds); + + Collection> tasks = + tasksMetaData.findTasks(DataFrameField.TASK_NAME, t -> dataFrameIdsSet.contains(t.getId())); + + for (PersistentTasksCustomMetaData.PersistentTask task : tasks) { + executorNodes.add(task.getExecutorNode()); + } + } + + return executorNodes.toArray(new String[0]); + } } diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java new file mode 100644 index 0000000000000..4be532de2a2be --- /dev/null +++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.action; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.OpenJobAction; + +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.hasItemInArray; + +public class TransportStopDataFrameTransformActionTests extends ESTestCase { + + public void testDataframeNodes() { + String dataFrameIdFoo = "df-id-foo"; + String dataFrameIdBar = "df-id-bar"; + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + tasksBuilder.addTask(dataFrameIdFoo, + DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdFoo), + new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment")); + tasksBuilder.addTask(dataFrameIdBar, + DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdBar), + new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment")); + tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"), + new PersistentTasksCustomMetaData.Assignment("node-3", "test assignment")); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build())) + .build(); + + String[] nodes = TransportStopDataFrameTransformAction.dataframeNodes(Arrays.asList(dataFrameIdFoo, dataFrameIdBar), cs); + assertEquals(2, nodes.length); + assertThat(nodes, hasItemInArray("node-1")); + assertThat(nodes, hasItemInArray("node-2")); + } + + public void testDataframeNodes_NoTasks() { + ClusterState emptyState = ClusterState.builder(new ClusterName("_name")).build(); + String[] nodes = TransportStopDataFrameTransformAction.dataframeNodes(Collections.singletonList("df-id"), emptyState); + assertEquals(0, nodes.length); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml index 23a28e14a86c4..96f6b6d0a4150 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml @@ -179,3 +179,45 @@ teardown: - do: data_frame.delete_data_frame_transform: transform_id: "airline-transform-start-later" + +--- +"Test stop all": + - do: + data_frame.put_data_frame_transform: + transform_id: "airline-transform-stop-all" + body: > + { + "source": { "index": "airline-data" }, + "dest": { "index": "airline-data-start-later" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - do: + data_frame.start_data_frame_transform: + transform_id: "airline-transform-stop-all" + - match: { started: true } + + - do: + data_frame.start_data_frame_transform: + transform_id: "airline-transform-start-stop" + - match: { started: true } + + - do: + data_frame.stop_data_frame_transform: + transform_id: "_all" + - match: { stopped: true } + + - do: + data_frame.get_data_frame_transform_stats: + transform_id: "*" + - match: { count: 2 } + - match: { transforms.0.state.indexer_state: "stopped" } + - match: { transforms.0.state.task_state: "stopped" } + - match: { transforms.1.state.indexer_state: "stopped" } + - match: { transforms.1.state.task_state: "stopped" } + + - do: + data_frame.delete_data_frame_transform: + transform_id: "airline-transform-stop-all" From 2619d8f5b4e500b4ec29b33339a09d7c94cf4800 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 15 Apr 2019 10:35:20 -0400 Subject: [PATCH 13/15] Better error messages when pipelines reference incompatible aggs (#40068) Pipelines require single-valued agg or a numeric to be returned. If they don't get that, they throw an exception. Unfortunately, this exception text is very confusing to users because it usually arises from pathing "through" multiple terms aggs. The final target is a numeric, but it's the intermediary aggs that cause the problem. This commit adds the current agg name to the exception message so the user knows which "level" is the issue. --- .../test/search.aggregation/300_pipeline.yml | 100 ++++++++++ .../aggregations/pipeline/BucketHelpers.java | 30 ++- .../pipeline/BucketHelpersTests.java | 185 ++++++++++++++++++ 3 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/300_pipeline.yml create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpersTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/300_pipeline.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/300_pipeline.yml new file mode 100644 index 0000000000000..0016c9f989486 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/300_pipeline.yml @@ -0,0 +1,100 @@ +setup: + - skip: + version: " - 7.99.99" #TODO change this after backport + reason: These new error messages were added in 7.1 + + - do: + indices.create: + index: test_1 + body: + settings: + number_of_replicas: 0 + mappings: + properties: + int_field: + type : integer + + - do: + bulk: + refresh: true + body: + - index: + _index: test_1 + _id: 1 + - int_field: 1 + - index: + _index: test_1 + _id: 2 + - int_field: 2 + - index: + _index: test_1 + _id: 3 + - int_field: 3 + - index: + _index: test_1 + _id: 4 + - int_field: 4 + +--- +"Max pipeline through terms agg": + + - do: + catch: /\[Object\[\]\] at aggregation \[the_terms_2\]/ + search: + rest_total_hits_as_int: true + body: + aggs: + the_terms: + terms: + field: "int_field" + aggs: + the_terms_2: + terms: + field: "int_field" + aggs: + the_max: + max: + field: "int_field" + the_bad_max: + max_bucket: + buckets_path: "the_terms>the_terms_2>the_max" + +--- +"Max pipeline on terms agg": + + - do: + catch: /\[LongTerms\] at aggregation \[the_terms_2\]/ + search: + rest_total_hits_as_int: true + body: + aggs: + the_terms: + terms: + field: "int_field" + aggs: + the_terms_2: + terms: + field: "int_field" + the_bad_max: + max_bucket: + buckets_path: "the_terms>the_terms_2" + +--- +"Max pipeline on percentiles agg without specifying percent": + + - do: + catch: /buckets_path must reference either a number value or a single value numeric metric aggregation, but \[the_percentiles\] contains multiple values. Please specify which to use\./ + search: + rest_total_hits_as_int: true + body: + aggs: + the_terms: + terms: + field: "int_field" + aggs: + the_percentiles: + percentiles: + field: "int_field" + the_bad_max: + max_bucket: + buckets_path: "the_terms>the_percentiles" diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java index d2c973ebec26c..1a863a209828b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java @@ -156,6 +156,7 @@ public static Double resolveBucketValue(MultiBucketsAggregation agg, InternalMultiBucketAggregation.InternalBucket bucket, List aggPathAsList, GapPolicy gapPolicy) { try { Object propertyValue = bucket.getProperty(agg.getName(), aggPathAsList); + if (propertyValue == null) { throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() + " must reference either a number value or a single value numeric metric aggregation"); @@ -166,9 +167,7 @@ public static Double resolveBucketValue(MultiBucketsAggregation agg, } else if (propertyValue instanceof InternalNumericMetricsAggregation.SingleValue) { value = ((InternalNumericMetricsAggregation.SingleValue) propertyValue).value(); } else { - throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() - + " must reference either a number value or a single value numeric metric aggregation, got: " - + propertyValue.getClass().getCanonicalName()); + throw formatResolutionError(agg, aggPathAsList, propertyValue); } // doc count never has missing values so gap policy doesn't apply here boolean isDocCountProperty = aggPathAsList.size() == 1 && "_count".equals(aggPathAsList.get(0)); @@ -188,4 +187,29 @@ public static Double resolveBucketValue(MultiBucketsAggregation agg, return null; } } + + /** + * Inspects where we are in the agg tree and tries to format a helpful error + */ + private static AggregationExecutionException formatResolutionError(MultiBucketsAggregation agg, + List aggPathAsList, Object propertyValue) { + String currentAggName; + Object currentAgg; + if (aggPathAsList.isEmpty()) { + currentAggName = agg.getName(); + currentAgg = agg; + } else { + currentAggName = aggPathAsList.get(0); + currentAgg = propertyValue; + } + if (currentAgg instanceof InternalNumericMetricsAggregation.MultiValue) { + return new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() + + " must reference either a number value or a single value numeric metric aggregation, but [" + currentAggName + + "] contains multiple values. Please specify which to use."); + } else { + return new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName() + + " must reference either a number value or a single value numeric metric aggregation, got: [" + + propertyValue.getClass().getSimpleName() + "] at aggregation [" + currentAggName + "]"); + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpersTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpersTests.java new file mode 100644 index 0000000000000..fbf8ad9d65a21 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpersTests.java @@ -0,0 +1,185 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.pipeline; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.aggregations.AggregationExecutionException; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class BucketHelpersTests extends ESTestCase { + + public void testReturnsObjectArray() { + + MultiBucketsAggregation agg = new MultiBucketsAggregation() { + @Override + public List getBuckets() { + return null; + } + + @Override + public String getName() { + return "foo"; + } + + @Override + public String getType() { + return null; + } + + @Override + public Map getMetaData() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return null; + } + }; + + InternalMultiBucketAggregation.InternalBucket bucket = new InternalMultiBucketAggregation.InternalBucket() { + @Override + public void writeTo(StreamOutput out) throws IOException { + + } + + @Override + public Object getKey() { + return null; + } + + @Override + public String getKeyAsString() { + return null; + } + + @Override + public long getDocCount() { + return 0; + } + + @Override + public Aggregations getAggregations() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return null; + } + + @Override + public Object getProperty(String containingAggName, List path) { + return new Object[0]; + } + }; + + AggregationExecutionException e = expectThrows(AggregationExecutionException.class, + () -> BucketHelpers.resolveBucketValue(agg, bucket, "foo>bar", BucketHelpers.GapPolicy.SKIP)); + + assertThat(e.getMessage(), equalTo("buckets_path must reference either a number value or a single value numeric " + + "metric aggregation, got: [Object[]] at aggregation [foo]")); + } + + public void testReturnMultiValueObject() { + + MultiBucketsAggregation agg = new MultiBucketsAggregation() { + @Override + public List getBuckets() { + return null; + } + + @Override + public String getName() { + return "foo"; + } + + @Override + public String getType() { + return null; + } + + @Override + public Map getMetaData() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return null; + } + }; + + InternalMultiBucketAggregation.InternalBucket bucket = new InternalMultiBucketAggregation.InternalBucket() { + @Override + public void writeTo(StreamOutput out) throws IOException { + + } + + @Override + public Object getKey() { + return null; + } + + @Override + public String getKeyAsString() { + return null; + } + + @Override + public long getDocCount() { + return 0; + } + + @Override + public Aggregations getAggregations() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return null; + } + + @Override + public Object getProperty(String containingAggName, List path) { + return mock(InternalTDigestPercentiles.class); + } + }; + + AggregationExecutionException e = expectThrows(AggregationExecutionException.class, + () -> BucketHelpers.resolveBucketValue(agg, bucket, "foo>bar", BucketHelpers.GapPolicy.SKIP)); + + assertThat(e.getMessage(), equalTo("buckets_path must reference either a number value or a single value numeric " + + "metric aggregation, but [foo] contains multiple values. Please specify which to use.")); + } +} From 656cc709b2e51fd8a0bddf1591becf60cc388f39 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 15 Apr 2019 11:27:38 -0400 Subject: [PATCH 14/15] Fix intervals section of auto date-histogram docs (#41203) This section should be at the same sub-level as other sections in the auto date-histogram docs, otherwise it is rendered on to another page and is confusing for users to understand what it's in reference to. --- .../aggregations/bucket/autodatehistogram-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc index e371674228bb4..dfc4f62a91b09 100644 --- a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc @@ -89,7 +89,7 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -=== Intervals +==== Intervals The interval of the returned buckets is selected based on the data collected by the aggregation so that the number of buckets returned is less than or equal to the number From 6aa52b1ac5f35c23fb4c39f4f7020966dcbf7c9a Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Mon, 15 Apr 2019 18:48:05 +0300 Subject: [PATCH 15/15] Fix the rpm and deb names for version starting with 7.0.0 (#41198) With the 7.0.0 release, we switched to download the packages instead of using locally built ones. This PR fixes the artifact names to include the architecture as introduced in the 7.0.0 release. --- .../gradle/vagrant/VagrantTestPlugin.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 9e6f7ef250719..121b40821bbc3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -214,9 +214,15 @@ class VagrantTestPlugin implements Plugin { } else { UPGRADE_FROM_ARCHIVES.each { // The version of elasticsearch that we upgrade *from* - dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") - if (upgradeFromVersion.onOrAfter('6.3.0')) { - dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + if (upgradeFromVersion.onOrAfter('7.0.0')) { + String arch = it == "rpm" ? "x86_64" : "amd64" + dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}-${arch}@${it}") + dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}-${arch}@${it}") + } else { + dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}") + if (upgradeFromVersion.onOrAfter('6.3.0')) { + dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}") + } } } }