diff --git a/docs/changelog/100232.yaml b/docs/changelog/100232.yaml new file mode 100644 index 0000000000000..3f8336b6c241c --- /dev/null +++ b/docs/changelog/100232.yaml @@ -0,0 +1,5 @@ +pr: 100232 +summary: "Tracing: Use `doPriv` when working with spans, use `SpanId`" +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/100238.yaml b/docs/changelog/100238.yaml new file mode 100644 index 0000000000000..70e3f5340e223 --- /dev/null +++ b/docs/changelog/100238.yaml @@ -0,0 +1,6 @@ +pr: 100238 +summary: "ESQL: Remove aliasing inside Eval" +area: ES|QL +type: bug +issues: + - 100174 diff --git a/docs/changelog/100273.yaml b/docs/changelog/100273.yaml new file mode 100644 index 0000000000000..4ccd52d033aa7 --- /dev/null +++ b/docs/changelog/100273.yaml @@ -0,0 +1,5 @@ +pr: 100273 +summary: Propagate cancellation in `GetHealthAction` +area: Health +type: bug +issues: [] diff --git a/docs/changelog/100284.yaml b/docs/changelog/100284.yaml new file mode 100644 index 0000000000000..956fc472d6656 --- /dev/null +++ b/docs/changelog/100284.yaml @@ -0,0 +1,5 @@ +pr: 100284 +summary: Defend against negative datafeed start times +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/100287.yaml b/docs/changelog/100287.yaml new file mode 100644 index 0000000000000..b92855a3342e2 --- /dev/null +++ b/docs/changelog/100287.yaml @@ -0,0 +1,5 @@ +pr: 100287 +summary: Add an assertion to the testTransformFeatureReset test case +area: Transform +type: bug +issues: [] diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java index 866f819609515..49fdc44681aa3 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java @@ -283,7 +283,7 @@ private Context getParentContext(ThreadContext threadContext) { public Releasable withScope(SpanId spanId) { final Context context = spans.get(spanId); if (context != null) { - var scope = context.makeCurrent(); + var scope = AccessController.doPrivileged((PrivilegedAction) context::makeCurrent); return scope::close; } return () -> {}; @@ -381,7 +381,10 @@ public void stopTrace(SpanId spanId) { final var span = Span.fromContextOrNull(spans.remove(spanId)); if (span != null) { logger.trace("Finishing trace [{}]", spanId); - span.end(); + AccessController.doPrivileged((PrivilegedAction) () -> { + span.end(); + return null; + }); } } @@ -390,7 +393,10 @@ public void stopTrace(SpanId spanId) { */ @Override public void stopTrace() { - Span.current().end(); + AccessController.doPrivileged((PrivilegedAction) () -> { + Span.current().end(); + return null; + }); } @Override diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 31d2f6a8e2171..5bb9c8b340ee9 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -20,6 +20,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.matchesRegex; public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase { @@ -100,6 +101,189 @@ public void testDefaultLogsSettingAndMapping() throws Exception { } } + @SuppressWarnings("unchecked") + public void testCustomMapping() throws Exception { + RestClient client = client(); + waitForLogs(client); + + { + Request request = new Request("POST", "/_component_template/logs@custom"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "index": { + "query": { + "default_field": ["custom-message"] + } + } + }, + "mappings": { + "properties": { + "numeric_field": { + "type": "integer" + }, + "socket": { + "properties": { + "ip": { + "type": "keyword" + } + } + } + } + } + } + } + """); + assertOK(client.performRequest(request)); + } + + String dataStreamName = "logs-generic-default"; + createDataStream(client, dataStreamName); + String backingIndex = getWriteBackingIndex(client, dataStreamName); + + // Verify that the custom settings.index.query.default_field overrides the default query field - "message" + Map settings = getSettings(client, backingIndex); + assertThat(settings.get("index.query.default_field"), is(List.of("custom-message"))); + + // Verify that the new field from the custom component template is applied + putMapping(client, backingIndex); + Map mappingProperties = getMappingProperties(client, backingIndex); + assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); + assertThat( + ((Map) mappingProperties.get("socket")).get("properties"), + equalTo(Map.of("ip", Map.of("type", "keyword"))) + ); + + // Insert valid doc and verify successful indexing + { + indexDoc(client, dataStreamName, """ + { + "test": "doc-with-ip", + "socket": { + "ip": "127.0.0.1" + } + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "doc-with-ip" + } + } + }, + "fields": ["socket.ip"] + } + """); + Map fields = ((Map>) results.get(0)).get("_source"); + assertThat(fields.get("socket"), is(Map.of("ip", "127.0.0.1"))); + } + } + + @SuppressWarnings("unchecked") + public void testLogsDefaultPipeline() throws Exception { + RestClient client = client(); + waitForLogs(client); + + { + Request request = new Request("POST", "/_component_template/logs@custom"); + request.setJsonEntity(""" + { + "template": { + "mappings": { + "properties": { + "custom_timestamp": { + "type": "date" + } + } + } + } + } + """); + assertOK(client.performRequest(request)); + } + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "set" : { + "field": "custom_timestamp", + "copy_from": "_ingest.timestamp" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + + String dataStreamName = "logs-generic-default"; + createDataStream(client, dataStreamName); + String backingIndex = getWriteBackingIndex(client, dataStreamName); + + // Verify mapping from custom logs + Map mappingProperties = getMappingProperties(client, backingIndex); + assertThat(((Map) mappingProperties.get("@timestamp")).get("type"), equalTo("date")); + + // no timestamp - testing default pipeline's @timestamp set processor + { + indexDoc(client, dataStreamName, """ + { + "message": "no_timestamp" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "message": { + "value": "no_timestamp" + } + } + }, + "fields": ["@timestamp", "custom_timestamp"] + } + """); + Map source = ((Map>) results.get(0)).get("_source"); + String timestamp = (String) source.get("@timestamp"); + assertThat(timestamp, matchesRegex("[0-9-]+T[0-9:.]+Z")); + assertThat(source.get("custom_timestamp"), is(timestamp)); + + Map fields = ((Map>) results.get(0)).get("fields"); + timestamp = ((List) fields.get("@timestamp")).get(0); + assertThat(timestamp, matchesRegex("[0-9-]+T[0-9:.]+Z")); + assertThat(((List) fields.get("custom_timestamp")).get(0), is(timestamp)); + } + + // verify that when a document is ingested with a timestamp, it does not get overridden + { + indexDoc(client, dataStreamName, """ + { + "message": "with_timestamp", + "@timestamp": "2023-05-10" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "message": { + "value": "with_timestamp" + } + } + }, + "fields": ["@timestamp", "custom_timestamp"] + } + """); + Map fields = ((Map>) results.get(0)).get("fields"); + assertThat(fields.get("@timestamp"), is(List.of("2023-05-10T00:00:00.000Z"))); + } + } + private static void waitForLogs(RestClient client) throws Exception { assertBusy(() -> { try { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_logs_custom_mappings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_logs_custom_mappings.yml deleted file mode 100644 index 27d524c55d6e5..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_logs_custom_mappings.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -"Basic logs@custom component template functionality test": - - do: - cluster.put_component_template: - name: logs@custom - body: - template: - settings: - index: - query: - default_field: [ "custom-message" ] - mappings: - properties: - numeric_field: - type: integer - socket: - properties: - ip: - type: keyword - - - do: - indices.create_data_stream: - name: logs-generic-default - - is_true: acknowledged - - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - indices.get_settings: - index: $idx0name - # verify that the custom settings.index.query.default_field overrides the default query field - "message" - - match: { .$idx0name.settings.index.query.default_field: ["custom-message"] } - - - do: - indices.get_mapping: - index: $idx0name - # verify that the new field from the custom component template is applied - - match: { .$idx0name.mappings.properties.numeric_field.type: "integer" } - - - do: - index: - index: logs-generic-default - refresh: true - body: - test: 'doc-with-ip' - socket: - ip: 127.0.0.1 - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'doc-with-ip' - fields: - - field: 'socket.ip' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.socket.ip: '127.0.0.1' } - - - do: - indices.get_mapping: - index: $idx0name - # test overriding of ECS dynamic template - - match: { .$idx0name.mappings.properties.socket.properties.ip.type: "keyword" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_logs_default_pipeline.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_logs_default_pipeline.yml deleted file mode 100644 index 619df2382d955..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/220_logs_default_pipeline.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -Test default logs-*-* pipeline: - - do: - # setting up a custom field mapping, to test custom pipeline - cluster.put_component_template: - name: logs@custom - body: - template: - mappings: - properties: - custom_timestamp: - type: date - - - do: - ingest.put_pipeline: - # testing custom pipeline - setting a custom timestamp with the same value used to set the `@timestamp` field when missing - id: "logs@custom" - body: > - { - "processors": [ - { - "set" : { - "field": "custom_timestamp", - "copy_from": "_ingest.timestamp" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - - is_true: acknowledged - - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.@timestamp.type: "date" } - - - do: - index: - index: logs-generic-default - refresh: true - body: - # no timestamp - testing default pipeline's @timestamp set processor - message: 'no_timestamp' - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - message: - value: 'no_timestamp' - fields: - - field: '@timestamp' - - field: 'custom_timestamp' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.@timestamp: '/[0-9-]+T[0-9:.]+Z/' } - - set: {hits.hits.0._source.custom_timestamp: custom_timestamp_source } - - match: { hits.hits.0._source.@timestamp: $custom_timestamp_source } - - match: { hits.hits.0.fields.@timestamp.0: '/[0-9-]+T[0-9:.]+Z/' } - - set: {hits.hits.0.fields.custom_timestamp.0: custom_timestamp_field } - - match: { hits.hits.0.fields.@timestamp.0: $custom_timestamp_field } - - # verify that when a document is ingested with a timestamp, it does not get overridden - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - message: 'with_timestamp' - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - message: - value: 'with_timestamp' - fields: - - field: '@timestamp' - - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.@timestamp.0: '2023-05-10T00:00:00.000Z' } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java index 47f0bc01aa9f5..148189152b30a 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java @@ -195,12 +195,8 @@ public void testDeleteByQuery() throws Exception { // Block the write thread pool writeThreadPool.submit(() -> { - try { - barrier.await(); - latch.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); + safeAwait(latch); }); // Ensure that the write thread blocking task is currently executing barrier.await(); diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java index d0b9ad11777e4..47c6e8faf15bf 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java @@ -27,10 +27,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; @@ -77,14 +74,10 @@ public void testSystemIndexMigrationCanBeInterruptedWithShutdown() throws Except if (task != null && task.getState() != null // Make sure the task is really started && hasBlocked.compareAndSet(false, true)) { - try { - logger.info("Task created"); - taskCreated.await(10, TimeUnit.SECONDS); // now we can test internalCluster().restartNode - - shutdownCompleted.await(10, TimeUnit.SECONDS); // waiting until the node has stopped - } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { - throw new AssertionError(e); - } + + logger.info("Task created"); + safeAwait(taskCreated); // now we can test internalCluster().restartNode + safeAwait(shutdownCompleted); // waiting until the node has stopped } }; @@ -96,12 +89,12 @@ public void testSystemIndexMigrationCanBeInterruptedWithShutdown() throws Except client().execute(PostFeatureUpgradeAction.INSTANCE, req); logger.info("migrate feature api called"); - taskCreated.await(10, TimeUnit.SECONDS); // waiting when the task is created + safeAwait(taskCreated); // waiting when the task is created internalCluster().restartNode(masterAndDataNode, new InternalTestCluster.RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { - shutdownCompleted.await(10, TimeUnit.SECONDS);// now we can release the master thread + safeAwait(shutdownCompleted); // now we can release the master thread return super.onNodeStopped(nodeName); } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java index 2ad07dd1c8557..77aecd2b77fed 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java @@ -132,13 +132,7 @@ private void flushThreadPool(ThreadPool threadPool, String executor) throws Inte var maxThreads = threadPool.info(executor).getMax(); var barrier = new CyclicBarrier(maxThreads + 1); for (int i = 0; i < maxThreads; i++) { - threadPool.executor(executor).execute(() -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } - }); + threadPool.executor(executor).execute(() -> safeAwait(barrier)); } barrier.await(10, TimeUnit.SECONDS); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index d52798f1db715..73dcce55aa2a0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -161,11 +161,7 @@ public void sendRequest( // only activated on primary if (action.equals(testActionName)) { actionRunningLatch.countDown(); - try { - actionWaitLatch.await(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(actionWaitLatch); } sender.sendRequest(connection, action, request, options, handler); } @@ -217,11 +213,7 @@ public void testRetryOnStoppedTransportService() throws Exception { // we would not hit the transport service closed case. primaryTransportService.addOnStopListener(() -> { primaryTestPlugin.actionWaitLatch.countDown(); - try { - assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(doneLatch); }); internalCluster().stopNode(primary); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java index 992dfaace6284..51b875ff534ff 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -337,11 +337,7 @@ public void testConcurrentAddBlock() throws InterruptedException { try { for (int i = 0; i < threads.length; i++) { threads[i] = new Thread(() -> { - try { - startClosing.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startClosing); try { indicesAdmin().prepareAddBlock(block, indexName).get(); assertIndexHasBlock(block, indexName); @@ -434,11 +430,7 @@ public void testAddBlockWhileDeletingIndices() throws Exception { try { for (final String indexToDelete : indices) { threads.add(new Thread(() -> { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); try { assertAcked(indicesAdmin().prepareDelete(indexToDelete)); } catch (final Exception e) { @@ -448,11 +440,7 @@ public void testAddBlockWhileDeletingIndices() throws Exception { } for (final String indexToBlock : indices) { threads.add(new Thread(() -> { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); try { indicesAdmin().prepareAddBlock(block, indexToBlock).get(); } catch (final Exception e) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java index daf94b6739f67..4253eeb3b057e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java @@ -88,14 +88,9 @@ public static class BlockingClusterSettingTestPlugin extends Plugin { if (blockOnce.compareAndSet(false, true)) { logger.debug("--> setting validation is now blocking cluster state update"); blockLatch.countDown(); - try { - logger.debug("--> setting validation is now waiting for release"); - releaseLatch.await(); - logger.debug("--> setting validation is done"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError(e); - } + logger.debug("--> setting validation is now waiting for release"); + safeAwait(releaseLatch); + logger.debug("--> setting validation is done"); } }, Setting.Property.NodeScope, Setting.Property.Dynamic); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index 1c4c9cc983255..1413ac453da1b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; @@ -211,7 +210,7 @@ public void testNodeNotReachableFromMaster() throws Exception { ensureStableCluster(3); } - public void testJoinWaitsForClusterApplier() throws Exception { + public void testJoinWaitsForClusterApplier() { startCluster(3); final var masterName = internalCluster().getMasterName(); @@ -223,14 +222,14 @@ public void testJoinWaitsForClusterApplier() throws Exception { final var barrier = new CyclicBarrier(2); internalCluster().getInstance(ClusterService.class, victimName).getClusterApplierService().onNewClusterState("block", () -> { try { - barrier.await(10, TimeUnit.SECONDS); - barrier.await(10, TimeUnit.SECONDS); + safeAwait(barrier); + safeAwait(barrier); return null; } catch (Exception e) { throw new AssertionError(e); } }, ActionListener.noop()); - barrier.await(10, TimeUnit.SECONDS); + safeAwait(barrier); // drop the victim from the cluster with a network disruption final var masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterName); @@ -255,7 +254,7 @@ public void testJoinWaitsForClusterApplier() throws Exception { // release the cluster applier thread logger.info("--> releasing block on victim's applier service"); - barrier.await(10, TimeUnit.SECONDS); + safeAwait(barrier); logger.info("--> waiting for cluster to heal"); ensureStableCluster(3); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java index 357a105ea463c..e22359c30265e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java @@ -115,17 +115,12 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> { if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) { recoveryStarted.countDown(); - try { - blockRecovery.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError(e); - } + safeAwait(blockRecovery); } connection.sendRequest(requestId, action, request, options); }); internalCluster().startDataOnlyNode(); - recoveryStarted.await(); + safeAwait(recoveryStarted); nodeWithReplica = internalCluster().startDataOnlyNode(nodeWithReplicaSettings); // AllocationService only calls GatewayAllocator if there are unassigned shards assertAcked(indicesAdmin().prepareCreate("dummy-index").setWaitForActiveShards(0)); @@ -362,11 +357,7 @@ public void testDoNotCancelRecoveryForBrokenNode() throws Exception { transportService.addSendBehavior((connection, requestId, action, request, options) -> { if (action.equals(PeerRecoveryTargetService.Actions.TRANSLOG_OPS)) { if (brokenNode.equals(connection.getNode().getName())) { - try { - newNodeStarted.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(newNodeStarted); throw new CircuitBreakingException("not enough memory for indexing", 100, 50, CircuitBreaker.Durability.TRANSIENT); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java index 9ecb2f87951b3..29e9d98852a4d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java @@ -194,12 +194,7 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> { if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) { recoveryStarted.countDown(); - try { - blockRecovery.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError(e); - } + safeAwait(blockRecovery); } connection.sendRequest(requestId, action, request, options); }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java new file mode 100644 index 0000000000000..7a9fd0b6ccf60 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthCancellationIT.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.health; + +import org.apache.http.HttpHost; +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.health.node.FetchHealthInfoCacheAction; +import org.elasticsearch.health.node.selection.HealthNode; +import org.elasticsearch.http.HttpInfo; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.transport.netty4.Netty4Plugin; + +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener; +import static org.elasticsearch.test.TaskAssertions.assertAllCancellableTasksAreCancelled; +import static org.elasticsearch.test.TaskAssertions.assertAllTasksHaveFinished; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) +public class GetHealthCancellationIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(getTestTransportPlugin(), MockTransportService.TestPlugin.class); + } + + @Override + protected Settings nodeSettings(int ordinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(ordinal, otherSettings)) + .put(NetworkModule.HTTP_DEFAULT_TYPE_SETTING.getKey(), Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) + .build(); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + public void testCancellation() throws Exception { + internalCluster().startMasterOnlyNode(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "master_node").build()); + internalCluster().startDataOnlyNode(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "data_node").build()); + + final CountDownLatch tasksBlockedLatch = new CountDownLatch(1); + final SubscribableListener fetchHealthInfoRequestReleaseListener = new SubscribableListener<>(); + for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { + ((MockTransportService) transportService).addRequestHandlingBehavior( + FetchHealthInfoCacheAction.NAME, + (handler, request, channel, task) -> { + tasksBlockedLatch.countDown(); + fetchHealthInfoRequestReleaseListener.addListener( + ActionListener.wrap(ignored -> handler.messageReceived(request, channel, task), e -> { + throw new AssertionError("unexpected", e); + }) + ); + } + ); + } + + final ClusterService clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); + final PlainActionFuture findHealthNodeFuture = PlainActionFuture.newFuture(); + // the health node might take a bit of time to be assigned by the persistent task framework so we wait until we have a health + // node in the cluster before proceeding with the test + // proceeding with the execution before the health node assignment would yield a non-deterministic behaviour as we + // wouldn't call the transport service anymore (there wouldn't be a node to fetch the health information from) + final ClusterStateListener clusterStateListener = event -> getHealthNodeIfPresent(event.state(), findHealthNodeFuture); + clusterService.addListener(clusterStateListener); + // look up the node in case the health node was assigned before we registered the listener + getHealthNodeIfPresent(clusterService.state(), findHealthNodeFuture); + DiscoveryNode healthNode = findHealthNodeFuture.get(10, TimeUnit.SECONDS); + assert healthNode != null : "the health node must be assigned"; + clusterService.removeListener(clusterStateListener); + + NodesInfoResponse nodesInfoResponse = clusterAdmin().prepareNodesInfo().get(); + for (NodeInfo node : nodesInfoResponse.getNodes()) { + if (node.getInfo(HttpInfo.class) != null + && Node.NODE_NAME_SETTING.get(node.getSettings()).equals(healthNode.getName()) == false) { + // we don't want the request to hit the health node as it will execute it locally (without going through our stub + // transport service) + TransportAddress publishAddress = node.getInfo(HttpInfo.class).address().publishAddress(); + InetSocketAddress address = publishAddress.address(); + getRestClient().setNodes( + List.of( + new org.elasticsearch.client.Node( + new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), "http") + ) + ) + ); + break; + } + } + + final Request request = new Request(HttpGet.METHOD_NAME, "/_health_report"); + final PlainActionFuture future = new PlainActionFuture<>(); + final Cancellable cancellable = getRestClient().performRequestAsync(request, wrapAsRestResponseListener(future)); + + assertFalse(future.isDone()); + safeAwait(tasksBlockedLatch); // must wait for the fetch health info request to start to avoid cancelling being handled earlier + cancellable.cancel(); + + assertAllCancellableTasksAreCancelled(FetchHealthInfoCacheAction.NAME); + assertAllCancellableTasksAreCancelled(GetHealthAction.NAME); + + fetchHealthInfoRequestReleaseListener.onResponse(null); + expectThrows(CancellationException.class, future::actionGet); + + assertAllTasksHaveFinished(FetchHealthInfoCacheAction.NAME); + assertAllTasksHaveFinished(GetHealthAction.NAME); + } + + private static void getHealthNodeIfPresent(ClusterState event, ActionListener healthNodeReference) { + DiscoveryNode healthNode = HealthNode.findHealthNode(event); + if (healthNode != null) { + healthNodeReference.onResponse(healthNode); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java index 1b330e19f62e8..7c0ef90ca8161 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java @@ -276,12 +276,7 @@ public void testDanglingIndicesImportedAndDeletedCannotBeReimported() throws Exc final Thread[] importThreads = new Thread[2]; for (int i = 0; i < importThreads.length; i++) { importThreads[i] = new Thread(() -> { - try { - startLatch.await(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new AssertionError(e); - } - + safeAwait(startLatch); while (isImporting.get()) { try { clusterAdmin().importDanglingIndex(new ImportDanglingIndexRequest(danglingIndexUUID, true)).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 98d2235a00223..d3aed4a3e2bf2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -393,11 +393,7 @@ public void testCancelNewShardRecoveryAndUsesExistingShardCopy() throws Exceptio transportService.addSendBehavior((connection, requestId, action, request, options) -> { if (PeerRecoveryTargetService.Actions.CLEAN_FILES.equals(action)) { phase1ReadyBlocked.countDown(); - try { - allowToCompletePhase1Latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(allowToCompletePhase1Latch); } connection.sendRequest(requestId, action, request, options); }); @@ -406,7 +402,7 @@ public void testCancelNewShardRecoveryAndUsesExistingShardCopy() throws Exceptio internalCluster().restartNode(nodeB, new InternalTestCluster.RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { - phase1ReadyBlocked.await(); + safeAwait(phase1ReadyBlocked); // nodeB stopped, peer recovery from nodeA to nodeC, it will be cancelled after nodeB get started. RecoveryResponse response = indicesAdmin().prepareRecoveries(INDEX_NAME).execute().actionGet(); @@ -903,11 +899,7 @@ public void testOngoingRecoveryAndMasterFailOver() throws Exception { transport.addSendBehavior((connection, requestId, action, request, options) -> { if (PeerRecoveryTargetService.Actions.CLEAN_FILES.equals(action) && blockRecovery.tryAcquire()) { phase1ReadyBlocked.countDown(); - try { - allowToCompletePhase1Latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(allowToCompletePhase1Latch); } connection.sendRequest(requestId, action, request, options); }); @@ -919,7 +911,7 @@ public void testOngoingRecoveryAndMasterFailOver() throws Exception { .put("index.routing.allocation.include._name", nodeWithPrimary + "," + nodeWithReplica), indexName ); - phase1ReadyBlocked.await(); + safeAwait(phase1ReadyBlocked); internalCluster().restartNode( clusterService().state().nodes().getMasterNode().getName(), new InternalTestCluster.RestartCallback() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java index 579ffea31ca65..d59883020ec83 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java @@ -109,11 +109,7 @@ public Optional getEngineFactory(IndexSettings indexSettings) { @Override public void skipTranslogRecovery() { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); super.skipTranslogRecovery(); } }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java index 2ce96dd0bea60..f20c7162e4715 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseIndexIT.java @@ -211,11 +211,7 @@ public void testConcurrentClose() throws InterruptedException { for (int i = 0; i < threads.length; i++) { threads[i] = new Thread(() -> { - try { - startClosing.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startClosing); try { indicesAdmin().prepareClose(indexName).get(); } catch (final Exception e) { @@ -275,11 +271,7 @@ public void testCloseWhileDeletingIndices() throws Exception { for (final String indexToDelete : indices) { threads.add(new Thread(() -> { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); try { assertAcked(indicesAdmin().prepareDelete(indexToDelete)); } catch (final Exception e) { @@ -289,11 +281,7 @@ public void testCloseWhileDeletingIndices() throws Exception { } for (final String indexToClose : indices) { threads.add(new Thread(() -> { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); try { indicesAdmin().prepareClose(indexToClose).get(); } catch (final Exception e) { @@ -320,19 +308,12 @@ public void testConcurrentClosesAndOpens() throws Exception { waitForDocs(1, indexer); final CountDownLatch latch = new CountDownLatch(1); - final Runnable waitForLatch = () -> { - try { - latch.await(); - } catch (final InterruptedException e) { - throw new AssertionError(e); - } - }; final List threads = new ArrayList<>(); for (int i = 0; i < randomIntBetween(1, 3); i++) { threads.add(new Thread(() -> { try { - waitForLatch.run(); + safeAwait(latch); indicesAdmin().prepareClose(indexName).get(); } catch (final Exception e) { throw new AssertionError(e); @@ -342,7 +323,7 @@ public void testConcurrentClosesAndOpens() throws Exception { for (int i = 0; i < randomIntBetween(1, 3); i++) { threads.add(new Thread(() -> { try { - waitForLatch.run(); + safeAwait(latch); assertAcked(indicesAdmin().prepareOpen(indexName).get()); } catch (final Exception e) { throw new AssertionError(e); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java index a677aebcbfaac..b67098c8f37a7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java @@ -196,9 +196,7 @@ public void testCloseWhileRelocatingShards() throws Exception { for (final String indexToClose : indices) { final Thread thread = new Thread(() -> { try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); + safeAwait(latch); } finally { release.countDown(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java index 2e89656205e0b..2d1c71591233e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java @@ -254,11 +254,7 @@ public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception if (letOneShardProceed.compareAndSet(false, true)) { // Let one shard continue. } else { - try { - shardTaskLatch.await(); // Bock the other shards. - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(shardTaskLatch); // Block the other shards. } }); } diff --git a/server/src/main/java/org/elasticsearch/Build.java b/server/src/main/java/org/elasticsearch/Build.java index be76564e52f6b..6c354b2fb5fa1 100644 --- a/server/src/main/java/org/elasticsearch/Build.java +++ b/server/src/main/java/org/elasticsearch/Build.java @@ -190,7 +190,7 @@ static URL getElasticsearchCodeSourceLocation() { public static Build readBuild(StreamInput in) throws IOException { final String flavor; if (in.getTransportVersion().before(TransportVersions.V_8_3_0) - || in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_039)) { + || in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { flavor = in.readString(); } else { flavor = "default"; @@ -221,7 +221,7 @@ public static Build readBuild(StreamInput in) throws IOException { public static void writeBuild(Build build, StreamOutput out) throws IOException { if (out.getTransportVersion().before(TransportVersions.V_8_3_0) - || out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_039)) { + || out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { out.writeString(build.flavor()); } out.writeString(build.type().displayName()); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 37ea55e9d6617..bd468e22a571a 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -85,25 +85,6 @@ static TransportVersion def(int id) { * Detached transport versions added below here. */ public static final TransportVersion V_8_500_020 = def(8_500_020); - public static final TransportVersion V_8_500_021 = def(8_500_021); - public static final TransportVersion V_8_500_022 = def(8_500_022); - public static final TransportVersion V_8_500_023 = def(8_500_023); - public static final TransportVersion V_8_500_024 = def(8_500_024); - public static final TransportVersion V_8_500_025 = def(8_500_025); - public static final TransportVersion V_8_500_026 = def(8_500_026); - public static final TransportVersion V_8_500_027 = def(8_500_027); - public static final TransportVersion V_8_500_028 = def(8_500_028); - public static final TransportVersion V_8_500_029 = def(8_500_029); - public static final TransportVersion V_8_500_030 = def(8_500_030); - public static final TransportVersion V_8_500_031 = def(8_500_031); - public static final TransportVersion V_8_500_032 = def(8_500_032); - public static final TransportVersion V_8_500_033 = def(8_500_033); - public static final TransportVersion V_8_500_034 = def(8_500_034); - public static final TransportVersion V_8_500_035 = def(8_500_035); - public static final TransportVersion V_8_500_036 = def(8_500_036); - public static final TransportVersion V_8_500_037 = def(8_500_037); - public static final TransportVersion V_8_500_038 = def(8_500_038); - public static final TransportVersion V_8_500_039 = def(8_500_039); public static final TransportVersion V_8_500_040 = def(8_500_040); public static final TransportVersion V_8_500_041 = def(8_500_041); public static final TransportVersion V_8_500_042 = def(8_500_042); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java index 26432922c1f24..31c5f57ab5eef 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.indices.analyze; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.broadcast.BroadcastRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -17,8 +18,6 @@ import java.util.Arrays; import java.util.Objects; -import static org.elasticsearch.TransportVersions.V_8_500_034; - /** * Request for reloading index search analyzers */ @@ -26,7 +25,7 @@ public class ReloadAnalyzersRequest extends BroadcastRequest readLinksBySymbol(InputStream inputStream) throws Exception { try (var parser = XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, inputStream)) { @@ -107,12 +109,34 @@ static Map readLinksBySymbol(InputStream inputStream) throws Exc } /** - * Compute the version component of the URL path (e.g. {@code 8.5} or {@code master}) for a particular version of Elasticsearch. Exposed - * for testing, but all items use {@link #VERSION_COMPONENT} ({@code getVersionComponent(Version.CURRENT, Build.CURRENT.isSnapshot())}) - * which relates to the current version and build. + * Compute the version component of the URL path (e.g. {@code 8.5}, {@code master} or {@code current}) for a particular version of + * Elasticsearch. Exposed for testing, but all items use {@link #VERSION_COMPONENT} + * ({@code getVersionComponent(Build.current().version(), Build.current().isSnapshot())}) which relates to the current version and + * build. */ - static String getVersionComponent(Version version, boolean isSnapshot) { - return isSnapshot && version.revision == 0 ? UNRELEASED_VERSION_COMPONENT : version.major + "." + version.minor; + static String getVersionComponent(String version, boolean isSnapshot) { + + final Pattern semanticVersionPattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d)+"); + + var matcher = semanticVersionPattern.matcher(version); + if (matcher.matches()) { + var major = matcher.group(1); + var minor = matcher.group(2); + var revision = matcher.group(3); + + // x.y.z versions with z>0 mean that x.y.0 is released so have a properly versioned docs link + if (isSnapshot && "0".equals(revision)) { + return UNRELEASED_VERSION_COMPONENT; + } + return major + "." + minor; + } + // Non-semantic version and snapshot -> point to the preliminary documentation for a future release (master) + if (isSnapshot) { + return UNRELEASED_VERSION_COMPONENT; + } + // Non-semantic, released version -> point to latest information (current release documentation, e.g. + // https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-plugins.html) + return CURRENT_VERSION_COMPONENT; } @Override diff --git a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java index b571c3f1f005a..0e4722a872c4e 100644 --- a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java +++ b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.service.ClusterService; @@ -203,8 +204,13 @@ public LocalAction( @Override protected void doExecute(Task task, Request request, ActionListener responseListener) { assert task instanceof CancellableTask; + final CancellableTask cancellableTask = (CancellableTask) task; + if (cancellableTask.notifyIfCancelled(responseListener)) { + return; + } + healthService.getHealth( - client, + new ParentTaskAssigningClient(client, clusterService.localNode(), task), request.indicatorName, request.verbose, request.size, diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java index 97bea569a3c95..fae3dd4069076 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java @@ -39,14 +39,14 @@ public MatchNoneQueryBuilder(String rewriteReason) { */ public MatchNoneQueryBuilder(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_029)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { rewriteReason = in.readOptionalString(); } } @Override protected void doWriteTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_029)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { out.writeOptionalString(rewriteReason); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index 7b7997ce78007..4f6ba803eb7ac 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -91,7 +91,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder void runAsync(Executor executor, CheckedSupplier {}; } - - @Override - public Releasable withScope(Task task) { - return () -> {}; - } }; interface AttributeKeys { diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index b162aa5b6c31a..d8b4ed1ff93c9 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -7,6 +7,7 @@ "CONCURRENT_REPOSITORY_WRITERS": "add-repository.html", "ARCHIVE_INDICES": "archive-indices.html", "HTTP_TRACER": "modules-network.html#http-rest-request-tracer", + "LOGGING": "logging.html", "BOOTSTRAP_CHECK_HEAP_SIZE": "_heap_size_check.html", "BOOTSTRAP_CHECK_FILE_DESCRIPTOR": "_file_descriptor_check.html", "BOOTSTRAP_CHECK_MEMORY_LOCK": "_memory_lock_check.html", diff --git a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java index 4e5b637a42960..09a24f6b76a8e 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -353,11 +352,7 @@ public String toString() { final var startBarrier = new CyclicBarrier(threads.length); for (int i = 0; i < threads.length; i++) { threads[i] = new Thread(() -> { - try { - startBarrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(startBarrier); if (randomBoolean()) { listener.onResponse(null); } else { diff --git a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java index e4469d39061ea..0b582e75c7a68 100644 --- a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; @@ -77,11 +77,10 @@ public static void beforeClass() throws Exception { mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal( - Settings.builder().put("node.name", "node1").build(), - boundAddress.publishAddress(), - randomBase64UUID() - ), + boundAddress -> DiscoveryNodeUtils.builder(randomBase64UUID()) + .applySettings(Settings.builder().put("node.name", "node1").build()) + .address(boundAddress.publishAddress()) + .build(), null, emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java index 8ed61f67d601c..6dea3c1239cdc 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.UUIDs; @@ -123,7 +123,10 @@ private TransportMultiSearchAction createTransportMultiSearchAction(boolean cont mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.builder(UUIDs.randomBase64UUID()) + .applySettings(settings) + .address(boundAddress.publishAddress()) + .build(), null, Collections.emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 4581d0a4f866c..0c2670348f9f6 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -233,11 +233,7 @@ protected void executePhaseOnShard( }); new Thread(() -> { - try { - awaitInitialRequests.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(awaitInitialRequests); Transport.Connection connection = getConnection(null, shard.getNodeId()); TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult( new ShardSearchContextId(UUIDs.randomBase64UUID(), contextIdGenerator.incrementAndGet()), diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java index 8acdf1dcb2147..bba7027f193f3 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -60,7 +59,10 @@ public void testParentTaskId() throws Exception { mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.builder(UUIDs.randomBase64UUID()) + .applySettings(settings) + .address(boundAddress.publishAddress()) + .build(), null, Collections.emptySet() ); @@ -119,7 +121,10 @@ public void testBatchExecute() { mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.builder(UUIDs.randomBase64UUID()) + .applySettings(settings) + .address(boundAddress.publishAddress()) + .build(), null, Collections.emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java index 54b98e26b8e64..d9bf05c413c1f 100644 --- a/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java @@ -47,11 +47,7 @@ public void onFailure(Exception e) { CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i = 0; i < numThreads; i++) { threads[i] = new Thread(() -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); while (count.incrementAndGet() <= groupSize) { listener.onResponse(null); } @@ -126,11 +122,7 @@ public void onFailure(Exception e) { CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i = 0; i < numThreads; i++) { threads[i] = new Thread(() -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); int c; while ((c = count.incrementAndGet()) <= groupSize + overage) { try { diff --git a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java index 10d53ed208ec9..1923047191b96 100644 --- a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java @@ -49,11 +49,7 @@ public void onFailure(Exception e) { CyclicBarrier barrier = new CyclicBarrier(numThreads); for (int i = 0; i < numThreads; i++) { threads[i] = new Thread(() -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); int c; while ((c = count.incrementAndGet()) <= groupSize) { listener.onResponse(c - 1); diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java index 843fe69f511ab..b1f49cb6efd6d 100644 --- a/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java @@ -15,7 +15,6 @@ import java.util.ArrayList; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -122,11 +121,7 @@ public String toString() { async = true; var ref = refs.acquire(); threads[i] = new Thread(() -> { - try { - assertTrue(startLatch.await(10, TimeUnit.SECONDS)); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startLatch); assertFalse(executed.get()); if (randomBoolean()) { ref.onResponse(null); diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index dde9f3d3bd61f..92155333dc507 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -851,12 +851,8 @@ private Runnable blockAllThreads(String executorName) throws Exception { final CountDownLatch latch = new CountDownLatch(1); for (int i = 0; i < numberOfThreads; i++) { executor.submit(() -> { - try { - barrier.await(); - latch.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); + safeAwait(latch); }); } barrier.await(); diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java index a3cbc2663516a..f2614f69cfe8b 100644 --- a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.service.ClusterService; @@ -78,11 +78,10 @@ public static void beforeClass() throws Exception { mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal( - Settings.builder().put("node.name", "node1").build(), - boundAddress.publishAddress(), - randomBase64UUID() - ), + boundAddress -> DiscoveryNodeUtils.builder(randomBase64UUID()) + .applySettings(Settings.builder().put("node.name", "node1").build()) + .address(boundAddress.publishAddress()) + .build(), null, emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index 07762f43c7312..d93a844476463 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -546,7 +546,7 @@ private TestTransportService(Transport transport, ThreadPool threadPool) { transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.create(UUIDs.randomBase64UUID()), null, emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java index c10702dd21638..9287e279fe2f5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java @@ -186,11 +186,7 @@ public void doRun() { final var seed = randomLong(); threads[i] = new Thread(() -> { final var random = new Random(seed); - try { - startBarrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(startBarrier); while (keepGoing.get()) { Thread.yield(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java index ab92ab7917f3b..02c1e731b2e73 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java @@ -12,13 +12,11 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -90,7 +88,7 @@ public void testIsNodeShuttingDown() { ); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); - nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node")); + nodes.add(DiscoveryNodeUtils.create("this_node")); nodes.localNodeId("this_node"); nodes.masterNodeId("this_node"); diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java index ab2e52c9b921c..cc700451931b5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java @@ -144,7 +144,10 @@ public void testDiscoveryNodeIsRemoteClusterClientUnset() { } private void runTestDiscoveryNodeIsRemoteClusterClient(final Settings settings, final boolean expected) { - final DiscoveryNode node = DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node"); + final DiscoveryNode node = DiscoveryNodeUtils.builder("node") + .applySettings(settings) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9200)) + .build(); assertThat(node.isRemoteClusterClient(), equalTo(expected)); if (expected) { assertThat(node.getRoles(), hasItem(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java index cb4239e838ec8..6ef9f274d092f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java @@ -101,7 +101,7 @@ public void testNodeSelection() { } public void testDefaultRolesImpliesTieredDataRoles() { - final DiscoveryNode node = DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), randomAlphaOfLength(8)); + final DiscoveryNode node = DiscoveryNodeUtils.create(randomAlphaOfLength(8)); assertThat(node.getRoles(), hasItem(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE)); assertThat(node.getRoles(), hasItem(DiscoveryNodeRole.DATA_HOT_NODE_ROLE)); assertThat(node.getRoles(), hasItem(DiscoveryNodeRole.DATA_WARM_NODE_ROLE)); @@ -110,7 +110,7 @@ public void testDefaultRolesImpliesTieredDataRoles() { public void testDataRoleDoesNotImplyTieredDataRoles() { final Settings settings = Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "data").build(); - final DiscoveryNode node = DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), randomAlphaOfLength(8)); + final DiscoveryNode node = DiscoveryNodeUtils.builder(randomAlphaOfLength(8)).applySettings(settings).build(); assertThat(node.getRoles(), not(hasItem(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE))); assertThat(node.getRoles(), not(hasItem(DiscoveryNodeRole.DATA_HOT_NODE_ROLE))); assertThat(node.getRoles(), not(hasItem(DiscoveryNodeRole.DATA_WARM_NODE_ROLE))); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java index c63ab61f9616d..f85f3fbd356d9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ContinuousComputationTests.java @@ -65,11 +65,7 @@ protected void processInput(Integer input) { final int threadIndex = i; valuePerThread[threadIndex] = randomInt(); threads[threadIndex] = new Thread(() -> { - try { - assertTrue(startLatch.await(10, TimeUnit.SECONDS)); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(startLatch); for (int j = 1000; j >= 0; j--) { computation.onNewInput(valuePerThread[threadIndex] = valuePerThread[threadIndex] + j); } @@ -90,13 +86,7 @@ protected void processInput(Integer input) { public void testSkipsObsoleteValues() throws Exception { final var barrier = new CyclicBarrier(2); - final Runnable await = () -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } - }; + final Runnable await = () -> safeAwait(barrier); final var initialInput = new Object(); final var becomesStaleInput = new Object(); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 5e9e5bd9ad1dc..966520070ad70 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -593,12 +593,8 @@ record QueueAndExecutor(MasterServiceTaskQueue queue, Executo final var task = new ExpectSuccessTask(); executor.executor().addExpectedTaskCount(1); submitThreads[i] = new Thread(() -> { - try { - assertTrue(submissionLatch.await(10, TimeUnit.SECONDS)); - executor.queue().submitTask(Thread.currentThread().getName(), task, null); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(submissionLatch); + executor.queue().submitTask(Thread.currentThread().getName(), task, null); }, "submit-thread-" + i); } diff --git a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java index 92e37ff005e51..0fabf78017304 100644 --- a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java +++ b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.common; -import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentFactory; @@ -24,13 +23,19 @@ public class ReferenceDocsTests extends ESTestCase { public void testVersionComponent() { // Snapshot x.y.0 versions are unreleased so link to master - assertEquals("master", getVersionComponent(Version.V_8_7_0, true)); + assertEquals("master", getVersionComponent("8.11.0", true)); // Snapshot x.y.z versions with z>0 mean that x.y.0 is released so have a properly versioned docs link - assertEquals("8.5", getVersionComponent(Version.V_8_5_1, true)); + assertEquals("8.5", getVersionComponent("8.5.1", true)); // Non-snapshot versions are to be released so have a properly versioned docs link - assertEquals("8.7", getVersionComponent(Version.V_8_7_0, false)); + assertEquals("8.7", getVersionComponent("8.7.0", false)); + + // Non-snapshot non-semantic versions link to latest docs + assertEquals("current", getVersionComponent("ABCDEF", false)); + + // Snapshot non-semantic versions are considered unreleased so link to master + assertEquals("master", getVersionComponent("ABCDEF", true)); } public void testResourceValidation() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java index d44037884fa4f..6f74972e93988 100644 --- a/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java +++ b/server/src/test/java/org/elasticsearch/common/cache/CacheTests.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -327,7 +326,7 @@ protected long now() { assertEquals(numberOfEntries, cache.stats().getEvictions()); } - public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, InterruptedException { + public void testComputeIfAbsentDeadlock() { final int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder() .setExpireAfterAccess(TimeValue.timeValueNanos(1)) @@ -336,27 +335,23 @@ public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, Interru final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); for (int i = 0; i < numberOfThreads; i++) { final Thread thread = new Thread(() -> { - try { - barrier.await(); - for (int j = 0; j < numberOfEntries; j++) { - try { - cache.computeIfAbsent(0, k -> Integer.toString(k)); - } catch (final ExecutionException e) { - throw new AssertionError(e); - } + safeAwait(barrier); + for (int j = 0; j < numberOfEntries; j++) { + try { + cache.computeIfAbsent(0, k -> Integer.toString(k)); + } catch (final ExecutionException e) { + throw new AssertionError(e); } - barrier.await(); - } catch (final BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); } + safeAwait(barrier); }); thread.start(); } // wait for all threads to be ready - barrier.await(); + safeAwait(barrier); // wait for all threads to finish - barrier.await(); + safeAwait(barrier); } // randomly promote some entries, step the clock forward, then check that the promoted entries remain and the @@ -601,7 +596,7 @@ public void testComputeIfAbsentLoadsSuccessfully() { } } - public void testComputeIfAbsentCallsOnce() throws BrokenBarrierException, InterruptedException { + public void testComputeIfAbsentCallsOnce() { int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder().build(); AtomicReferenceArray flags = new AtomicReferenceArray<>(numberOfEntries); @@ -614,31 +609,27 @@ public void testComputeIfAbsentCallsOnce() throws BrokenBarrierException, Interr CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); for (int i = 0; i < numberOfThreads; i++) { Thread thread = new Thread(() -> { - try { - barrier.await(); - for (int j = 0; j < numberOfEntries; j++) { - try { - cache.computeIfAbsent(j, key -> { - assertTrue(flags.compareAndSet(key, false, true)); - return Integer.toString(key); - }); - } catch (ExecutionException e) { - failures.add(e); - break; - } + safeAwait(barrier); + for (int j = 0; j < numberOfEntries; j++) { + try { + cache.computeIfAbsent(j, key -> { + assertTrue(flags.compareAndSet(key, false, true)); + return Integer.toString(key); + }); + } catch (ExecutionException e) { + failures.add(e); + break; } - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); } + safeAwait(barrier); }); thread.start(); } // wait for all threads to be ready - barrier.await(); + safeAwait(barrier); // wait for all threads to finish - barrier.await(); + safeAwait(barrier); assertThat(failures, is(empty())); } @@ -653,7 +644,7 @@ public void testComputeIfAbsentThrowsExceptionIfLoaderReturnsANullValue() { } } - public void testDependentKeyDeadlock() throws BrokenBarrierException, InterruptedException { + public void testDependentKeyDeadlock() { class Key { private final int key; @@ -689,11 +680,7 @@ public int hashCode() { for (int i = 0; i < numberOfThreads; i++) { Thread thread = new Thread(() -> { try { - try { - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(barrier); Random random = new Random(random().nextLong()); for (int j = 0; j < numberOfEntries; j++) { Key key = new Key(random.nextInt(numberOfEntries)); @@ -745,10 +732,10 @@ public int hashCode() { }, 1, 1, TimeUnit.SECONDS); // everything is setup, release the hounds - barrier.await(); + safeAwait(barrier); // wait for either deadlock to be detected or the threads to terminate - deadlockLatch.await(); + safeAwait(deadlockLatch); // shutdown the watchdog service scheduler.shutdown(); @@ -758,7 +745,7 @@ public int hashCode() { assertFalse("deadlock", deadlock.get()); } - public void testCachePollution() throws BrokenBarrierException, InterruptedException { + public void testCachePollution() { int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder().build(); @@ -766,52 +753,48 @@ public void testCachePollution() throws BrokenBarrierException, InterruptedExcep for (int i = 0; i < numberOfThreads; i++) { Thread thread = new Thread(() -> { - try { - barrier.await(); - Random random = new Random(random().nextLong()); - for (int j = 0; j < numberOfEntries; j++) { - Integer key = random.nextInt(numberOfEntries); - boolean first; - boolean second; - do { - first = random.nextBoolean(); - second = random.nextBoolean(); - } while (first && second); - if (first) { - try { - cache.computeIfAbsent(key, k -> { - if (random.nextBoolean()) { - return Integer.toString(k); - } else { - throw new Exception("testCachePollution"); - } - }); - } catch (ExecutionException e) { - assertNotNull(e.getCause()); - assertThat(e.getCause(), instanceOf(Exception.class)); - assertEquals(e.getCause().getMessage(), "testCachePollution"); - } - } else if (second) { - cache.invalidate(key); - } else { - cache.get(key); + safeAwait(barrier); + Random random = new Random(random().nextLong()); + for (int j = 0; j < numberOfEntries; j++) { + Integer key = random.nextInt(numberOfEntries); + boolean first; + boolean second; + do { + first = random.nextBoolean(); + second = random.nextBoolean(); + } while (first && second); + if (first) { + try { + cache.computeIfAbsent(key, k -> { + if (random.nextBoolean()) { + return Integer.toString(k); + } else { + throw new Exception("testCachePollution"); + } + }); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(Exception.class)); + assertEquals(e.getCause().getMessage(), "testCachePollution"); } + } else if (second) { + cache.invalidate(key); + } else { + cache.get(key); } - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); } + safeAwait(barrier); }); thread.start(); } // wait for all threads to be ready - barrier.await(); + safeAwait(barrier); // wait for all threads to finish - barrier.await(); + safeAwait(barrier); } - public void testExceptionThrownDuringConcurrentComputeIfAbsent() throws BrokenBarrierException, InterruptedException { + public void testExceptionThrownDuringConcurrentComputeIfAbsent() { int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder().build(); @@ -820,60 +803,52 @@ public void testExceptionThrownDuringConcurrentComputeIfAbsent() throws BrokenBa final String key = randomAlphaOfLengthBetween(2, 32); for (int i = 0; i < numberOfThreads; i++) { Thread thread = new Thread(() -> { - try { - barrier.await(); - for (int j = 0; j < numberOfEntries; j++) { - try { - String value = cache.computeIfAbsent(key, k -> { throw new RuntimeException("failed to load"); }); - fail("expected exception but got: " + value); - } catch (ExecutionException e) { - assertNotNull(e.getCause()); - assertThat(e.getCause(), instanceOf(RuntimeException.class)); - assertEquals(e.getCause().getMessage(), "failed to load"); - } + safeAwait(barrier); + for (int j = 0; j < numberOfEntries; j++) { + try { + String value = cache.computeIfAbsent(key, k -> { throw new RuntimeException("failed to load"); }); + fail("expected exception but got: " + value); + } catch (ExecutionException e) { + assertNotNull(e.getCause()); + assertThat(e.getCause(), instanceOf(RuntimeException.class)); + assertEquals(e.getCause().getMessage(), "failed to load"); } - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); } + safeAwait(barrier); }); thread.start(); } // wait for all threads to be ready - barrier.await(); + safeAwait(barrier); // wait for all threads to finish - barrier.await(); + safeAwait(barrier); } // test that the cache is not corrupted under lots of concurrent modifications, even hitting the same key // here be dragons: this test did catch one subtle bug during development; do not remove lightly - public void testTorture() throws BrokenBarrierException, InterruptedException { + public void testTorture() { int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder().setMaximumWeight(1000).weigher((k, v) -> 2).build(); CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); for (int i = 0; i < numberOfThreads; i++) { Thread thread = new Thread(() -> { - try { - barrier.await(); - Random random = new Random(random().nextLong()); - for (int j = 0; j < numberOfEntries; j++) { - Integer key = random.nextInt(numberOfEntries); - cache.put(key, Integer.toString(j)); - } - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); + safeAwait(barrier); + Random random = new Random(random().nextLong()); + for (int j = 0; j < numberOfEntries; j++) { + Integer key = random.nextInt(numberOfEntries); + cache.put(key, Integer.toString(j)); } + safeAwait(barrier); }); thread.start(); } // wait for all threads to be ready - barrier.await(); + safeAwait(barrier); // wait for all threads to finish - barrier.await(); + safeAwait(barrier); cache.refresh(); assertEquals(500, cache.count()); diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java index 8134b39672cc3..7298512603b7a 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java @@ -95,7 +95,7 @@ public void onResponse(Releasable releasable) { assertNoRunningTasks(taskRunner); } - public void testEnqueueSpawnsNewTasksUpToMax() throws Exception { + public void testEnqueueSpawnsNewTasksUpToMax() { int maxTasks = randomIntBetween(1, maxThreads); final int enqueued = maxTasks - 1; // So that it is possible to run at least one more task final int newTasks = randomIntBetween(1, 10); @@ -113,9 +113,7 @@ public void onFailure(Exception e) { @Override public void onResponse(Releasable releasable) { try { - taskBlocker.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); + safeAwait(taskBlocker); } finally { executedCountDown.countDown(); releasable.close(); @@ -138,7 +136,7 @@ public void onResponse(Releasable releasable) { } taskBlocker.countDown(); /// Eventually all tasks are executed - assertTrue(executedCountDown.await(10, TimeUnit.SECONDS)); + safeAwait(executedCountDown); assertTrue(queue.isEmpty()); assertNoRunningTasks(taskRunner); } diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java index b8888f77480cf..fb9bde31e8fc4 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java @@ -176,7 +176,7 @@ public void run() { terminate(executor); } - public void testScaleUp() throws Exception { + public void testScaleUp() { final int min = between(1, 3); final int max = between(min + 1, 6); final CyclicBarrier barrier = new CyclicBarrier(max + 1); @@ -198,23 +198,19 @@ public void testScaleUp() throws Exception { final CountDownLatch latch = new CountDownLatch(1); pool.execute(() -> { latch.countDown(); - try { - barrier.await(); - barrier.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); + safeAwait(barrier); }); // wait until thread executes this task // otherwise, a task might be queued - latch.await(); + safeAwait(latch); } - barrier.await(); + safeAwait(barrier); assertThat("wrong pool size", pool.getPoolSize(), equalTo(max)); assertThat("wrong active size", pool.getActiveCount(), equalTo(max)); - barrier.await(); + safeAwait(barrier); terminate(pool); } @@ -240,23 +236,19 @@ public void testScaleDown() throws Exception { final CountDownLatch latch = new CountDownLatch(1); pool.execute(() -> { latch.countDown(); - try { - barrier.await(); - barrier.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); + safeAwait(barrier); }); // wait until thread executes this task // otherwise, a task might be queued - latch.await(); + safeAwait(latch); } - barrier.await(); + safeAwait(barrier); assertThat("wrong pool size", pool.getPoolSize(), equalTo(max)); assertThat("wrong active size", pool.getActiveCount(), equalTo(max)); - barrier.await(); + safeAwait(barrier); assertBusy(() -> { assertThat("wrong active count", pool.getActiveCount(), equalTo(0)); assertThat("idle threads didn't shrink below max. (" + pool.getPoolSize() + ")", pool.getPoolSize(), lessThan(max)); diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java index 3c999bb468949..4ed3b2348c3ab 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java @@ -17,7 +17,6 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -118,13 +117,13 @@ public void testTryAcquire() throws Exception { int timeout = randomFrom(0, 5, 10); List threads = IntStream.range(0, numberOfThreads).mapToObj(i -> new Thread(() -> { try { - barrier.await(10, TimeUnit.SECONDS); + safeAwait(barrier); try (ReleasableLock locked = lock.tryAcquire(TimeValue.timeValueMillis(timeout))) { if (locked != null) { lockedCounter.incrementAndGet(); } } - } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { + } catch (InterruptedException e) { throw new AssertionError(e); } })).toList(); diff --git a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java index 4ec7e1491c5d3..7c3b478c107c5 100644 --- a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java @@ -114,11 +114,7 @@ public void testResolvesAddressesInBackgroundAndIgnoresConcurrentCalls() throws } seedHostsResolver.resolveConfiguredHosts(resolvedAddresses -> { - try { - assertTrue(startLatch.await(30, TimeUnit.SECONDS)); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startLatch); resolvedAddressesRef.set(resolvedAddresses); endLatch.countDown(); }); diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index c33819911b99e..0573845a73db0 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -558,6 +558,7 @@ public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOE } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100319") public void testIndexCompatibilityChecks() throws IOException { final Settings settings = buildEnvSettings(Settings.EMPTY); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index f020f1d6bd3c5..e17522cd1efef 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -70,11 +71,10 @@ public void testRecoverStateUpdateTask() throws Exception { GatewayService service = createService(Settings.builder()); ClusterStateUpdateTask clusterStateUpdateTask = service.new RecoverStateUpdateTask(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - settings(IndexVersion.current()).put(masterNode()).build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(settings(IndexVersion.current()).put(masterNode()).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); ClusterState stateWithBlock = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK).build()) diff --git a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeExecutorTests.java b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeExecutorTests.java index 1dcce78a1c6ea..071cc4ea5b459 100644 --- a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeExecutorTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -157,7 +157,7 @@ private ClusterState initialState() { Metadata.Builder metadata = Metadata.builder(); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); - nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), localNodeId)); + nodes.add(DiscoveryNodeUtils.create(localNodeId)); nodes.localNodeId(localNodeId); nodes.masterNodeId(localNodeId); diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index e9451ee620d7c..2a72b1fe40ec6 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicInteger; @@ -202,11 +201,7 @@ void getStats(int threadIndex, String... fieldPatterns) { } void start() { - try { - cyclicBarrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - throw new AssertionError(e); - } + safeAwait(cyclicBarrier); } CompletionStats getResult(int index) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index ac279b8549933..aed83cf8abd95 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -2429,11 +2429,7 @@ class OpAndVersion { for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); - try { - startGun.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startGun); for (int op = 0; op < opsPerThread; op++) { Engine.Get engineGet = new Engine.Get(true, false, doc.id()); try (Engine.GetResult get = engine.get(engineGet, mappingLookup, documentParser, randomSearcherWrapper())) { @@ -4375,11 +4371,7 @@ public void testRetryConcurrently() throws InterruptedException, IOException { for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); - try { - startGun.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startGun); int docOffset; while ((docOffset = offset.incrementAndGet()) < docs.size()) { try { @@ -4492,11 +4484,7 @@ public void testAppendConcurrently() throws InterruptedException, IOException { @Override public void run() { startGun.countDown(); - try { - startGun.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startGun); assertThat(engine.getVersionMap().values(), empty()); int docOffset; while ((docOffset = offset.incrementAndGet()) < docs.size()) { @@ -4556,11 +4544,7 @@ public void afterRefresh(boolean didRefresh) throws IOException { if (i == 1) { throw new MockDirectoryWrapper.FakeIOException(); } else if (i == 2) { - try { - start.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(start); throw new ElasticsearchException("something completely different"); } } @@ -4578,11 +4562,7 @@ public void afterRefresh(boolean didRefresh) throws IOException { Engine.Index index = randomBoolean() ? indexForDoc(doc) : randomAppendOnly(doc, false, docId); internalEngine.index(index); Runnable r = () -> { - try { - join.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(join); try { internalEngine.refresh("test"); fail(); @@ -7226,11 +7206,7 @@ public void testRefreshDoesNotBlockClosing() throws Exception { @Override public void beforeRefresh() { refreshStarted.countDown(); - try { - engineClosed.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(engineClosed); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index 64c70f93f8a3f..4bde3d5688d06 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -105,7 +105,7 @@ public void testAppendWhileRecovering() throws Exception { public void run() { try { latch.countDown(); - latch.await(); + safeAwait(latch); shards.appendDocs(numDocs - 1); } catch (Exception e) { throw new AssertionError(e); @@ -126,11 +126,7 @@ public void cleanFiles( ) { super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetadata, ActionListener.runAfter(listener, () -> { latch.countDown(); - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); })); } } @@ -175,11 +171,7 @@ public IndexResult index(Index op) throws IOException { indexedOnPrimary.countDown(); // prevent the indexing on the primary from returning (it was added to Lucene and translog already) // to make sure that this operation is replicated to the replica via recovery, then via replication. - try { - recoveryDone.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(recoveryDone); } return result; } @@ -202,11 +194,7 @@ public IndexResult index(Index op) throws IOException { (shard, node) -> new RecoveryTarget(shard, node, 0L, null, null, recoveryListener) { @Override public void prepareForTranslogOperations(int totalTranslogOps, ActionListener listener) { - try { - indexedOnPrimary.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(indexedOnPrimary); super.prepareForTranslogOperations(totalTranslogOps, listener); } } diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java index 6773b06729a8e..798f52cfbdc19 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java @@ -538,11 +538,7 @@ public void indexTranslogOperations( } catch (final Exception e) { throw new AssertionError(e); } - try { - phaseTwoStartLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(phaseTwoStartLatch); super.indexTranslogOperations( operations, totalTranslogOps, @@ -898,11 +894,7 @@ public long addDocument(final Iterable doc) throws IOE if (latch != null) { latch.countDown(); } - try { - block.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(block); } return super.addDocument(doc); } diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index a368498116095..de8dbc3a1515e 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -726,12 +725,12 @@ public void testPersistRetentionLeasesUnderConcurrency() throws IOException { final String id = Integer.toString(length + i); threads[i] = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); replicationTracker.addRetentionLease(id, retainingSequenceNumber, "test-" + id, ActionListener.noop()); replicationTracker.persistRetentionLeases(path); - barrier.await(); - } catch (final BrokenBarrierException | InterruptedException | WriteStateException e) { + safeAwait(barrier); + } catch (final WriteStateException e) { throw new AssertionError(e); } }); @@ -740,13 +739,13 @@ public void testPersistRetentionLeasesUnderConcurrency() throws IOException { try { // synchronize the threads invoking ReplicationTracker#persistRetentionLeases(Path path) - barrier.await(); + safeAwait(barrier); // wait for all the threads to finish - barrier.await(); + safeAwait(barrier); for (int i = 0; i < numberOfThreads; i++) { threads[i].join(); } - } catch (final BrokenBarrierException | InterruptedException e) { + } catch (final InterruptedException e) { throw new AssertionError(e); } assertThat(replicationTracker.loadRetentionLeases(path), equalTo(replicationTracker.getRetentionLeases())); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java index e61c9620a8988..ba661b97ce35c 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java @@ -192,15 +192,15 @@ public void testMarkAllocationIdAsInSync() throws Exception { final CyclicBarrier barrier = new CyclicBarrier(2); final Thread thread = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); tracker.markAllocationIdAsInSync(replicaId.getId(), randomLongBetween(NO_OPS_PERFORMED, localCheckpoint - 1)); - barrier.await(); - } catch (BrokenBarrierException | InterruptedException e) { + safeAwait(barrier); + } catch (InterruptedException e) { throw new AssertionError(e); } }); thread.start(); - barrier.await(); + safeAwait(barrier); assertBusy(() -> assertTrue(tracker.pendingInSync())); final long updatedLocalCheckpoint = randomLongBetween(1 + localCheckpoint, Long.MAX_VALUE); // there is a shard copy pending in sync, the global checkpoint can not advance @@ -210,7 +210,7 @@ public void testMarkAllocationIdAsInSync() throws Exception { // we are implicitly marking the pending in sync copy as in sync with the current global checkpoint, no advancement should occur tracker.updateLocalCheckpoint(replicaId.getId(), localCheckpoint); assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO)); - barrier.await(); + safeAwait(barrier); thread.join(); // now we expect that the global checkpoint would advance tracker.markAllocationIdAsInSync(replicaId.getId(), updatedLocalCheckpoint); diff --git a/server/src/test/java/org/elasticsearch/index/shard/GlobalCheckpointListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/GlobalCheckpointListenersTests.java index 70387f98d7c3a..a9723a76e913c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/GlobalCheckpointListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/GlobalCheckpointListenersTests.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -455,7 +454,7 @@ public void testConcurrency() throws Exception { final AtomicBoolean closed = new AtomicBoolean(); final Thread updatingThread = new Thread(() -> { // synchronize starting with the listener thread and the main test thread - awaitQuietly(barrier); + safeAwait(barrier); for (int i = 0; i < numberOfIterations; i++) { if (i > numberOfIterations / 2 && rarely() && closed.get() == false) { closed.set(true); @@ -470,13 +469,13 @@ public void testConcurrency() throws Exception { } } // synchronize ending with the listener thread and the main test thread - awaitQuietly(barrier); + safeAwait(barrier); }); final List invocations = new CopyOnWriteArrayList<>(); final Thread listenersThread = new Thread(() -> { // synchronize starting with the updating thread and the main test thread - awaitQuietly(barrier); + safeAwait(barrier); for (int i = 0; i < numberOfIterations; i++) { final AtomicBoolean invocation = new AtomicBoolean(); invocations.add(invocation); @@ -502,7 +501,7 @@ public void accept(final long g, final Exception e) { ); } // synchronize ending with the updating thread and the main test thread - awaitQuietly(barrier); + safeAwait(barrier); }); updatingThread.start(); listenersThread.start(); @@ -652,13 +651,4 @@ public void accept(final long g, final Exception e) { return globalCheckpointListener; } } - - private void awaitQuietly(final CyclicBarrier barrier) { - try { - barrier.await(); - } catch (final BrokenBarrierException | InterruptedException e) { - throw new AssertionError(e); - } - } - } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 4850cd4f4cf5e..3c01515ee5df2 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -3993,11 +3993,7 @@ public void testFlushOnIdleConcurrentFlushDoesNotWait() throws Exception { IndexShard shard = newStartedShard(false, Settings.EMPTY, config -> new InternalEngine(config) { @Override protected void commitIndexWriter(final IndexWriter writer, final Translog translog) throws IOException { - try { - readyToCompleteFlushLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(readyToCompleteFlushLatch); super.commitIndexWriter(writer, translog); } }); @@ -4278,22 +4274,18 @@ public void recoverFromTranslog( ActionListener listener ) { readyToCloseLatch.countDown(); - try { - closeDoneLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(closeDoneLatch); super.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo, listener); } }); Thread closeShardThread = new Thread(() -> { try { - readyToCloseLatch.await(); + safeAwait(readyToCloseLatch); shard.close("testing", false); // in integration tests, this is done as a listener on IndexService. MockFSDirectoryFactory.checkIndex(logger, shard.store(), shard.shardId); - } catch (InterruptedException | IOException e) { + } catch (IOException e) { throw new AssertionError(e); } finally { closeDoneLatch.countDown(); @@ -4317,7 +4309,7 @@ public void recoverFromTranslog( TimeValue.timeValueMinutes(1L) ); - engineResetLatch.await(); + safeAwait(engineResetLatch); closeShardThread.join(); @@ -4352,7 +4344,7 @@ public void recoverFromTranslog( Thread snapshotThread = new Thread(() -> { try { - readyToSnapshotLatch.await(); + safeAwait(readyToSnapshotLatch); shard.snapshotStoreMetadata(); try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(randomBoolean())) { shard.store().getMetadata(indexCommitRef.getIndexCommit()); @@ -4360,7 +4352,7 @@ public void recoverFromTranslog( try (Engine.IndexCommitRef indexCommitRef = shard.acquireSafeIndexCommit()) { shard.store().getMetadata(indexCommitRef.getIndexCommit()); } - } catch (InterruptedException | IOException e) { + } catch (IOException e) { throw new AssertionError(e); } finally { snapshotDoneLatch.countDown(); @@ -4611,13 +4603,9 @@ public void testCloseShardWhileEngineIsWarming() throws Exception { CountDownLatch warmerBlocking = new CountDownLatch(1); IndexShard shard = newShard(true, Settings.EMPTY, config -> { Engine.Warmer warmer = reader -> { - try { - warmerStarted.countDown(); - warmerBlocking.await(); - config.getWarmer().warm(reader); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + warmerStarted.countDown(); + safeAwait(warmerBlocking); + config.getWarmer().warm(reader); }; EngineConfig configWithWarmer = new EngineConfig( config.getShardId(), diff --git a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 0c21e80290bd3..7535f900ff2d1 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -405,11 +405,7 @@ public void testSkipIfPendingAlready() throws Exception { IndexShard shard = newStartedShard(randomBoolean(), Settings.EMPTY, config -> new InternalEngine(config) { @Override public void writeIndexingBuffer() throws IOException { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); super.writeIndexingBuffer(); } }); diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 8a1aee1bed717..4195c98f32391 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -394,10 +394,8 @@ void overLimitTriggered(boolean leader) { for (int i = 0; i < threadCount; ++i) { threads.add(new Thread(() -> { try { - barrier.await(10, TimeUnit.SECONDS); + safeAwait(barrier); service.checkParentLimit(0, "test-thread"); - } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { - throw new AssertionError(e); } catch (CircuitBreakingException e) { // very rare logger.info("Thread got semi-unexpected circuit breaking exception", e); @@ -592,11 +590,7 @@ void overLimitTriggered(boolean leader) { CyclicBarrier barrier = new CyclicBarrier(threadCount + 1); AtomicReference countDown = new AtomicReference<>(new CountDownLatch(randomIntBetween(1, 20))); List threads = IntStream.range(0, threadCount).mapToObj(i -> new Thread(() -> { - try { - barrier.await(10, TimeUnit.SECONDS); - } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { - throw new AssertionError(e); - } + safeAwait(barrier); do { HierarchyCircuitBreakerService.MemoryUsage input = new HierarchyCircuitBreakerService.MemoryUsage( randomLongBetween(0, 100), @@ -613,12 +607,12 @@ void overLimitTriggered(boolean leader) { })).toList(); threads.forEach(Thread::start); - barrier.await(20, TimeUnit.SECONDS); + safeAwait(barrier); int iterationCount = randomIntBetween(1, 5); for (int i = 0; i < iterationCount; ++i) { memoryUsage.set(randomLongBetween(0, 100)); - assertTrue(countDown.get().await(20, TimeUnit.SECONDS)); + safeAwait(countDown.get()); assertThat(leaderTriggerCount.get(), lessThanOrEqualTo(i + 1)); assertThat(leaderTriggerCount.get(), greaterThanOrEqualTo(i / 2 + 1)); time.addAndGet(randomLongBetween(interval, interval * 2)); @@ -666,12 +660,8 @@ public void testG1LockTimeout() throws Exception { void overLimitTriggered(boolean leader) { if (leader) { startedBlocking.countDown(); - try { - // this is the central assertion - the overLimit call below should complete in a timely manner. - assertThat(blockingUntil.await(10, TimeUnit.SECONDS), is(true)); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + // this is the central assertion - the overLimit call below should complete in a timely manner. + safeAwait(blockingUntil); } } }; diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index e6223ff3e2bef..c95e72d24bfaa 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -59,6 +59,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexStateServiceUtils; import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedShard; @@ -220,7 +221,10 @@ protected ExecutorService createThreadPoolExecutor() { transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.builder(UUIDs.randomBase64UUID()) + .applySettings(SETTINGS) + .address(boundAddress.publishAddress()) + .build(), clusterSettings, Collections.emptySet(), Tracer.NOOP diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 277d0472d738f..89ecd0c618c4e 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -518,7 +518,10 @@ private IndicesClusterStateService createIndicesClusterStateService( mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.builder(UUIDs.randomBase64UUID()) + .applySettings(settings) + .address(boundAddress.publishAddress()) + .build(), null, Collections.emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java index efc615403e169..91cdb1d85bc8a 100644 --- a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java +++ b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -124,8 +124,8 @@ public void testNodeRemoval() throws Exception { ClusterState previousState = ClusterState.builder(new ClusterName("cluster")) .nodes( DiscoveryNodes.builder() - .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node1")) - .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2")) + .add(DiscoveryNodeUtils.create("node1", new TransportAddress(TransportAddress.META_ADDRESS, 9200))) + .add(DiscoveryNodeUtils.create("node2", new TransportAddress(TransportAddress.META_ADDRESS, 9201))) ) .build(); ClusterState newState = ClusterState.builder(previousState) diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index f6a8d3a10aafb..6adfe18709220 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -515,7 +515,7 @@ public void testPeriodicRecheckOffMaster() { // Now simulate the node ceasing to be the master builder = ClusterState.builder(clusterState); nodes = DiscoveryNodes.builder(clusterState.nodes()); - nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "a_new_master_node")); + nodes.add(DiscoveryNodeUtils.create("a_new_master_node")); nodes.masterNodeId("a_new_master_node"); ClusterState nonMasterClusterState = builder.nodes(nodes).build(); event = new ClusterChangedEvent("test", nonMasterClusterState, clusterState); @@ -1037,7 +1037,7 @@ private ClusterState initialState() { } DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); - nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node")); + nodes.add(DiscoveryNodeUtils.create("this_node")); nodes.localNodeId("this_node"); nodes.masterNodeId("this_node"); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java index ec5c1842aed25..7c85cba4c34eb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; @@ -82,7 +81,7 @@ private ClusterState createInitialClusterState(int nonLocalNodesCount, Settings state.metadata(Metadata.builder().generateClusterUuidIfNeeded()); state.routingTable(RoutingTable.builder().build()); DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(); - nodes.add(DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), "this_node")); + nodes.add(DiscoveryNodeUtils.builder("this_node").applySettings(settings).build()); for (int i = 0; i < nonLocalNodesCount; i++) { nodes.add(DiscoveryNodeUtils.create("other_node_" + i)); } diff --git a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java index 7349af72d682a..9e680615019dc 100644 --- a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java +++ b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java @@ -159,8 +159,7 @@ public void testTCPProbe() throws Exception { // mocking a cluster change event, with a master down ClusterState previousState = ClusterState.builder(new ClusterName("cluster")) .nodes( - DiscoveryNodes.builder() - .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2")) + DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("node2", new TransportAddress(TransportAddress.META_ADDRESS, 9201))) ) .build(); @@ -193,8 +192,7 @@ public void testStatusChange() throws Exception { ClusterState previousState = ClusterState.builder(new ClusterName("cluster")) .nodes( - DiscoveryNodes.builder() - .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2")) + DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("node2", new TransportAddress(TransportAddress.META_ADDRESS, 9201))) ) .build(); diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 810f8e897e938..95389b5632613 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -73,7 +74,7 @@ public void setUp() throws Exception { mock(Transport.class), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + boundAddress -> DiscoveryNodeUtils.create(UUIDs.randomBase64UUID(), boundAddress.publishAddress()), null, Collections.emptySet() ); diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index 8e9e43267b4fa..3ee2b56f5d698 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -129,15 +129,11 @@ public void testSnapshotShardSizes() throws Exception { final Repository mockRepository = new FilterRepository(mock(Repository.class)) { @Override public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) { - try { - assertThat(indexId.getName(), equalTo(indexName)); - assertThat(shardId.id(), allOf(greaterThanOrEqualTo(0), lessThan(numberOfShards))); - latch.await(); - getShardSnapshotStatusCount.incrementAndGet(); - return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, expectedShardSizes[shardId.id()], null); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + assertThat(indexId.getName(), equalTo(indexName)); + assertThat(shardId.id(), allOf(greaterThanOrEqualTo(0), lessThan(numberOfShards))); + safeAwait(latch); + getShardSnapshotStatusCount.incrementAndGet(); + return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, expectedShardSizes[shardId.id()], null); } }; when(repositoriesService.repository("_repo")).thenReturn(mockRepository); diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java index 9f770c1f34a3d..8d7a486ee79f0 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java @@ -338,7 +338,7 @@ public void testScalingThreadPoolRejectDuringShutdown() throws Exception { if (t == 0) { threads[t] = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); scalingExecutor.shutdown(); } catch (Exception e) { throw new AssertionError(e); @@ -347,7 +347,7 @@ public void testScalingThreadPoolRejectDuringShutdown() throws Exception { } else { threads[t] = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); execute(scalingExecutor, () -> {}, executed, rejected, failed); } catch (Exception e) { throw new AssertionError(e); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java index 135d0378a158e..47d75a78d65d4 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterClientTests.java @@ -236,11 +236,7 @@ public void testQuicklySkipUnavailableClusters() throws Exception { ) { CountDownLatch latch = new CountDownLatch(1); service.addConnectBehavior(remoteTransport, (transport, discoveryNode, profile, listener) -> { - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); listener.onFailure(new ConnectTransportException(discoveryNode, "simulated")); }); service.start(); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 2d0956fae5499..198eff76b172d 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -373,7 +373,7 @@ public void run() { latch.countDown(); } } - latch.await(); + safeAwait(latch); } catch (Exception ex) { throw new AssertionError(ex); } @@ -786,7 +786,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted final int numGetCalls = randomIntBetween(1000, 10000); getThreads[i] = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); for (int j = 0; j < numGetCalls; j++) { try { Transport.Connection lowLevelConnection = connection.getConnection(); @@ -806,7 +806,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted final int numDisconnects = randomIntBetween(5, 10); modifyingThreads[i] = new Thread(() -> { try { - barrier.await(); + safeAwait(barrier); for (int j = 0; j < numDisconnects; j++) { DiscoveryNode node = randomFrom(discoverableNodes); try { diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index c5aa918fdc0e2..2c10f47955c4c 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -51,11 +51,7 @@ public void testHandlersCompleteAtShutdown() throws Exception { for (int i = 0; i < threads.length; i++) { final var seed = randomLong(); threads[i] = new Thread(() -> { - try { - startBarrier.await(10, TimeUnit.SECONDS); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(startBarrier); final var random = new Random(seed); while (keepGoing.get() && requestPermits.tryAcquire()) { nodeB.transportService.sendRequest( diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java index 6064573af15c3..3adb448926f84 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java @@ -11,6 +11,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -52,10 +53,6 @@ public void onFailure(Exception ex) { new PostWriteRefresh(transportService), null ).run(); - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + ESTestCase.safeAwait(latch); } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java b/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java index 5a70c9c461475..215c30525fd40 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeUtils.java @@ -10,8 +10,10 @@ import org.elasticsearch.Version; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.node.Node; import java.util.Map; import java.util.Objects; @@ -128,6 +130,12 @@ public Builder externalId(String externalId) { return this; } + public Builder applySettings(Settings settings) { + return name(Node.NODE_NAME_SETTING.get(settings)).attributes(Node.NODE_ATTRIBUTES.getAsMap(settings)) + .roles(DiscoveryNode.getRolesFromSettings(settings)) + .externalId(Node.NODE_EXTERNAL_ID_SETTING.get(settings)); + } + public DiscoveryNode build() { if (address == null) { address = buildNewFakeTransportAddress(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 03e61d5cb3037..bcb42f519a290 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1173,11 +1173,7 @@ public static void concurrentlyApplyOps(List ops, InternalEngi for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); - try { - startGun.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(startGun); int docOffset; while ((docOffset = offset.incrementAndGet()) < ops.size()) { try { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 51b3a9b4be15c..3a9b918e654e5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -209,11 +209,10 @@ public MockTransportService( new StubbableTransport(transport), threadPool, interceptor, - (boundAddress) -> DiscoveryNode.createLocal( - settings, - boundAddress.publishAddress(), - settings.get(Node.NODE_NAME_SETTING.getKey(), UUIDs.randomBase64UUID()) - ), + (boundAddress) -> DiscoveryNodeUtils.builder(settings.get(Node.NODE_NAME_SETTING.getKey(), UUIDs.randomBase64UUID())) + .applySettings(settings) + .address(boundAddress.publishAddress()) + .build(), clusterSettings, createTaskManager(settings, threadPool, Set.of(), Tracer.NOOP) ); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java index dad4972a7a9d8..eaade3bbc8eef 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java @@ -57,7 +57,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -336,11 +335,7 @@ public void testConcurrentStateUpdate() throws Exception { client.respondStats((request, listener) -> { CountDownLatch latch = new CountDownLatch(1); threads.add(startThread(() -> { - try { - assertThat(latch.await(10, TimeUnit.SECONDS), is(true)); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); listener.onResponse(response); })); threads.add(startThread(() -> { diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java index d955b521891f2..8c33e32dbf0c4 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java @@ -373,12 +373,7 @@ public void testThreadSafety() throws InterruptedException { final Set listenersCalled = newConcurrentSet(); for (int threadIndex = 0; threadIndex < threads.length; threadIndex++) { threads[threadIndex] = new Thread(() -> { - try { - startLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } - + safeAwait(startLatch); while (countDown.tryAcquire()) { waitForRandomRange(fileContents, sparseFileTracker, listenersCalled::add, gap -> processGap(fileContents, gap)); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 845997872ed8d..26eb241c0293e 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -275,7 +275,7 @@ public void testReadRequestsReturnLatestMappingVersion() throws Exception { && XContentMapValues.extractValue("properties.balance.type", imd.mapping().sourceAsMap()) != null) { try { logger.info("--> block ClusterService from exposing new mapping version"); - latch.await(); + safeAwait(latch); } catch (Exception e) { throw new AssertionError(e); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 4a8e16e58e13c..0abfc6e911d2d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -571,7 +571,7 @@ private void runFollowTest(CheckedBiConsumer { try { latch.countDown(); - latch.await(); + safeAwait(latch); fetchOperations(taskIsCompleted, lastFetchedSeqNo, leader, follower); } catch (Exception e) { throw new AssertionError(e); @@ -581,7 +581,7 @@ private void runFollowTest(CheckedBiConsumer indices = Map.of(index.getName(), indexMetadata); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java index 74ef92edd35d0..3f4b1adf8253b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ShrunkShardsAllocatedStepTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; @@ -77,11 +78,10 @@ public void testConditionMet() { Index shrinkIndex = shrunkIndexMetadata.getIndex(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); IndexRoutingTable.Builder builder = IndexRoutingTable.builder(shrinkIndex); for (int i = 0; i < shrinkNumberOfShards; i++) { @@ -121,11 +121,10 @@ public void testConditionNotMetBecauseOfActive() { Index shrinkIndex = shrunkIndexMetadata.getIndex(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); IndexRoutingTable.Builder builder = IndexRoutingTable.builder(shrinkIndex); for (int i = 0; i < shrinkNumberOfShards; i++) { @@ -157,11 +156,10 @@ public void testConditionNotMetBecauseOfShrunkIndexDoesntExistYet() { .build(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java index 64cc24eb8eee2..e9364a37ea30d 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java @@ -98,11 +98,7 @@ public void testFrom() throws IOException { ) .build(); - DiscoveryNode discoveryNode = DiscoveryNode.createLocal( - Settings.EMPTY, - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - "test" - ); + DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("test", new TransportAddress(TransportAddress.META_ADDRESS, 9300)); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); boolean clusterIssueFound = randomBoolean(); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java index 0f445fe50e9d7..ad1626fd2f4b3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java @@ -73,7 +73,7 @@ public class RuleQueryBuilder extends AbstractQueryBuilder { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_500_033; + return TransportVersions.V_8_500_040; } public RuleQueryBuilder(QueryBuilder organicQuery, Map matchCriteria, String rulesetId) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index 0640347a7ea91..c22cf7d390628 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -281,7 +281,7 @@ private Event(StreamInput in) throws IOException { } else { fetchFields = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_038)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { missing = in.readBoolean(); } else { missing = index.isEmpty(); @@ -304,7 +304,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(fetchFields, StreamOutput::writeWriteable); } } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_038)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_500_040)) { // for BWC, 8.9.1+ does not have "missing" attribute, but it considers events with an empty index "" as missing events // see https://github.com/elastic/elasticsearch/pull/98130 out.writeBoolean(missing); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java index 765fd94d4c6be..080cc26d81eb2 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java @@ -289,7 +289,7 @@ private List mutateEvents(List original, TransportVersion version) e.id(), e.source(), version.onOrAfter(TransportVersions.V_7_13_0) ? e.fetchFields() : null, - version.onOrAfter(TransportVersions.V_8_500_038) ? e.missing() : e.index().isEmpty() + version.onOrAfter(TransportVersions.V_8_500_040) ? e.missing() : e.index().isEmpty() ) ); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java index 80f396695fc2f..5b58e7bcf5c30 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java @@ -44,12 +44,16 @@ default String getWriteableName() { NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "BooleanBlock", BooleanBlock::readFrom); private static BooleanBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static BooleanBlock readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return BooleanVector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return BooleanVector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try (BooleanBlock.Builder builder = ((BlockStreamInput) in).blockFactory().newBooleanBlockBuilder(positions)) { + try (BooleanBlock.Builder builder = in.blockFactory().newBooleanBlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java index 9409212a9c998..9c48ac61d5a1b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java @@ -48,12 +48,16 @@ default String getWriteableName() { NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "BytesRefBlock", BytesRefBlock::readFrom); private static BytesRefBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static BytesRefBlock readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return BytesRefVector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return BytesRefVector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try (BytesRefBlock.Builder builder = ((BlockStreamInput) in).blockFactory().newBytesRefBlockBuilder(positions)) { + try (BytesRefBlock.Builder builder = in.blockFactory().newBytesRefBlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java index 806ee6d3680bc..a3dba750556ab 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java @@ -44,12 +44,16 @@ default String getWriteableName() { NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "DoubleBlock", DoubleBlock::readFrom); private static DoubleBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static DoubleBlock readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return DoubleVector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return DoubleVector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try (DoubleBlock.Builder builder = ((BlockStreamInput) in).blockFactory().newDoubleBlockBuilder(positions)) { + try (DoubleBlock.Builder builder = in.blockFactory().newDoubleBlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java index 580da5e5a7415..d343428aab2bc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java @@ -44,12 +44,16 @@ default String getWriteableName() { NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "IntBlock", IntBlock::readFrom); private static IntBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static IntBlock readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return IntVector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return IntVector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try (IntBlock.Builder builder = ((BlockStreamInput) in).blockFactory().newIntBlockBuilder(positions)) { + try (IntBlock.Builder builder = in.blockFactory().newIntBlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java index 2db757efd7091..9ff3a5ba116a4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java @@ -44,12 +44,16 @@ default String getWriteableName() { NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "LongBlock", LongBlock::readFrom); private static LongBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static LongBlock readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return LongVector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return LongVector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try (LongBlock.Builder builder = ((BlockStreamInput) in).blockFactory().newLongBlockBuilder(positions)) { + try (LongBlock.Builder builder = in.blockFactory().newLongBlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index 0e93bc1ee5e90..ad5dfbf298200 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -214,7 +214,7 @@ public IntBlock newConstantIntBlockWith(int value, int positions, long preAdjust public IntVector newConstantIntVector(int value, int positions) { adjustBreaker(ConstantIntVector.RAM_BYTES_USED, false); var v = new ConstantIntVector(value, positions, this); - assert v.ramBytesUsed() == ConstantLongVector.RAM_BYTES_USED; + assert v.ramBytesUsed() == ConstantIntVector.RAM_BYTES_USED; return v; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java index 18f3ed7ba61bf..0265013eb2029 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java @@ -97,8 +97,16 @@ public Page(StreamInput in) throws IOException { int positionCount = in.readVInt(); int blockPositions = in.readVInt(); Block[] blocks = new Block[blockPositions]; - for (int blockIndex = 0; blockIndex < blockPositions; blockIndex++) { - blocks[blockIndex] = in.readNamedWriteable(Block.class); + boolean success = false; + try { + for (int blockIndex = 0; blockIndex < blockPositions; blockIndex++) { + blocks[blockIndex] = in.readNamedWriteable(Block.class); + } + success = true; + } finally { + if (success == false) { + Releasables.closeExpectNoException(blocks); + } } this.positionCount = positionCount; this.blocks = blocks; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st index 81a0d3de7f8f7..e8ccc83b51351 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st @@ -60,12 +60,16 @@ $endif$ NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "$Type$Block", $Type$Block::readFrom); private static $Type$Block readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + private static $Type$Block readFrom(BlockStreamInput in) throws IOException { final boolean isVector = in.readBoolean(); if (isVector) { - return $Type$Vector.readFrom(((BlockStreamInput) in).blockFactory(), in).asBlock(); + return $Type$Vector.readFrom(in.blockFactory(), in).asBlock(); } final int positions = in.readVInt(); - try ($Type$Block.Builder builder = ((BlockStreamInput) in).blockFactory().new$Type$BlockBuilder(positions)) { + try ($Type$Block.Builder builder = in.blockFactory().new$Type$BlockBuilder(positions)) { for (int i = 0; i < positions; i++) { if (in.readBoolean()) { builder.appendNull(); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java index 206e6b48db2f7..84c654f8946fb 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java @@ -104,7 +104,6 @@ private Response sortByManyLongs(int count) throws IOException { /** * This groups on about 200 columns which is a lot but has never caused us trouble. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testGroupOnSomeLongs() throws IOException { initManyLongs(); Map map = XContentHelper.convertToMap( @@ -168,6 +167,7 @@ public void testSmallConcat() throws IOException { assertMap(map, matchesMap().entry("columns", columns).entry("values", values)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testHugeConcat() throws IOException { initSingleDocIndex(); assertCircuitBreaks(() -> concat(10)); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 1655686ab4f20..953fb65bd1eec 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -342,6 +342,7 @@ public enum Type { LOOKUP.put("BYTE", INTEGER); // add also the types with short names + LOOKUP.put("BOOL", BOOLEAN); LOOKUP.put("I", INTEGER); LOOKUP.put("L", LONG); LOOKUP.put("UL", UNSIGNED_LONG); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec index 8d20e48ef1552..54bc481c54b48 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec @@ -99,8 +99,7 @@ Anneke Preusig | Anneke | Preusig ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -dissectStats-Ignore +dissectStats from employees | eval x = concat(gender, " foobar") | dissect x "%{a} %{b}" | stats n = max(emp_no) by a | keep a, n | sort a asc; a:keyword | n:integer diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec index 75dfdd783a7cd..9dc9444de0155 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec @@ -83,8 +83,7 @@ Anneke Preusig | Anneke | Preusig ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -grokStats-Ignore +grokStats from employees | eval x = concat(gender, " foobar") | grok x "%{WORD:a} %{WORD:b}" | stats n = max(emp_no) by a | keep a, n | sort a asc; a:keyword | n:integer diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec index 8e5f3459ca95c..13a8b8f66fc4f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec @@ -205,8 +205,7 @@ emp_no:integer | languages:integer | gender:keyword | first_name:keyword | abc:i 10100 | 4 | F | Hironobu | 3 ; -# awaitsfix https://github.com/elastic/elasticsearch/issues/99826 -projectFromWithStatsAfterLimit-Ignore +projectFromWithStatsAfterLimit from employees | sort emp_no | keep gender, avg_worked_seconds, first_name, last_name | limit 10 | stats m = max(avg_worked_seconds) by gender; m:long | gender:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index 799f8821d97da..44cf92254298b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -84,7 +84,8 @@ x:integer | z:integer 4 | 8 ; -renameProjectEval +# AwaitsFix https://github.com/elastic/elasticsearch/issues/100356 +renameProjectEval-Ignore from employees | sort emp_no | eval y = languages | rename languages as x | keep x, y | eval x2 = x + 1 | eval y2 = y + 2 | limit 3; x:integer | y:integer | x2:integer | y2:integer @@ -93,6 +94,17 @@ x:integer | y:integer | x2:integer | y2:integer 4 | 4 | 5 | 6 ; +# AwaitsFix https://github.com/elastic/elasticsearch/issues/100356 +duplicateProjectEval-Ignore +from employees | eval y = languages, x = languages | keep x, y | eval x2 = x + 1 | eval y2 = y + 2 | limit 3; + +x:integer | y:integer | x2:integer | y2:integer +2 | 2 | 3 | 4 +5 | 5 | 6 | 7 +4 | 4 | 5 | 6 +; + + renameWithFilterPushedToES from employees | sort emp_no | rename emp_no as x | keep languages, first_name, last_name, x | where x > 10030 and x < 10040 | limit 5; @@ -147,3 +159,21 @@ y:integer | x:date 10076 | 1985-07-09T00:00:00.000Z 10061 | 1985-09-17T00:00:00.000Z ; + +# AwaitsFix https://github.com/elastic/elasticsearch/issues/100356 +renameIntertwinedWithSort-Ignore +FROM employees | eval x = salary | rename x as y | rename y as x | sort x | rename x as y | limit 10; + +avg_worked_seconds:l | birth_date:date | emp_no:i | first_name:s | gender:s | height:d | height.float:d | height.half_float:d | height.scaled_float:d| hire_date:date | is_rehired:bool | job_positions:s | languages:i | languages.byte:i | languages.long:l | languages.short:i | last_name:s | salary:i | salary_change:d | salary_change.int:i | salary_change.keyword:s | salary_change.long:l | still_hired:bool | y:i + +390266432 | 1959-08-19T00:00:00.000Z | 10015 | Guoxiang | null | 1.66 | 1.659999966621399 | 1.66015625 | 1.6600000000000001 | 1987-07-02T00:00:00.000Z | [false, false, false, true]| [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] | 5 | 5 | 5 | 5 | Nooteboom | 25324 | [12.4, 14.25] | [12, 14] | [12.40, 14.25] | [12, 14] | true | 25324 +203838153 | 1953-02-08T00:00:00.000Z | 10035 | null | M | 1.81 | 1.809999942779541 | 1.8095703125 | 1.81 | 1988-09-05T00:00:00.000Z | false | [Data Scientist, Senior Python Developer] | 5 | 5 | 5 | 5 | Chappelet | 25945 | [-6.58, -2.54] | [-6, -2] | [-2.54, -6.58] | [-6, -2] | false | 25945 +313407352 | 1964-10-18T00:00:00.000Z | 10092 | Valdiodio | F | 1.75 | 1.75 | 1.75 | 1.75 | 1989-09-22T00:00:00.000Z | [false, false, true, true] | [Accountant, Junior Developer] | 1 | 1 | 1 | 1 | Niizuma | 25976 | [-6.77, 0.39, 8.3, 8.78] | [-6, 0, 8, 8] | [-6.77, 0.39, 8.30,8.78] | [-6, 0, 8, 8] | false | 25976 +248451647 | null | 10048 | Florian | M | 2.0 | 2.0 | 2.0 | 2.0 | 1985-02-24T00:00:00.000Z | [true, true] | Internship | 3 | 3 | 3 | 3 | Syrotiuk | 26436 | null | null | null | null | false | 26436 +324356269 | 1954-05-30T00:00:00.000Z | 10057 | Ebbe | F | 1.59 | 1.590000033378601 | 1.58984375 | 1.59 | 1992-01-15T00:00:00.000Z | null | [Head Human Resources, Python Developer] | 4 | 4 | 4 | 4 | Callaway | 27215 | [-6.73, -5.27, -2.43, 1.03] | [-6, -5, -2, 1] | [-2.43, -5.27, -6.73, 1.03]| [-6, -5, -2, 1] | true | 27215 +359067056 | 1960-05-25T00:00:00.000Z | 10084 | Tuval | M | 1.51 | 1.5099999904632568 | 1.509765625 | 1.51 | 1995-12-15T00:00:00.000Z | false | Principal Support Engineer | 1 | 1 | 1 | 1 | Kalloufi | 28035 | null | null | null | null | true | 28035 +359208133 | 1953-04-03T00:00:00.000Z | 10026 | Yongqiao | M | 2.1 | 2.0999999046325684 | 2.099609375 | 2.1 | 1995-03-20T00:00:00.000Z | [false, true] | Reporting Analyst | null | null | null | null | Berztiss | 28336 | [-7.37, 10.62, 11.20] | [-7, 10, 11] | [-7.37, 10.62, 11.20] | [-7, 10, 11] | true | 28336 +233999584 | 1962-11-26T00:00:00.000Z | 10068 | Charlene | M | 1.58 | 1.5800000429153442 | 1.580078125 | 1.58 | 1987-08-07T00:00:00.000Z | true | Architect | 3 | 3 | 3 | 3 | Brattka | 28941 | [-5.61, -5.29, 3.43] | [-5, -5, 3] | [-5.29, -5.61, 3.43] | [-5, -5, 3] | true | 28941 +341158890 | 1961-10-15T00:00:00.000Z | 10060 | Breannda | M | 1.42 | 1.4199999570846558 | 1.419921875 | 1.42 | 1987-11-02T00:00:00.000Z | [false, false, false, true]| [Business Analyst, Data Scientist, Senior Team Lead] | 2 | 2 | 2 | 2 | Billingsley | 29175 | [-1.76, -0.85] | [-1, 0] | [-0.85, -1.76] | [-1, 0] | true | 29175 +246355863 | null | 10042 | Magy | F | 1.44 | 1.440000057220459 | 1.4404296875 | 1.44 | 1993-03-21T00:00:00.000Z | null | [Architect, Business Analyst, Internship, Junior Developer] | 3 | 3 | 3 | 3 | Stamatiou | 30404 | [-9.28, 9.42] | [-9, 9] | [-9.28, 9.42] | [-9, 9] | true | 30404 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec index 50f0d5b72dd41..1b424dd876caa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/row.csv-spec @@ -170,24 +170,21 @@ a:integer | b:integer | y:integer 1 | 2 | null ; -// AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -rowWithNullsInCount-Ignore +rowWithNullsInCount row a = 1.5, b = 2.6, c = null | eval s = null + a + b | stats c = count(s); c:long 0 ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -rowWithNullsInAvg-Ignore +rowWithNullsInAvg row a = 1.5, b = 2.6, c = null | eval s = null + a + b | stats c = avg(s); c:double null ; -// AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -rowWithNullsInAvg2-Ignore +rowWithNullsInAvg2 row a = 1.5, b = 2.6, c = null | eval s = a - b * c | stats avg(s); avg(s):double @@ -228,16 +225,14 @@ row a = 1 | limit 0; a:integer ; -// AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -rowWithMultipleStats-Ignore +rowWithMultipleStats row a = 1+3, b = 2, ab = 5 | eval x = 1 + b + 5 | stats avg = avg(x), min(x), max(x), count(x), avg(x), avg(ab), avg(a); avg:double | min(x):integer | max(x):integer | count(x):long | avg(x):double | avg(ab):double | avg(a):double 8.0 | 8 | 8 | 1 | 8.0 | 5.0 | 4.0 ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -rowWithMultipleStatsOverNull-Ignore +rowWithMultipleStatsOverNull row x=1, y=2 | eval tot = null + y + x | stats c=count(tot), a=avg(tot), mi=min(tot), ma=max(tot), s=sum(tot); c:long | a:double | mi:integer | ma:integer | s:long diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index 2bb6a2be9832f..60c5fc94ba0d6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -1,5 +1,4 @@ -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -showInfo-Ignore +showInfo show info | stats v = count(version); v:long diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 47a0bd92a56ba..d671ba6ec13b1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -525,8 +525,7 @@ min(salary):i | max(salary):i | c:l 25324 | 74999 | 100 ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -statsWithLiterals-Ignore +statsWithLiterals from employees | limit 10 | eval x = 1 | stats c = count(x); c:l diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec index 08196b2d7726d..df1fa6e67f279 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec @@ -203,8 +203,7 @@ bad 5.2.9-SNAPSHOT ; -# AwaitsFix https://github.com/elastic/elasticsearch/issues/99826 -groupByVersionCast-Ignore +groupByVersionCast FROM apps | EVAL g = TO_VER(CONCAT("1.", TO_STR(version))) | STATS id = MAX(id) BY g | SORT id | DROP g; id:i diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index fd4fe13b9c1b1..46d85746c3990 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.action; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.Build; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -66,6 +67,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100365") public class EsqlActionIT extends AbstractEsqlIntegTestCase { long epoch = System.currentTimeMillis(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index 69dd9c1a50202..53a915046b45f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -105,37 +105,39 @@ public Block.Ref eval(Page page) { */ private Block eval(Block lhs, Block rhs) { int positionCount = lhs.getPositionCount(); - BooleanBlock.Builder result = BooleanBlock.newBlockBuilder(positionCount); - for (int p = 0; p < positionCount; p++) { - if (lhs.getValueCount(p) > 1) { - result.appendNull(); - continue; + try (BooleanBlock.Builder result = BooleanBlock.newBlockBuilder(positionCount, lhs.blockFactory())) { + for (int p = 0; p < positionCount; p++) { + if (lhs.getValueCount(p) > 1) { + result.appendNull(); + continue; + } + if (rhs.getValueCount(p) > 1) { + result.appendNull(); + continue; + } + Boolean v = bl.function() + .apply( + lhs.isNull(p) ? null : ((BooleanBlock) lhs).getBoolean(lhs.getFirstValueIndex(p)), + rhs.isNull(p) ? null : ((BooleanBlock) rhs).getBoolean(rhs.getFirstValueIndex(p)) + ); + if (v == null) { + result.appendNull(); + continue; + } + result.appendBoolean(v); } - if (rhs.getValueCount(p) > 1) { - result.appendNull(); - continue; - } - Boolean v = bl.function() - .apply( - lhs.isNull(p) ? null : ((BooleanBlock) lhs).getBoolean(lhs.getFirstValueIndex(p)), - rhs.isNull(p) ? null : ((BooleanBlock) rhs).getBoolean(rhs.getFirstValueIndex(p)) - ); - if (v == null) { - result.appendNull(); - continue; - } - result.appendBoolean(v); + return result.build(); } - return result.build(); } private Block eval(BooleanVector lhs, BooleanVector rhs) { int positionCount = lhs.getPositionCount(); - BooleanVector.Builder result = BooleanVector.newVectorBuilder(positionCount); - for (int p = 0; p < positionCount; p++) { - result.appendBoolean(bl.function().apply(lhs.getBoolean(p), rhs.getBoolean(p))); + try (var result = BooleanVector.newVectorFixedBuilder(positionCount, lhs.blockFactory())) { + for (int p = 0; p < positionCount; p++) { + result.appendBoolean(bl.function().apply(lhs.getBoolean(p), rhs.getBoolean(p))); + } + return result.build().asBlock(); } - return result.build().asBlock(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 8d060a739e8f8..bd02eb9cd4b8e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -87,8 +87,13 @@ protected List> batches() { } protected static List> rules() { - var substitutions = new Batch<>("Substitutions", Limiter.ONCE, new SubstituteSurrogates(), new ReplaceRegexMatch() - // new ReplaceTextFieldAttributesWithTheKeywordSubfield() + var substitutions = new Batch<>( + "Substitutions", + Limiter.ONCE, + new SubstituteSurrogates(), + new ReplaceRegexMatch(), + new ReplaceAliasingEvalWithProject() + // new ReplaceTextFieldAttributesWithTheKeywordSubfield() ); var operators = new Batch<>( @@ -851,4 +856,85 @@ protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { return new Equals(regexMatch.source(), regexMatch.field(), literal); } } + + /** + * Replace aliasing evals (eval x=a) with a projection which can be further combined / simplified. + * The rule gets applied only if there's another project (Project/Stats) above it. + * + * Needs to take into account shadowing of potentially intermediate fields: + * eval x = a + 1, y = x, z = y + 1, y = z, w = y + 1 + * The output should be + * eval x = a + 1, z = a + 1 + 1, w = a + 1 + 1 + * project x, z, z as y, w + */ + static class ReplaceAliasingEvalWithProject extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan logicalPlan) { + Holder enabled = new Holder<>(false); + + return logicalPlan.transformDown(p -> { + // found projection, turn enable flag on + if (p instanceof Aggregate || p instanceof Project) { + enabled.set(true); + } else if (enabled.get() && p instanceof Eval eval) { + p = rule(eval); + } + + return p; + }); + } + + private LogicalPlan rule(Eval eval) { + LogicalPlan plan = eval; + + // holds simple aliases such as b = a, c = b, d = c + AttributeMap basicAliases = new AttributeMap<>(); + // same as above but keeps the original expression + AttributeMap basicAliasSources = new AttributeMap<>(); + + List keptFields = new ArrayList<>(); + + var fields = eval.fields(); + for (int i = 0, size = fields.size(); i < size; i++) { + Alias field = fields.get(i); + Expression child = field.child(); + var attribute = field.toAttribute(); + // put the aliases in a separate map to separate the underlying resolve from other aliases + if (child instanceof Attribute) { + basicAliases.put(attribute, child); + basicAliasSources.put(attribute, field); + } else { + // be lazy and start replacing name aliases only if needed + if (basicAliases.size() > 0) { + // update the child through the field + field = (Alias) field.transformUp(e -> basicAliases.resolve(e, e)); + } + keptFields.add(field); + } + } + + // at least one alias encountered, move it into a project + if (basicAliases.size() > 0) { + // preserve the eval output (takes care of shadowing and order) but replace the basic aliases + List projections = new ArrayList<>(eval.output()); + // replace the removed aliases with their initial definition - however use the output to preserve the shadowing + for (int i = projections.size() - 1; i >= 0; i--) { + NamedExpression project = projections.get(i); + projections.set(i, basicAliasSources.getOrDefault(project, project)); + } + + LogicalPlan child = eval.child(); + if (keptFields.size() > 0) { + // replace the eval with just the kept fields + child = new Eval(eval.source(), eval.child(), keptFields); + } + // put the projection in place + plan = new Project(eval.source(), child, projections); + } + + return plan; + } + + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java index 8b91b9818a65a..871d3751b225d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java @@ -111,9 +111,10 @@ public Layout build() { for (NameId id : set.nameIds) { ChannelAndType next = new ChannelAndType(channel, set.type); ChannelAndType prev = layout.put(id, next); - if (prev != null) { - throw new IllegalArgumentException("Name [" + id + "] is on two channels [" + prev + "] and [" + next + "]"); - } + // Do allow multiple name to point to the same channel - see https://github.com/elastic/elasticsearch/pull/100238 + // if (prev != null) { + // throw new IllegalArgumentException("Name [" + id + "] is on two channels [" + prev + "] and [" + next + "]"); + // } } } return new DefaultLayout(Collections.unmodifiableMap(layout), numberOfChannels); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 3ea8ffe242919..e10469a4ff97d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -127,6 +127,10 @@ public void execute( configuration ); final List collectedPages = Collections.synchronizedList(new ArrayList<>()); + listener = listener.delegateResponse((l, e) -> { + collectedPages.forEach(p -> Releasables.closeExpectNoException(p::releaseBlocks)); + l.onFailure(e); + }); PhysicalPlan coordinatorPlan = new OutputExec(coordinatorAndDataNodePlan.v1(), collectedPages::add); PhysicalPlan dataNodePlan = coordinatorAndDataNodePlan.v2(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 3cab0604b0688..80fd51cacd163 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.Stat; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; -import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -208,27 +207,31 @@ public void testCountFieldWithFilter() { } /** - * Expects - for now + * Expects * LimitExec[500[INTEGER]] - * \_AggregateExec[[],[COUNT(hidden_s{r}#8) AS c],FINAL,null] + * \_AggregateExec[[],[COUNT(salary{f}#20) AS c],FINAL,null] * \_ExchangeExec[[count{r}#25, seen{r}#26],true] - * \_AggregateExec[[],[COUNT(hidden_s{r}#8) AS c],PARTIAL,8] - * \_EvalExec[[salary{f}#20 AS s, s{r}#3 AS hidden_s]] - * \_FieldExtractExec[salary{f}#20] - * \_EsQueryExec[test], query[{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"lt":10050,"boost":1.0}}}}}] - * [_doc{f}#42], limit[], sort[] estimatedRowSize[16] + * \_EsStatsQueryExec[test], stats[Stat[name=salary, type=COUNT, query={ + * "exists" : { + * "field" : "salary", + * "boost" : 1.0 + * } */ - // TODO: the eval is not yet optimized away public void testCountFieldWithEval() { var plan = plan(""" from test | eval s = salary | rename s as sr | eval hidden_s = sr | rename emp_no as e | where e < 10050 | stats c = count(hidden_s) """, IS_SV_STATS); + var limit = as(plan, LimitExec.class); var agg = as(limit.child(), AggregateExec.class); var exg = as(agg.child(), ExchangeExec.class); - agg = as(exg.child(), AggregateExec.class); - var eval = as(agg.child(), EvalExec.class); + var esStatsQuery = as(exg.child(), EsStatsQueryExec.class); + + assertThat(esStatsQuery.limit(), is(nullValue())); + assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen")); + var stat = as(esStatsQuery.stats().get(0), Stat.class); + assertThat(stat.query(), is(QueryBuilders.existsQuery("salary"))); } // optimized doesn't know yet how to push down count over field diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 78cc971ceb61b..848a7bc12aeef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Min; import org.elasticsearch.xpack.esql.expression.function.aggregate.Percentile; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; @@ -55,6 +56,7 @@ import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Nullability; import org.elasticsearch.xpack.ql.expression.ReferenceAttribute; +import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; @@ -1250,14 +1252,13 @@ public void testInvalidFoldDueToReplacement() { | keep x """); - var project = as(plan, EsqlProject.class); + var project = as(plan, Project.class); + assertThat(Expressions.names(project.projections()), contains("x")); + var child = aliased(project.projections().get(0), FieldAttribute.class); + assertThat(Expressions.name(child), is("emp_no")); var limit = as(project.child(), Limit.class); var filter = as(limit.child(), Filter.class); - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().get(0), Alias.class); - assertThat(Expressions.name(alias.child()), is("emp_no")); - var source = as(eval.child(), EsRelation.class); + var source = as(filter.child(), EsRelation.class); } public void testEnrich() { @@ -1509,6 +1510,12 @@ public void testPruneChainedEval() { var source = as(limit.child(), EsRelation.class); } + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[],[COUNT(salary{f}#1345) AS c]] + * \_EsRelation[test][_meta_field{f}#1346, emp_no{f}#1340, first_name{f}#..] + */ public void testPruneEvalDueToStats() { var plan = plan(""" from test @@ -1519,14 +1526,10 @@ public void testPruneEvalDueToStats() { var limit = as(plan, Limit.class); var aggregate = as(limit.child(), Aggregate.class); - assertThat(aggregate.aggregates(), hasSize(1)); - var alias = as(aggregate.aggregates().get(0), Alias.class); - var count = as(alias.child(), Count.class); - var eval = as(aggregate.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var field = as(eval.fields().get(0), Alias.class); - assertThat(field.name(), is("x")); - var source = as(eval.child(), EsRelation.class); + var aggs = aggregate.aggregates(); + assertThat(Expressions.names(aggs), contains("c")); + aggFieldName(aggs.get(0), Count.class, "salary"); + var source = as(aggregate.child(), EsRelation.class); } public void testPruneUnusedAggSimple() { @@ -1546,6 +1549,12 @@ public void testPruneUnusedAggSimple() { var source = as(agg.child(), EsRelation.class); } + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[],[COUNT(salary{f}#19) AS x]] + * \_EsRelation[test][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..] + */ public void testPruneUnusedAggMixedWithEval() { var plan = plan(""" from test @@ -1554,15 +1563,13 @@ public void testPruneUnusedAggMixedWithEval() { | keep x """); - var project = as(plan, Project.class); - var eval = as(project.child(), Eval.class); - var limit = as(eval.child(), Limit.class); + var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); assertThat(agg.groupings(), hasSize(0)); - assertThat(agg.aggregates(), hasSize(1)); - var aggOne = as(agg.aggregates().get(0), Alias.class); - assertThat(aggOne.name(), is("c")); - var count = as(aggOne.child(), Count.class); + var aggs = agg.aggregates(); + assertThat(aggs, hasSize(1)); + assertThat(Expressions.names(aggs), contains("x")); + aggFieldName(agg.aggregates().get(0), Count.class, "salary"); var source = as(agg.child(), EsRelation.class); } @@ -1586,6 +1593,14 @@ public void testPruneUnusedAggsChainedAgg() { var source = as(agg.child(), EsRelation.class); } + /** + * Expects + * Project[[c{r}#342]] + * \_Limit[500[INTEGER]] + * \_Filter[min{r}#348 > 10[INTEGER]] + * \_Aggregate[[],[COUNT(salary{f}#367) AS c, MIN(salary{f}#367) AS min]] + * \_EsRelation[test][_meta_field{f}#368, emp_no{f}#362, first_name{f}#36..] + */ public void testPruneMixedAggInsideUnusedEval() { var plan = plan(""" from test @@ -1600,15 +1615,22 @@ public void testPruneMixedAggInsideUnusedEval() { var project = as(plan, Project.class); var limit = as(project.child(), Limit.class); var filter = as(limit.child(), Filter.class); - var eval = as(filter.child(), Eval.class); - var agg = as(eval.child(), Aggregate.class); + var agg = as(filter.child(), Aggregate.class); assertThat(agg.groupings(), hasSize(0)); var aggs = agg.aggregates(); - assertThat(aggs, hasSize(2)); assertThat(Expressions.names(aggs), contains("c", "min")); + aggFieldName(aggs.get(0), Count.class, "salary"); + aggFieldName(aggs.get(1), Min.class, "salary"); var source = as(agg.child(), EsRelation.class); } + /** + * Expects + * Eval[[max{r}#6 + min{r}#9 + c{r}#3 AS x, min{r}#9 AS y, c{r}#3 AS z]] + * \_Limit[500[INTEGER]] + * \_Aggregate[[],[COUNT(salary{f}#26) AS c, MAX(salary{f}#26) AS max, MIN(salary{f}#26) AS min]] + * \_EsRelation[test][_meta_field{f}#27, emp_no{f}#21, first_name{f}#22, ..] + */ public void testNoPruningWhenDealingJustWithEvals() { var plan = plan(""" from test @@ -1623,6 +1645,13 @@ public void testNoPruningWhenDealingJustWithEvals() { var agg = as(limit.child(), Aggregate.class); } + /** + * Expects + * Project[[y{r}#6 AS z]] + * \_Eval[[emp_no{f}#11 + 1[INTEGER] AS y]] + * \_Limit[500[INTEGER]] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + */ public void testNoPruningWhenChainedEvals() { var plan = plan(""" from test @@ -1631,12 +1660,19 @@ public void testNoPruningWhenChainedEvals() { """); var project = as(plan, Project.class); + assertThat(Expressions.names(project.projections()), contains("z")); var eval = as(project.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x", "y", "z")); + assertThat(Expressions.names(eval.fields()), contains("y")); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), EsRelation.class); } + /** + * Expects + * Project[[salary{f}#20 AS x, emp_no{f}#15 AS y]] + * \_Limit[500[INTEGER]] + * \_EsRelation[test][_meta_field{f}#21, emp_no{f}#15, first_name{f}#16, ..] + */ public void testPruningDuplicateEvals() { var plan = plan(""" from test @@ -1647,12 +1683,257 @@ public void testPruningDuplicateEvals() { """); var project = as(plan, Project.class); - var eval = as(project.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x", "y")); - var limit = as(eval.child(), Limit.class); + var projections = project.projections(); + assertThat(Expressions.names(projections), contains("x", "y")); + var child = aliased(projections.get(0), FieldAttribute.class); + assertThat(child.name(), is("salary")); + child = aliased(projections.get(1), FieldAttribute.class); + assertThat(child.name(), is("emp_no")); + + var limit = as(project.child(), Limit.class); var source = as(limit.child(), EsRelation.class); } + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[],[COUNT(salary{f}#24) AS cx, COUNT(emp_no{f}#19) AS cy]] + * \_EsRelation[test][_meta_field{f}#25, emp_no{f}#19, first_name{f}#20, ..] + */ + public void testPruneEvalAliasOnAggUngrouped() { + var plan = plan(""" + from test + | eval x = emp_no, x = salary + | eval y = salary + | eval y = emp_no + | stats cx = count(x), cy = count(y) + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cx", "cy")); + aggFieldName(aggs.get(0), Count.class, "salary"); + aggFieldName(aggs.get(1), Count.class, "emp_no"); + var source = as(agg.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[x{r}#6],[COUNT(emp_no{f}#17) AS cy, salary{f}#22 AS x]] + * \_EsRelation[test][_meta_field{f}#23, emp_no{f}#17, first_name{f}#18, ..] + */ + public void testPruneEvalAliasOnAggGroupedByAlias() { + var plan = plan(""" + from test + | eval x = emp_no, x = salary + | eval y = salary + | eval y = emp_no + | stats cy = count(y) by x + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "x")); + aggFieldName(aggs.get(0), Count.class, "emp_no"); + var x = aliased(aggs.get(1), FieldAttribute.class); + assertThat(x.name(), is("salary")); + var source = as(agg.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#22],[COUNT(emp_no{f}#20) AS cy, MIN(salary{f}#25) AS cx, gender{f}#22]] + * \_EsRelation[test][_meta_field{f}#26, emp_no{f}#20, first_name{f}#21, ..] + */ + public void testPruneEvalAliasOnAggGrouped() { + var plan = plan(""" + from test + | eval x = emp_no, x = salary + | eval y = salary + | eval y = emp_no + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "emp_no"); + aggFieldName(aggs.get(1), Min.class, "salary"); + var by = as(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(by), is("gender")); + var source = as(agg.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#21],[COUNT(emp_no{f}#19) AS cy, MIN(salary{f}#24) AS cx, gender{f}#21]] + * \_EsRelation[test][_meta_field{f}#25, emp_no{f}#19, first_name{f}#20, ..] + */ + public void testPruneEvalAliasMixedWithRenameOnAggGrouped() { + var plan = plan(""" + from test + | eval x = emp_no, x = salary + | rename salary as x + | eval y = emp_no + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "emp_no"); + aggFieldName(aggs.get(1), Min.class, "salary"); + var by = as(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(by), is("gender")); + var source = as(agg.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#19],[COUNT(x{r}#3) AS cy, MIN(x{r}#3) AS cx, gender{f}#19]] + * \_Eval[[emp_no{f}#17 + 1[INTEGER] AS x]] + * \_EsRelation[test][_meta_field{f}#23, emp_no{f}#17, first_name{f}#18, ..] + */ + public void testEvalAliasingAcrossCommands() { + var plan = plan(""" + from test + | eval x = emp_no + 1 + | eval y = x + | eval z = y + 1 + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "x"); + aggFieldName(aggs.get(1), Min.class, "x"); + var by = as(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(by), is("gender")); + var eval = as(agg.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x")); + var source = as(eval.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#19],[COUNT(x{r}#3) AS cy, MIN(x{r}#3) AS cx, gender{f}#19]] + * \_Eval[[emp_no{f}#17 + 1[INTEGER] AS x]] + * \_EsRelation[test][_meta_field{f}#23, emp_no{f}#17, first_name{f}#18, ..] + */ + public void testEvalAliasingInsideSameCommand() { + var plan = plan(""" + from test + | eval x = emp_no + 1, y = x, z = y + 1 + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "x"); + aggFieldName(aggs.get(1), Min.class, "x"); + var by = as(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(by), is("gender")); + var eval = as(agg.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x")); + var source = as(eval.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#22],[COUNT(z{r}#9) AS cy, MIN(x{r}#3) AS cx, gender{f}#22]] + * \_Eval[[emp_no{f}#20 + 1[INTEGER] AS x, x{r}#3 + 1[INTEGER] AS z]] + * \_EsRelation[test][_meta_field{f}#26, emp_no{f}#20, first_name{f}#21, ..] + */ + public void testEvalAliasingInsideSameCommandWithShadowing() { + var plan = plan(""" + from test + | eval x = emp_no + 1, y = x, z = y + 1, y = z + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "z"); + aggFieldName(aggs.get(1), Min.class, "x"); + var by = as(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(by), is("gender")); + var eval = as(agg.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x", "z")); + var source = as(eval.child(), EsRelation.class); + } + + public void testPruneRenameOnAgg() { + var plan = plan(""" + from test + | rename emp_no as x + | rename salary as y + | stats cy = count(y), cx = min(x) by gender + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "gender")); + aggFieldName(aggs.get(0), Count.class, "salary"); + aggFieldName(aggs.get(1), Min.class, "emp_no"); + + var source = as(agg.child(), EsRelation.class); + } + + /** + * Expects + * Limit[500[INTEGER]] + * \_Aggregate[[gender{f}#14],[COUNT(salary{f}#17) AS cy, MIN(emp_no{f}#12) AS cx, gender{f}#14]] + * \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..] + */ + public void testPruneRenameOnAggBy() { + var plan = plan(""" + from test + | rename emp_no as x + | rename salary as y, gender as g + | stats cy = count(y), cx = min(x) by g + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var aggs = agg.aggregates(); + assertThat(Expressions.names(aggs), contains("cy", "cx", "g")); + aggFieldName(aggs.get(0), Count.class, "salary"); + aggFieldName(aggs.get(1), Min.class, "emp_no"); + var groupby = aliased(aggs.get(2), FieldAttribute.class); + assertThat(Expressions.name(groupby), is("gender")); + + var source = as(agg.child(), EsRelation.class); + } + + private T aliased(Expression exp, Class clazz) { + var alias = as(exp, Alias.class); + return as(alias.child(), clazz); + } + + private void aggFieldName(Expression exp, Class aggType, String fieldName) { + var alias = as(exp, Alias.class); + var af = as(alias.child(), aggType); + var field = af.field(); + assertThat(Expressions.name(field), is(fieldName)); + } + private LogicalPlan optimizedPlan(String query) { return plan(query); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 9f3bef6d064e6..2b25035e4097d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -103,7 +103,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -//@TestLogging(value = "org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer:TRACE", reason = "debug") +//@TestLogging(value = "org.elasticsearch.xpack.esql.optimizer:TRACE", reason = "debug") public class PhysicalPlanOptimizerTests extends ESTestCase { private static final String PARAM_FORMATTING = "%1$s"; @@ -244,6 +244,16 @@ public void testExactlyOneExtractorPerFieldWithPruning() { assertThat(query.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); } + /** + * Expects + * LimitExec[500[INTEGER]] + * \_AggregateExec[[],[SUM(salary{f}#882) AS x],FINAL,null] + * \_ExchangeExec[[sum{r}#887, seen{r}#888],true] + * \_FragmentExec[filter=null, estimatedRowSize=0, fragment=[ + * Aggregate[[],[SUM(salary{f}#882) AS x]] + * \_Filter[ROUND(emp_no{f}#877) > 10[INTEGER]] + * \_EsRelation[test][_meta_field{f}#883, emp_no{f}#877, first_name{f}#87..]]] + */ public void testDoubleExtractorPerFieldEvenWithAliasNoPruningDueToImplicitProjection() { var plan = physicalPlan(""" from test @@ -261,9 +271,7 @@ public void testDoubleExtractorPerFieldEvenWithAliasNoPruningDueToImplicitProjec aggregate = as(exchange.child(), AggregateExec.class); assertThat(aggregate.estimatedRowSize(), equalTo(Long.BYTES)); - var eval = as(aggregate.child(), EvalExec.class); - - var extract = as(eval.child(), FieldExtractExec.class); + var extract = as(aggregate.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("salary")); var filter = as(extract.child(), FilterExec.class); @@ -271,7 +279,7 @@ public void testDoubleExtractorPerFieldEvenWithAliasNoPruningDueToImplicitProjec assertThat(names(extract.attributesToExtract()), contains("emp_no")); var query = source(extract.child()); - assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES * 4 /* for doc id, emp_no, salary, and c */)); + assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES * 3 /* for doc id, emp_no and salary*/)); } public void testTripleExtractorPerField() { @@ -903,16 +911,12 @@ public void testQueryWithLimitSort() throws Exception { /** * Expected - * - * ProjectExec[[emp_no{f}#7, x{r}#4]] - * \_TopNExec[[Order[emp_no{f}#7,ASC,LAST]],5[INTEGER]] - * \_ExchangeExec[] - * \_ProjectExec[[emp_no{f}#7, x{r}#4]] - * \_TopNExec[[Order[emp_no{f}#7,ASC,LAST]],5[INTEGER]] - * \_FieldExtractExec[emp_no{f}#7] - * \_EvalExec[[first_name{f}#8 AS x]] - * \_FieldExtractExec[first_name{f}#8] - * \_EsQueryExec[test], query[][_doc{f}#14], limit[] + * ProjectExec[[emp_no{f}#7, first_name{f}#8 AS x]] + * \_TopNExec[[Order[emp_no{f}#7,ASC,LAST]],5[INTEGER],0] + * \_ExchangeExec[[],false] + * \_ProjectExec[[emp_no{f}#7, first_name{f}#8]] + * \_FieldExtractExec[emp_no{f}#7, first_name{f}#8] + * \_EsQueryExec[test], query[][_doc{f}#28], limit[5], sort[[FieldSort[field=emp_no{f}#7, direction=ASC, nulls=LAST]]]... */ public void testLocalProjectIncludeLocalAlias() throws Exception { var optimized = optimizedPlan(physicalPlan(""" @@ -928,11 +932,9 @@ public void testLocalProjectIncludeLocalAlias() throws Exception { var exchange = asRemoteExchange(topN.child()); project = as(exchange.child(), ProjectExec.class); - assertThat(names(project.projections()), contains("emp_no", "x")); - topN = as(project.child(), TopNExec.class); - var extract = as(topN.child(), FieldExtractExec.class); - var eval = as(extract.child(), EvalExec.class); - extract = as(eval.child(), FieldExtractExec.class); + assertThat(names(project.projections()), contains("emp_no", "first_name")); + var extract = as(project.child(), FieldExtractExec.class); + var source = as(extract.child(), EsQueryExec.class); } /** diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTaskTests.java index 03135990f0d51..b3146e81d08fc 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTaskTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -153,11 +154,10 @@ private void setupIndexPolicy(String policyName) { .put(IndexMetadata.builder(indexMetadata)) .build(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); clusterState = ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java index 27a8d1160d890..f47fc38206183 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleServiceTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.Lifecycle.State; @@ -97,11 +98,10 @@ public void prepareServices() { nodeId = randomAlphaOfLength(10); ExecutorService executorService = mock(ExecutorService.class); clusterService = mock(ClusterService.class); - masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); now = randomNonNegativeLong(); Clock clock = Clock.fixed(Instant.ofEpochMilli(now), ZoneId.of(randomFrom(ZoneId.getAvailableZoneIds()))); @@ -593,18 +593,16 @@ public void testIndicesOnShuttingDownNodesInDangerousStep() { .masterNodeId(nodeId) .add(masterNode) .add( - DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9301), - "regular_node" - ) + DiscoveryNodeUtils.builder("regular_node") + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9301)) + .build() ) .add( - DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9302), - "shutdown_node" - ) + DiscoveryNodeUtils.builder("shutdown_node") + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9302)) + .build() ) .build() ) diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java index ef250f0a42fd6..cd1a5d3744f02 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -219,11 +220,10 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception { logger.info("--> metadata: {}", Strings.toString(builder)); } String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) @@ -301,11 +301,10 @@ public void testUpdateChangedPolicy() { .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata) .build(); String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) @@ -385,11 +384,10 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except logger.info("--> metadata: {}", Strings.toString(builder)); } String nodeId = randomAlphaOfLength(10); - DiscoveryNode masterNode = DiscoveryNode.createLocal( - NodeRoles.masterNode(settings(IndexVersion.current()).build()), - new TransportAddress(TransportAddress.META_ADDRESS, 9300), - nodeId - ); + DiscoveryNode masterNode = DiscoveryNodeUtils.builder(nodeId) + .applySettings(NodeRoles.masterNode(settings(IndexVersion.current()).build())) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9300)) + .build(); ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) .metadata(metadata) .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build()) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 47b801fb209d3..113ed9a5aa686 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -823,6 +823,38 @@ public void testStartDatafeed_GivenTimeout_Returns408() throws Exception { assertThat(e.status(), equalTo(RestStatus.REQUEST_TIMEOUT)); } + public void testStartDatafeed_GivenNegativeStartTime_Returns408() throws Exception { + client().admin().indices().prepareCreate("data-1").setMapping("time", "type=date").get(); + long numDocs = 100; + long now = System.currentTimeMillis(); + long oneWeekAgo = now - 604800000; + long negativeStartTime = -1000; + indexDocs(logger, "data-1", numDocs, oneWeekAgo, now); + + String jobId = "job-for-start-datafeed-timeout"; + String datafeedId = jobId + "-datafeed"; + + Job.Builder job = createScheduledJob(jobId); + putJob(job); + openJob(job.getId()); + assertBusy(() -> assertEquals(getJobStats(job.getId()).get(0).getState(), JobState.OPENED)); + + DatafeedConfig.Builder datafeedConfigBuilder = createDatafeedBuilder( + job.getId() + "-datafeed", + job.getId(), + Collections.singletonList("data-1") + ); + DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); + putDatafeed(datafeedConfig); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> new StartDatafeedAction.Request(datafeedId, negativeStartTime) + ); + + assertThat(e.getMessage(), equalTo("[start] must not be negative [-1000].")); + } + public void testStart_GivenAggregateMetricDoubleWithoutAggs() throws Exception { final String index = "index-with-aggregate-metric-double"; String mapping = """ diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Alias.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Alias.java index 7f22cece89b00..df9f3c1d20eec 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Alias.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Alias.java @@ -56,8 +56,12 @@ protected NodeInfo info() { return NodeInfo.create(this, Alias::new, name(), qualifier, child, id(), synthetic()); } + public Alias replaceChild(Expression child) { + return new Alias(source(), name(), qualifier, child, id(), synthetic()); + } + @Override - public Expression replaceChildren(List newChildren) { + public Alias replaceChildren(List newChildren) { return new Alias(source(), name(), qualifier, newChildren.get(0), id(), synthetic()); } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index a1807026861da..16034354d0ff2 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -899,11 +899,7 @@ protected void doSaveState(IndexerState state, Map position, Run public List triggerAndWaitForCompletion(long now) throws Exception { assertTrue(maybeTriggerAsyncJob(now)); - try { - latch.await(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); if (exc != null) { throw exc; } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 965e37f34cbfe..f993debf86fae 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -66,7 +66,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -369,9 +368,9 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio indexRequestBuilders.add(client().prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); } try { - cyclicBarrier.await(); + safeAwait(cyclicBarrier); indexRandom(true, true, indexRequestBuilders); - } catch (InterruptedException | BrokenBarrierException e) { + } catch (InterruptedException e) { throw new AssertionError(e); } refresh(indexName); @@ -382,11 +381,7 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio }); final Thread snapshotThread = new Thread(() -> { - try { - cyclicBarrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - throw new AssertionError(e); - } + safeAwait(cyclicBarrier); createFullSnapshot(fsRepoName, snapshotName); }); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotsRelocationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotsRelocationIntegTests.java index f72a429a62f2a..a45d6b5c72a6f 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotsRelocationIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotsRelocationIntegTests.java @@ -53,12 +53,8 @@ public void testRelocationWaitsForPreWarm() throws Exception { final CountDownLatch latch = new CountDownLatch(1); for (int i = 0; i < preWarmThreads; i++) { executor.execute(() -> { - try { - barrier.await(); - latch.await(); - } catch (Exception e) { - throw new AssertionError(e); - } + safeAwait(barrier); + safeAwait(latch); }); } logger.info("--> waiting for prewarm threads to all become blocked"); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java index e29cbadf3e231..997fd8ddf134a 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java @@ -429,11 +429,7 @@ private int indexRandomDocsInCache(final int minDocs, final int maxDocs, final l ActionListener.running(latch::countDown) ); } - try { - latch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(latch); return nbDocs; } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java index 3d854d8ffa834..a7c03733e7b5f 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java @@ -345,20 +345,14 @@ private static class BlockingEvictionListener implements CacheFile.EvictionListe public void onEviction(CacheFile evictedCacheFile) { try { evictionLatch.countDown(); - releaseLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); + safeAwait(releaseLatch); } finally { evictedCacheFile.release(this); } } public void waitForBlock() { - try { - evictionLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(evictionLatch); } public void unblock() { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java index 1cd3014009168..8fbc20498d49a 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java @@ -388,16 +388,12 @@ public void force(boolean metaData) throws IOException { super.force(metaData); return; } - try { - blockingLatch.countDown(); - releasingLatch.await(); - if (failFSync.get()) { - throw new IOException("Simulated"); - } else { - super.force(metaData); - } - } catch (InterruptedException e) { - throw new AssertionError(e); + blockingLatch.countDown(); + safeAwait(releasingLatch); + if (failFSync.get()) { + throw new IOException("Simulated"); + } else { + super.force(metaData); } } }; @@ -409,11 +405,7 @@ public void blockFSyncForPath(Path path, boolean failure) { } public void waitForBlock() { - try { - blockingLatch.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + safeAwait(blockingLatch); } public void unblock() { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java index 7b02495c7227b..48e97b7afb897 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java @@ -105,6 +105,10 @@ public void clusterChanged(ClusterChangedEvent event) { clusterService.removeListener(this); metadataVersion.set(event.state().metadata().version()); savedClusterState.countDown(); + } else if (reservedState != null) { + logger.debug(() -> "Got reserved state update without error metadata: " + reservedState); + } else { + logger.debug(() -> "Got cluster state update: " + event.source()); } } }); @@ -112,7 +116,10 @@ public void clusterChanged(ClusterChangedEvent event) { return new Tuple<>(savedClusterState, metadataVersion); } - @TestLogging(value = "org.elasticsearch.common.file:DEBUG", reason = "https://github.com/elastic/elasticsearch/issues/98391") + @TestLogging( + value = "org.elasticsearch.common.file:DEBUG,org.elasticsearch.xpack.security:DEBUG,org.elasticsearch.cluster.metadata:DEBUG", + reason = "https://github.com/elastic/elasticsearch/issues/98391" + ) public void testFailsOnStartMasterNodeWithError() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3f23e62bad729..ce622ddf6fa69 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -149,6 +149,8 @@ import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCS_CLUSTER_PRIVILEGE_NAMES; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.WORKFLOWS_RESTRICTION_VERSION; import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; public class ApiKeyService { @@ -1309,12 +1311,12 @@ public void crossClusterApiKeyUsageStats(ActionListener> lis listener.onResponse(Map.of()); return; } - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { logger.debug("security index does not exist"); listener.onResponse(Map.of("total", 0, "ccs", 0, "ccr", 0, "ccs_ccr", 0)); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", "api_key")) @@ -1638,11 +1640,11 @@ private void findApiKeysForUserRealmApiKeyIdAndNameCombination( Function hitParser, ActionListener> listener ) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(Collections.emptyList()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("doc_type", "api_key")); QueryBuilder realmsQuery = filterForRealmNames(realmNames); @@ -1879,7 +1881,7 @@ long lastTimeWhenApiKeysRemoverWasTriggered() { } private void maybeStartApiKeyRemover() { - if (securityIndex.isAvailable()) { + if (securityIndex.isAvailable(PRIMARY_SHARDS)) { if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { inactiveApiKeysRemover.submit(client.threadPool()); lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); @@ -1935,12 +1937,12 @@ public void getApiKeys( public void queryApiKeys(SearchRequest searchRequest, boolean withLimitedBy, ActionListener listener) { ensureEnabled(); - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { logger.debug("security index does not exist"); listener.onResponse(QueryApiKeyResponse.emptyResponse()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute( listener::onFailure, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index 060f53ee4d8ad..bec9e90c9f190 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -34,6 +34,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; /** * Responsible for cleaning the invalidated and expired tokens from the security indices (`main` and `tokens`). @@ -68,10 +69,10 @@ final class ExpiredTokenRemover extends AbstractRunnable { @Override public void doRun() { final List indicesWithTokens = new ArrayList<>(); - if (securityTokensIndex.isAvailable()) { + if (securityTokensIndex.isAvailable(PRIMARY_SHARDS)) { indicesWithTokens.add(securityTokensIndex.aliasName()); } - if (securityMainIndex.isAvailable() && checkMainIndexForExpiredTokens) { + if (securityMainIndex.isAvailable(PRIMARY_SHARDS) && checkMainIndexForExpiredTokens) { indicesWithTokens.add(securityMainIndex.aliasName()); } if (indicesWithTokens.isEmpty()) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index e543248b8ad1d..794bcd96a66c3 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -147,6 +147,8 @@ import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; /** * Service responsible for the creation, validation, and other management of {@link UserToken} @@ -552,10 +554,10 @@ private void getTokenDocById( ActionListener listener ) { final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion); - final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze(); - if (frozenTokensIndex.isAvailable() == false) { + final SecurityIndexManager frozenTokensIndex = tokensIndex.defensiveCopy(); + if (frozenTokensIndex.isAvailable(PRIMARY_SHARDS) == false) { logger.warn("failed to get access token [{}] because index [{}] is not available", tokenId, tokensIndex.aliasName()); - listener.onFailure(frozenTokensIndex.getUnavailableReason()); + listener.onFailure(frozenTokensIndex.getUnavailableReason(PRIMARY_SHARDS)); return; } final GetRequest getRequest = client.prepareGet(tokensIndex.aliasName(), getTokenDocumentId(tokenId)).request(); @@ -1168,13 +1170,13 @@ private void findTokenFromRefreshToken( onFailure.accept(ex); } }; - final SecurityIndexManager frozenTokensIndex = tokensIndexManager.freeze(); + final SecurityIndexManager frozenTokensIndex = tokensIndexManager.defensiveCopy(); if (frozenTokensIndex.indexExists() == false) { logger.warn("index [{}] does not exist so we can't find token from refresh token", frozenTokensIndex.aliasName()); - listener.onFailure(frozenTokensIndex.getUnavailableReason()); - } else if (frozenTokensIndex.isAvailable() == false) { + listener.onFailure(new IndexNotFoundException(frozenTokensIndex.aliasName())); + } else if (frozenTokensIndex.isAvailable(SEARCH_SHARDS) == false) { logger.debug("index [{}] is not available to find token from refresh token, retrying", frozenTokensIndex.aliasName()); - maybeRetryOnFailure.accept(frozenTokensIndex.getUnavailableReason()); + maybeRetryOnFailure.accept(frozenTokensIndex.getUnavailableReason(SEARCH_SHARDS)); } else { final SearchRequest request = client.prepareSearch(tokensIndexManager.aliasName()) .setQuery( @@ -1786,11 +1788,11 @@ private void searchActiveTokens( */ private void sourceIndicesWithTokensAndRun(ActionListener> listener) { final List indicesWithTokens = new ArrayList<>(2); - final SecurityIndexManager frozenTokensIndex = securityTokensIndex.freeze(); + final SecurityIndexManager frozenTokensIndex = securityTokensIndex.defensiveCopy(); if (frozenTokensIndex.indexExists()) { // an existing tokens index always contains tokens (if available and version allows) - if (false == frozenTokensIndex.isAvailable()) { - listener.onFailure(frozenTokensIndex.getUnavailableReason()); + if (false == frozenTokensIndex.isAvailable(SEARCH_SHARDS)) { + listener.onFailure(frozenTokensIndex.getUnavailableReason(SEARCH_SHARDS)); return; } if (false == frozenTokensIndex.isIndexUpToDate()) { @@ -1806,14 +1808,14 @@ private void sourceIndicesWithTokensAndRun(ActionListener> listener } indicesWithTokens.add(frozenTokensIndex.aliasName()); } - final SecurityIndexManager frozenMainIndex = securityMainIndex.freeze(); + final SecurityIndexManager frozenMainIndex = securityMainIndex.defensiveCopy(); if (frozenMainIndex.indexExists()) { // main security index _might_ contain tokens if the tokens index has been created recently if (false == frozenTokensIndex.indexExists() || frozenTokensIndex.getCreationTime() .isAfter(clock.instant().minus(ExpiredTokenRemover.MAXIMUM_TOKEN_LIFETIME_HOURS, ChronoUnit.HOURS))) { - if (false == frozenMainIndex.isAvailable()) { - listener.onFailure(frozenMainIndex.getUnavailableReason()); + if (false == frozenMainIndex.isAvailable(SEARCH_SHARDS)) { + listener.onFailure(frozenMainIndex.getUnavailableReason(SEARCH_SHARDS)); return; } if (false == frozenMainIndex.isIndexUpToDate()) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 76029b779d8d9..36f78682b6bd1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -64,6 +64,8 @@ import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; /** @@ -118,11 +120,11 @@ public void getUsers(String[] userNames, final ActionListener> listener.onFailure(t); }; - final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = this.securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(Collections.emptyList()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else if (userNames.length == 1) { // optimization for single user lookup final String username = userNames[0]; getUserAndPassword( @@ -160,11 +162,11 @@ public void getUsers(String[] userNames, final ActionListener> } void getUserCount(final ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = this.securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(0L); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute( listener::onFailure, @@ -187,8 +189,8 @@ void getUserCount(final ActionListener listener) { * Async method to retrieve a user and their password */ private void getUserAndPassword(final String user, final ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); - if (frozenSecurityIndex.isAvailable() == false) { + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); + if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { if (frozenSecurityIndex.indexExists() == false) { logger.trace("could not retrieve user [{}] because security index does not exist", user); } else { @@ -537,11 +539,11 @@ private void setReservedUserEnabled( } public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(false); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { DeleteRequest request = client.prepareDelete(SECURITY_MAIN_ALIAS, getIdForUser(USER_DOC_TYPE, deleteUserRequest.username())) @@ -595,11 +597,11 @@ void verifyPassword(String username, final SecureString password, ActionListener } void getReservedUserInfo(String username, ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(null); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute( listener::onFailure, @@ -648,11 +650,11 @@ public void onFailure(Exception e) { } void getAllReservedUserInfo(ActionListener> listener) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(Collections.emptyMap()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute( listener::onFailure, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java index 07651e0272df4..a16eeb7cf4e0f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java @@ -65,6 +65,8 @@ import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; public class IndexServiceAccountTokenStore extends CachingServiceAccountTokenStore { @@ -168,11 +170,11 @@ void createToken( } void findTokensFor(ServiceAccountId accountId, ActionListener> listener) { - final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = this.securityIndex.defensiveCopy(); if (false == frozenSecurityIndex.indexExists()) { listener.onResponse(List.of()); - } else if (false == frozenSecurityIndex.isAvailable()) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (false == frozenSecurityIndex.isAvailable(SEARCH_SHARDS)) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { final Supplier contextSupplier = client.threadPool() @@ -204,11 +206,11 @@ void findTokensFor(ServiceAccountId accountId, ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = this.securityIndex.defensiveCopy(); if (false == frozenSecurityIndex.indexExists()) { listener.onResponse(false); - } else if (false == frozenSecurityIndex.isAvailable()) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (false == frozenSecurityIndex.isAvailable(PRIMARY_SHARDS)) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { final ServiceAccountId accountId = new ServiceAccountId(request.getNamespace(), request.getServiceName()); if (false == ServiceAccountService.isServiceAccountPrincipal(accountId.asPrincipal())) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index f49558ad6875d..18d9070d08a33 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -62,6 +62,8 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; @@ -245,11 +247,11 @@ public void onFailure(Exception e) { } private void innerDeleteMapping(DeleteRoleMappingRequest request, ActionListener listener) { - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(false); - } else if (securityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (securityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin( @@ -293,19 +295,18 @@ public void getRoleMappings(Set names, ActionListener> listener) { - if (securityIndex.isAvailable()) { - loadMappings(listener); - } else { - logger.info("The security index is not yet available - no role mappings can be loaded"); - if (logger.isDebugEnabled()) { - logger.debug( - "Security Index [{}] [exists: {}] [available: {}]", - SECURITY_MAIN_ALIAS, - securityIndex.indexExists(), - securityIndex.isAvailable() - ); - } + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); + if (frozenSecurityIndex.indexExists() == false) { + logger.debug("The security does not index exist - no role mappings can be loaded"); + listener.onResponse(Collections.emptyList()); + } else if (frozenSecurityIndex.indexIsClosed()) { + logger.debug("The security index exists but is closed - no role mappings can be loaded"); listener.onResponse(Collections.emptyList()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + logger.debug("The security index exists but is not available - no role mappings can be loaded"); + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); + } else { + loadMappings(listener); } } @@ -319,7 +320,7 @@ private void getMappings(ActionListener> listener) { * */ public void usageStats(ActionListener> listener) { - if (securityIndex.isAvailable() == false) { + if (securityIndex.isAvailable(SEARCH_SHARDS) == false) { reportStats(listener, Collections.emptyList()); } else { getMappings(ActionListener.wrap(mappings -> reportStats(listener, mappings), listener::onFailure)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index 10f5539b953b6..0e509c8af26b0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -74,6 +74,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor.DOC_TYPE_VALUE; import static org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor.Fields.APPLICATION; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; /** @@ -189,11 +191,11 @@ public void getPrivileges( private void innerGetPrivileges(Collection applications, ActionListener> listener) { assert applications != null && applications.size() > 0 : "Application names are required (found " + applications + ")"; - final SecurityIndexManager frozenSecurityIndex = securityIndexManager.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndexManager.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(Collections.emptyList()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(SEARCH_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(SEARCH_SHARDS)); } else { securityIndexManager.checkIndexVersionThenExecute(listener::onFailure, () -> { @@ -419,11 +421,11 @@ public void deletePrivileges( WriteRequest.RefreshPolicy refreshPolicy, ActionListener>> listener ) { - final SecurityIndexManager frozenSecurityIndex = securityIndexManager.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndexManager.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(Collections.emptyMap()); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { securityIndexManager.checkIndexVersionThenExecute(listener::onFailure, () -> { ActionListener groupListener = new GroupedActionListener<>(names.size(), ActionListener.wrap(responses -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index 085863fdb5e31..31875655a7ee9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -69,6 +69,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; /** @@ -133,12 +135,12 @@ public void getRoleDescriptors(Set names, final ActionListener { QueryBuilder query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE); @@ -208,11 +210,11 @@ public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionLi return; } - final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { listener.onResponse(false); - } else if (frozenSecurityIndex.isAvailable() == false) { - listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS)); } else { securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { DeleteRequest request = client.prepareDelete(SECURITY_MAIN_ALIAS, getIdForRole(deleteRoleRequest.name())).request(); @@ -309,7 +311,7 @@ public void onFailure(Exception e) { public void usageStats(ActionListener> listener) { Map usageStats = Maps.newMapWithExpectedSize(3); - if (securityIndex.isAvailable() == false) { + if (securityIndex.isAvailable(SEARCH_SHARDS) == false) { usageStats.put("size", 0L); usageStats.put("fls", false); usageStats.put("dls", false); @@ -406,12 +408,12 @@ public String toString() { } private void getRoleDescriptor(final String roleId, ActionListener resultListener) { - final SecurityIndexManager frozenSecurityIndex = this.securityIndex.freeze(); + final SecurityIndexManager frozenSecurityIndex = this.securityIndex.defensiveCopy(); if (frozenSecurityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! resultListener.onResponse(RoleRetrievalResult.success(Collections.emptySet())); - } else if (frozenSecurityIndex.isAvailable() == false) { - resultListener.onResponse(RoleRetrievalResult.failure(frozenSecurityIndex.getUnavailableReason())); + } else if (frozenSecurityIndex.isAvailable(PRIMARY_SHARDS) == false) { + resultListener.onResponse(RoleRetrievalResult.failure(frozenSecurityIndex.getUnavailableReason(PRIMARY_SHARDS))); } else { securityIndex.checkIndexVersionThenExecute( e -> resultListener.onResponse(RoleRetrievalResult.failure(e)), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java index 7be1d0f96c043..054583d94cbb1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java @@ -99,6 +99,8 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_PROFILE_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.security.authc.Authentication.isFileOrNativeRealm; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.PRIMARY_SHARDS; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_PROFILE_ALIAS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.VERSION_SECURITY_PROFILE_ORIGIN; @@ -261,7 +263,7 @@ public void suggestProfile(SuggestProfilesRequest request, TaskId parentTaskId, 0, new TotalHits(0, TotalHits.Relation.EQUAL_TO) ); - })).ifPresent(frozenProfileIndex -> { + }), SEARCH_SHARDS).ifPresent(frozenProfileIndex -> { final SearchRequest searchRequest = buildSearchRequestForSuggest(request, parentTaskId); frozenProfileIndex.checkIndexVersionThenExecute( @@ -334,7 +336,7 @@ public void usageStats(ActionListener> listener) { tryFreezeAndCheckIndex(listener.map(response -> { // index does not exist assert response == null : "only null response can reach here"; return Map.of("total", 0L, "enabled", 0L, "recent", 0L); - })).ifPresent(frozenProfileIndex -> { + }), SEARCH_SHARDS).ifPresent(frozenProfileIndex -> { final MultiSearchRequest multiSearchRequest = client.prepareMultiSearch() .add( client.prepareSearch(SECURITY_PROFILE_ALIAS) @@ -445,7 +447,7 @@ SearchRequest buildSearchRequestForSuggest(SuggestProfilesRequest request, TaskI } private void getVersionedDocument(String uid, ActionListener listener) { - tryFreezeAndCheckIndex(listener).ifPresent(frozenProfileIndex -> { + tryFreezeAndCheckIndex(listener, PRIMARY_SHARDS).ifPresent(frozenProfileIndex -> { final GetRequest getRequest = new GetRequest(SECURITY_PROFILE_ALIAS, uidToDocId(uid)); frozenProfileIndex.checkIndexVersionThenExecute( listener::onFailure, @@ -472,7 +474,7 @@ private void getVersionedDocuments(Collection uids, ActionListener { + tryFreezeAndCheckIndex(listener, PRIMARY_SHARDS).ifPresent(frozenProfileIndex -> { frozenProfileIndex.checkIndexVersionThenExecute( listener::onFailure, () -> new OriginSettingClient(client, getActionOrigin()).prepareMultiGet() @@ -544,7 +546,7 @@ private void searchVersionedDocumentsForSubjects( listener.onResponse(new SubjectSearchResultsAndErrors<>(List.of(), Map.of())); return; } - tryFreezeAndCheckIndex(listener).ifPresent(frozenProfileIndex -> { + tryFreezeAndCheckIndex(listener, SEARCH_SHARDS).ifPresent(frozenProfileIndex -> { frozenProfileIndex.checkIndexVersionThenExecute(listener::onFailure, () -> { final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); subjects.forEach(subject -> multiSearchRequest.add(buildSearchRequestForSubject(subject))); @@ -1006,14 +1008,17 @@ private static XContentBuilder wrapProfileDocumentWithoutApplicationData(Profile * Freeze the profile index check its availability and return it if everything is ok. * Otherwise it calls the listener with null and returns an empty Optional. */ - private Optional tryFreezeAndCheckIndex(ActionListener listener) { - final SecurityIndexManager frozenProfileIndex = profileIndex.freeze(); + private Optional tryFreezeAndCheckIndex( + ActionListener listener, + SecurityIndexManager.Availability availability + ) { + final SecurityIndexManager frozenProfileIndex = profileIndex.defensiveCopy(); if (false == frozenProfileIndex.indexExists()) { logger.debug("profile index does not exist"); listener.onResponse(null); return Optional.empty(); - } else if (false == frozenProfileIndex.isAvailable()) { - listener.onFailure(frozenProfileIndex.getUnavailableReason()); + } else if (false == frozenProfileIndex.isAvailable(availability)) { + listener.onFailure(frozenProfileIndex.getUnavailableReason(availability)); return Optional.empty(); } return Optional.of(frozenProfileIndex); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index bdc25098f1760..62bb20322f185 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.Tuple; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -67,31 +68,45 @@ public class SecurityIndexManager implements ClusterStateListener { private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class); + /** + * When checking availability, check for availability of search or availability of all primaries + **/ + public enum Availability { + SEARCH_SHARDS, + PRIMARY_SHARDS + } + private final Client client; private final SystemIndexDescriptor systemIndexDescriptor; private final List> stateChangeListeners = new CopyOnWriteArrayList<>(); private volatile State state; + private final boolean defensiveCopy; public static SecurityIndexManager buildSecurityIndexManager( Client client, ClusterService clusterService, SystemIndexDescriptor descriptor ) { - final SecurityIndexManager securityIndexManager = new SecurityIndexManager(client, descriptor, State.UNRECOVERED_STATE); + final SecurityIndexManager securityIndexManager = new SecurityIndexManager(client, descriptor, State.UNRECOVERED_STATE, false); clusterService.addListener(securityIndexManager); return securityIndexManager; } - private SecurityIndexManager(Client client, SystemIndexDescriptor descriptor, State state) { + private SecurityIndexManager(Client client, SystemIndexDescriptor descriptor, State state, boolean defensiveCopy) { this.client = client; this.state = state; this.systemIndexDescriptor = descriptor; + this.defensiveCopy = defensiveCopy; } - public SecurityIndexManager freeze() { - return new SecurityIndexManager(null, systemIndexDescriptor, state); + /** + * Creates a defensive to protect against the underlying state changes. Should be called prior to making decisions and that same copy + * should be reused for multiple checks in the same workflow. + */ + public SecurityIndexManager defensiveCopy() { + return new SecurityIndexManager(null, systemIndexDescriptor, state, true); } public String aliasName() { @@ -102,6 +117,10 @@ public boolean indexExists() { return this.state.indexExists(); } + public boolean indexIsClosed() { + return this.state.indexState == IndexMetadata.State.CLOSE; + } + public Instant getCreationTime() { return this.state.creationTime; } @@ -114,8 +133,27 @@ public boolean isIndexUpToDate() { return this.state.isIndexUpToDate; } - public boolean isAvailable() { - return this.state.indexAvailable; + /** + * Optimization to avoid making unnecessary calls when we know the underlying shard state. This call will check that the index exists, + * is discoverable from the alias, is not closed, and will determine if available based on the {@link Availability} parameter. + * @param availability Check availability for search or write/update/real time get workflows. Write/update/realtime get workflows + * should check for availability of primary shards. Search workflows should check availability of search shards + * (which may or may not also be the primary shards). + * @return + * when checking for search: true if all searchable shards for the security index are available + * when checking for primary: true if all primary shards for the security index are available + */ + public boolean isAvailable(Availability availability) { + switch (availability) { + case SEARCH_SHARDS -> { + return this.state.indexAvailableForSearch; + } + case PRIMARY_SHARDS -> { + return this.state.indexAvailableForWrite; + } + } + // can never happen + throw new IllegalStateException("Unexpected availability enumeration. This is bug, please contact support."); } public boolean isMappingUpToDate() { @@ -126,19 +164,34 @@ public boolean isStateRecovered() { return this.state != State.UNRECOVERED_STATE; } - public ElasticsearchException getUnavailableReason() { - final State state = this.state; // use a local copy so all checks execute against the same state! - if (state.indexAvailable) { - throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable"); + public ElasticsearchException getUnavailableReason(Availability availability) { + // ensure usage of a local copy so all checks execute against the same state! + if (defensiveCopy == false) { + throw new IllegalStateException("caller must make sure to use a defensive copy"); } - + final State state = this.state; if (state.indexState == IndexMetadata.State.CLOSE) { return new IndexClosedException(new Index(state.concreteIndexName, ClusterState.UNKNOWN_UUID)); } else if (state.indexExists()) { - return new UnavailableShardsException( - null, - "at least one primary shard for the index [" + state.concreteIndexName + "] is unavailable" - ); + assert state.indexAvailableForSearch == false || state.indexAvailableForWrite == false; + if (Availability.PRIMARY_SHARDS.equals(availability) && state.indexAvailableForWrite == false) { + return new UnavailableShardsException( + null, + "at least one primary shard for the index [" + state.concreteIndexName + "] is unavailable" + ); + } else if (Availability.SEARCH_SHARDS.equals(availability) && state.indexAvailableForSearch == false) { + // The current behavior is that when primaries are unavailable and replicas can not be promoted then + // any replicas will be marked as unavailable as well. This is applicable in stateless where there index only primaries + // with non-promotable replicas (i.e. search only shards). In the case "at least one search ... is unavailable" is + // a technically correct statement, but it may be unavailable because it is not promotable and the primary is unavailable + return new UnavailableShardsException( + null, + "at least one search shard for the index [" + state.concreteIndexName + "] is unavailable" + ); + } else { + // should never happen + throw new IllegalStateException("caller must ensure original availability matches the current availability"); + } } else { return new IndexNotFoundException(state.concreteIndexName); } @@ -174,7 +227,9 @@ public void clusterChanged(ClusterChangedEvent event) { final Instant creationTime = indexMetadata != null ? Instant.ofEpochMilli(indexMetadata.getCreationDate()) : null; final boolean isIndexUpToDate = indexMetadata == null || INDEX_FORMAT_SETTING.get(indexMetadata.getSettings()) == systemIndexDescriptor.getIndexFormat(); - final boolean indexAvailable = checkIndexAvailable(event.state()); + Tuple available = checkIndexAvailable(event.state()); + final boolean indexAvailableForWrite = available.v1(); + final boolean indexAvailableForSearch = available.v2(); final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final Version mappingVersion = oldestIndexMappingVersion(event.state()); final String concreteIndexName = indexMetadata == null @@ -199,7 +254,8 @@ public void clusterChanged(ClusterChangedEvent event) { final State newState = new State( creationTime, isIndexUpToDate, - indexAvailable, + indexAvailableForSearch, + indexAvailableForWrite, mappingIsUpToDate, mappingVersion, concreteIndexName, @@ -230,24 +286,35 @@ public void onStateRecovered(Consumer recoveredStateConsumer) { stateChangeListeners.add(stateChangeListener); } - private boolean checkIndexAvailable(ClusterState state) { + private Tuple checkIndexAvailable(ClusterState state) { final String aliasName = systemIndexDescriptor.getAliasName(); IndexMetadata metadata = resolveConcreteIndex(aliasName, state.metadata()); if (metadata == null) { logger.debug("Index [{}] is not available - no metadata", aliasName); - return false; + return new Tuple<>(false, false); } if (metadata.getState() == IndexMetadata.State.CLOSE) { logger.warn("Index [{}] is closed", aliasName); - return false; + return new Tuple<>(false, false); } + boolean allPrimaryShards = false; + boolean searchShards = false; final IndexRoutingTable routingTable = state.routingTable().index(metadata.getIndex()); - if (routingTable == null || routingTable.allPrimaryShardsActive() == false) { - logger.debug("Index [{}] is not yet active", aliasName); - return false; - } else { - return true; + if (routingTable != null && routingTable.allPrimaryShardsActive()) { + allPrimaryShards = true; + } + if (routingTable != null && routingTable.readyForSearch(state)) { + searchShards = true; + } + if (allPrimaryShards == false || searchShards == false) { + logger.debug( + "Index [{}] is not fully available. all primary shards available [{}], search shards available, [{}]", + aliasName, + allPrimaryShards, + searchShards + ); } + return new Tuple<>(allPrimaryShards, searchShards); } private boolean checkIndexMappingUpToDate(ClusterState clusterState) { @@ -482,10 +549,11 @@ public static boolean isIndexDeleted(State previousState, State currentState) { * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null, null); + public static final State UNRECOVERED_STATE = new State(null, false, false, false, false, null, null, null, null, null, null); public final Instant creationTime; public final boolean isIndexUpToDate; - public final boolean indexAvailable; + public final boolean indexAvailableForSearch; + public final boolean indexAvailableForWrite; public final boolean mappingUpToDate; public final Version mappingVersion; public final String concreteIndexName; @@ -497,7 +565,8 @@ public static class State { public State( Instant creationTime, boolean isIndexUpToDate, - boolean indexAvailable, + boolean indexAvailableForSearch, + boolean indexAvailableForWrite, boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, @@ -508,7 +577,8 @@ public State( ) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; - this.indexAvailable = indexAvailable; + this.indexAvailableForSearch = indexAvailableForSearch; + this.indexAvailableForWrite = indexAvailableForWrite; this.mappingUpToDate = mappingUpToDate; this.mappingVersion = mappingVersion; this.concreteIndexName = concreteIndexName; @@ -525,7 +595,8 @@ public boolean equals(Object o) { State state = (State) o; return Objects.equals(creationTime, state.creationTime) && isIndexUpToDate == state.isIndexUpToDate - && indexAvailable == state.indexAvailable + && indexAvailableForSearch == state.indexAvailableForSearch + && indexAvailableForWrite == state.indexAvailableForWrite && mappingUpToDate == state.mappingUpToDate && Objects.equals(mappingVersion, state.mappingVersion) && Objects.equals(concreteIndexName, state.concreteIndexName) @@ -543,7 +614,8 @@ public int hashCode() { return Objects.hash( creationTime, isIndexUpToDate, - indexAvailable, + indexAvailableForSearch, + indexAvailableForWrite, mappingUpToDate, mappingVersion, concreteIndexName, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java index bb19625c61101..2f45bafe493bb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommand.java @@ -11,11 +11,11 @@ import joptsimple.OptionSpec; import joptsimple.OptionSpecBuilder; -import org.elasticsearch.Version; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.ProcessInfo; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.cli.KeyStoreAwareCommand; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureString; @@ -249,12 +249,7 @@ private void checkClusterHealthWithRetries( terminal.errorPrintln("Failed to determine the health of the cluster. Cluster health is currently RED."); terminal.errorPrintln("This means that some cluster data is unavailable and your cluster is not fully functional."); terminal.errorPrintln( - "The cluster logs (https://www.elastic.co/guide/en/elasticsearch/reference/" - + Version.CURRENT.major - + "." - + Version.CURRENT.minor - + "/logging.html)" - + " might contain information/indications for the underlying cause" + "The cluster logs (" + ReferenceDocs.LOGGING + ")" + " might contain information/indications for the underlying cause" ); terminal.errorPrintln("It is recommended that you resolve the issues with your cluster before continuing"); terminal.errorPrintln("It is very likely that the command will fail when run against an unhealthy cluster."); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java index 61aa0e22fd905..2a6fad9c81f53 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java @@ -173,8 +173,9 @@ public void setup() throws Exception { ((Runnable) inv.getArguments()[1]).run(); return null; }).when(securityIndex).checkIndexVersionThenExecute(anyConsumer(), any(Runnable.class)); - when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index f65dda28be125..a748de0c89413 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -263,12 +263,13 @@ protected void ((Runnable) inv.getArguments()[1]).run(); return null; }).when(securityIndex).checkIndexVersionThenExecute(anyConsumer(), any(Runnable.class)); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); when(securityIndex.isIndexUpToDate()).thenReturn(true); when(securityIndex.getCreationTime()).thenReturn(Clock.systemUTC().instant()); when(securityIndex.aliasName()).thenReturn(".security"); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); final MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.isAllowed(Security.TOKEN_SERVICE_FEATURE)).thenReturn(true); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java index 9020d45041cea..e3631a785b9f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java @@ -204,8 +204,9 @@ public void setup() throws Exception { ((Runnable) inv.getArguments()[1]).run(); return null; }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class)); - when(securityIndex.isAvailable()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); final MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.isAllowed(Security.TOKEN_SERVICE_FEATURE)).thenReturn(true); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index a0f7892c3319d..6b9594c1c68ea 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -78,10 +78,12 @@ public void setup() { } public void testInvalidateTokensWhenIndexUnavailable() throws Exception { - when(securityIndex.isAvailable()).thenReturn(false); + + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(false); when(securityIndex.indexExists()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); - when(securityIndex.getUnavailableReason()).thenReturn(new ElasticsearchException("simulated")); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)) + .thenReturn(new ElasticsearchException("simulated")); final TokenService tokenService = new TokenService( SETTINGS, Clock.systemUTC(), @@ -122,10 +124,10 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { } public void testInvalidateTokensWhenIndexClosed() throws Exception { - when(securityIndex.isAvailable()).thenReturn(false); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); when(securityIndex.indexExists()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); - when(securityIndex.getUnavailableReason()).thenReturn( + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn( new IndexClosedException(new Index(INTERNAL_SECURITY_TOKENS_INDEX_7, ClusterState.UNKNOWN_UUID)) ); final TokenService tokenService = new TokenService( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java index 00f478f68b6ba..b6a1523b09784 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java @@ -110,7 +110,8 @@ public void terminateThreadPool() throws InterruptedException { public void testAnonymousUser() { NativeUsersStore usersStore = mock(NativeUsersStore.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); AnonymousUser anonymousUser = new AnonymousUser(settings); ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, threadPool); reservedRealm.initRealmRef( @@ -183,7 +184,8 @@ public void onFailure(Exception e) { public void testReservedUsersOnly() { NativeUsersStore usersStore = mock(NativeUsersStore.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = new ReservedRealm( @@ -272,7 +274,8 @@ public void testGetAllUsers() { ); NativeUsersStore usersStore = mock(NativeUsersStore.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = new ReservedRealm( mock(Environment.class), @@ -377,7 +380,8 @@ public void testGetUsersWithProfileUidException() { ); NativeUsersStore usersStore = mock(NativeUsersStore.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = new ReservedRealm( mock(Environment.class), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 117d1f1fe14bb..a0a1b622cf36e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -877,7 +877,7 @@ public void testCrossClusterApiKeyUsageStatsAreZerosWhenIndexDoesNotExist() { public void testCrossClusterApiKeyUsageFailsWhenIndexNotAvailable() { securityIndex = SecurityMocks.mockSecurityIndexManager(".security", true, false); final ElasticsearchException expectedException = new ElasticsearchException("not available"); - when(securityIndex.getUnavailableReason()).thenReturn(expectedException); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(expectedException); final ApiKeyService apiKeyService = createApiKeyService(); final PlainActionFuture> future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index cb4bbc383764c..cf343f790d85c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -1909,8 +1909,9 @@ public void testAuthenticateWithToken() throws Exception { String token = tokenFuture.get().getAccessToken(); when(client.prepareMultiGet()).thenReturn(new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE)); mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), expected, Map.of(), false, null, client); - when(securityIndex.freeze()).thenReturn(securityIndex); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { threadContext.putHeader("Authorization", "Bearer " + token); @@ -2014,8 +2015,9 @@ public void testInvalidToken() throws Exception { } public void testExpiredToken() throws Exception { - when(securityIndex.freeze()).thenReturn(securityIndex); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); User user = new User("_username", "r1"); final Authentication expected = AuthenticationTestHelper.builder() @@ -2501,6 +2503,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { true, true, true, + true, null, concreteSecurityIndexName, indexStatus, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 4c276993381b5..35335fd5e4a53 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -922,14 +922,14 @@ public void testIndexNotAvailable() throws Exception { final SecurityIndexManager tokensIndex; if (pre72OldNode != null) { tokensIndex = securityMainIndex; - when(securityTokensIndex.isAvailable()).thenReturn(false); + when(securityTokensIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); when(securityTokensIndex.indexExists()).thenReturn(false); - when(securityTokensIndex.freeze()).thenReturn(securityTokensIndex); + when(securityTokensIndex.defensiveCopy()).thenReturn(securityTokensIndex); } else { tokensIndex = securityTokensIndex; - when(securityMainIndex.isAvailable()).thenReturn(false); + when(securityMainIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); when(securityMainIndex.indexExists()).thenReturn(false); - when(securityMainIndex.freeze()).thenReturn(securityMainIndex); + when(securityMainIndex.defensiveCopy()).thenReturn(securityMainIndex); } try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) { PlainActionFuture future = new PlainActionFuture<>(); @@ -937,8 +937,10 @@ public void testIndexNotAvailable() throws Exception { tokenService.tryAuthenticateToken(bearerToken3, future); assertNull(future.get()); - when(tokensIndex.isAvailable()).thenReturn(false); - when(tokensIndex.getUnavailableReason()).thenReturn(new UnavailableShardsException(null, "unavailable")); + when(tokensIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); + when(tokensIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn( + new UnavailableShardsException(null, "unavailable") + ); when(tokensIndex.indexExists()).thenReturn(true); future = new PlainActionFuture<>(); final SecureString bearerToken2 = Authenticator.extractBearerTokenFromHeader(requestContext); @@ -951,7 +953,7 @@ public void testIndexNotAvailable() throws Exception { tokenService.tryAuthenticateToken(bearerToken1, future); assertNull(future.get()); - when(tokensIndex.isAvailable()).thenReturn(true); + when(tokensIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); when(tokensIndex.indexExists()).thenReturn(true); mockGetTokenFromAccessTokenBytes(tokenService, newTokenBytes.v1(), authentication, false, null); future = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 7dc5af1717fda..b9cc599609ea1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -37,6 +37,7 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { true, true, true, + true, null, concreteSecurityIndexName, indexStatus, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 5f195477d57a0..4e364518bb7f3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -317,10 +317,11 @@ private void respondToGetUserRequest(String username, SecureString password, Str @SuppressWarnings("unchecked") private NativeUsersStore startNativeUsersStore() { SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); when(securityIndex.isIndexUpToDate()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); doAnswer((i) -> { Runnable action = (Runnable) i.getArguments()[1]; action.run(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index afc6d5c17e135..e9af65bd8fc4a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -423,8 +423,11 @@ public void testRealmWithTemplatedRoleMapping() throws Exception { ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, sslService, threadPool); SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); - when(mockSecurityIndex.isAvailable()).thenReturn(true); + when(mockSecurityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(mockSecurityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(mockSecurityIndex.isIndexUpToDate()).thenReturn(true); + when(mockSecurityIndex.indexExists()).thenReturn(true); + when(mockSecurityIndex.defensiveCopy()).thenReturn(mockSecurityIndex); Client mockClient = mock(Client.class); when(mockClient.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index 34c9a7e3b0b0f..9bbf4dd312d27 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -490,8 +490,11 @@ public void testLdapRealmWithTemplatedRoleMapping() throws Exception { RealmConfig config = getRealmConfig(REALM_IDENTIFIER, settings); SecurityIndexManager mockSecurityIndex = mock(SecurityIndexManager.class); - when(mockSecurityIndex.isAvailable()).thenReturn(true); + when(mockSecurityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(mockSecurityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(mockSecurityIndex.isIndexUpToDate()).thenReturn(true); + when(mockSecurityIndex.defensiveCopy()).thenReturn(mockSecurityIndex); + when(mockSecurityIndex.indexExists()).thenReturn(true); Client mockClient = mock(Client.class); when(mockClient.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java index f536a696a8e23..2dec4eb8ea2b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java @@ -132,10 +132,11 @@ protected void cacheInvalidatorRegistry = mock(CacheInvalidatorRegistry.class); securityIndex = mock(SecurityIndexManager.class); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); when(securityIndex.isIndexUpToDate()).thenReturn(true); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); doAnswer((i) -> { Runnable action = (Runnable) i.getArguments()[1]; action.run(); @@ -375,7 +376,7 @@ public void testDeleteToken() { public void testIndexStateIssues() { // Index not exists Mockito.reset(securityIndex); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); when(securityIndex.indexExists()).thenReturn(false); final ServiceAccountId accountId = new ServiceAccountId(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)); @@ -394,11 +395,13 @@ public void testIndexStateIssues() { // Index exists but not available Mockito.reset(securityIndex); - when(securityIndex.freeze()).thenReturn(securityIndex); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); when(securityIndex.indexExists()).thenReturn(true); - when(securityIndex.isAvailable()).thenReturn(false); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(false); final ElasticsearchException e = new ElasticsearchException("fail"); - when(securityIndex.getUnavailableReason()).thenReturn(e); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(e); + when(securityIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(e); final PlainActionFuture> future3 = new PlainActionFuture<>(); store.findTokensFor(accountId, future3); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 1a0634f3234a6..16ef229ed5436 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -125,7 +125,10 @@ public void testResolveRoles() throws Exception { ScriptModule.CORE_CONTEXTS, () -> 1L ); - when(securityIndex.isAvailable()).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(true); + when(securityIndex.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(true); + when(securityIndex.indexExists()).thenReturn(true); + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, securityIndex, scriptService) { @Override @@ -190,6 +193,7 @@ private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthS isUpToDate, true, true, + true, null, concreteSecurityIndexName, healthStatus, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index f30fb242abc13..46a78f1055a6f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -1527,6 +1527,7 @@ public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, Clust isIndexUpToDate, true, true, + true, null, concreteSecurityIndexName, healthStatus, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index 10ed5c66f3c15..01d3ca6db354e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -126,9 +126,10 @@ public void searchScroll(SearchScrollRequest request, ActionListener { assertThat(invocationOnMock.getArguments().length, equalTo(2)); assertThat(invocationOnMock.getArguments()[1], instanceOf(Runnable.class)); @@ -976,6 +977,7 @@ private SecurityIndexManager.State dummyState( isIndexUpToDate, true, true, + true, null, concreteSecurityIndexName, healthStatus, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/profile/ProfileServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/profile/ProfileServiceTests.java index c9346ba488838..35efb12b278f2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/profile/ProfileServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/profile/ProfileServiceTests.java @@ -297,8 +297,8 @@ public void testGetProfileSubjectsNoIndex() throws Exception { assertThat(resultsAndErrors.errors().size(), is(0)); when(profileIndex.indexExists()).thenReturn(true); ElasticsearchException unavailableException = new ElasticsearchException("mock profile index unavailable"); - when(profileIndex.isAvailable()).thenReturn(false); - when(profileIndex.getUnavailableReason()).thenReturn(unavailableException); + when(profileIndex.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(false); + when(profileIndex.getUnavailableReason(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(unavailableException); PlainActionFuture>> future2 = new PlainActionFuture<>(); profileService.getProfileSubjects(randomList(1, 5, () -> randomAlphaOfLength(20)), future2); ExecutionException e = expectThrows(ExecutionException.class, () -> future2.get()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index d8fc00a2f1560..89d667de56c37 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -59,6 +59,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( true, true, true, + true, Version.CURRENT, ".security", ClusterHealthStatus.GREEN, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index 1e0969c96c0de..c8f86957f84a3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -123,7 +123,8 @@ public void testIndexWithUpToDateMappingAndTemplate() { manager.clusterChanged(event(markShardsAvailable(clusterStateBuilder))); assertThat(manager.indexExists(), Matchers.equalTo(true)); - assertThat(manager.isAvailable(), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(true)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); } @@ -164,6 +165,96 @@ public void testIndexWithoutPrimaryShards() { assertIndexUpToDateButNotAvailable(); } + public void testIndexAvailability() { + assertInitialState(); + final ClusterState cs = createClusterState( + TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7, + SecuritySystemIndices.SECURITY_MAIN_ALIAS + ).build(); + Index index = cs.metadata().index(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7).getIndex(); + ShardId shardId = new ShardId(index, 0); + ShardRouting primary = ShardRouting.newUnassigned( + shardId, + true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""), + ShardRouting.Role.INDEX_ONLY + ); + ShardRouting replica = ShardRouting.newUnassigned( + shardId, + false, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null), + ShardRouting.Role.SEARCH_ONLY + ); + String nodeId = ESTestCase.randomAlphaOfLength(8); + String nodeId2 = ESTestCase.randomAlphaOfLength(8); + + // primary/index unavailable, replica/search unavailable + IndexShardRoutingTable.Builder indxShardRoutingTableBuilder = IndexShardRoutingTable.builder(shardId) + .addShard( + primary.initialize(nodeId, null, primary.getExpectedShardSize()) + .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "")) + ) + .addShard( + replica.initialize(nodeId2, null, replica.getExpectedShardSize()) + .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "")) + ); + IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index).addIndexShard(indxShardRoutingTableBuilder); + RoutingTable routingTable = RoutingTable.builder().add(indexRoutingTableBuilder.build()).build(); + ClusterState.Builder clusterStateBuilder = ClusterState.builder(cs); + clusterStateBuilder.routingTable(routingTable); + ClusterState clusterState = clusterStateBuilder.build(); + manager.clusterChanged(event(clusterState)); + assertThat(manager.indexExists(), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(false)); + assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); + assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); + + // primary/index available, replica/search available + indxShardRoutingTableBuilder = IndexShardRoutingTable.builder(shardId) + .addShard( + primary.initialize(nodeId, null, primary.getExpectedShardSize()).moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE) + ) + .addShard( + replica.initialize(nodeId2, null, replica.getExpectedShardSize()) + .moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE) // start replica + ); + indexRoutingTableBuilder = IndexRoutingTable.builder(index).addIndexShard(indxShardRoutingTableBuilder); + routingTable = RoutingTable.builder().add(indexRoutingTableBuilder.build()).build(); + clusterStateBuilder = ClusterState.builder(cs); + clusterStateBuilder.routingTable(routingTable); + clusterState = clusterStateBuilder.build(); + manager.clusterChanged(event(clusterState)); + assertThat(manager.indexExists(), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(true)); + assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); + assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); + + // primary/index available, replica/search unavailable + indxShardRoutingTableBuilder = IndexShardRoutingTable.builder(shardId) + .addShard( + primary.initialize(nodeId, null, primary.getExpectedShardSize()).moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE) + ) + .addShard(replica.initialize(nodeId2, null, replica.getExpectedShardSize())); // initialized, but not started + indexRoutingTableBuilder = IndexRoutingTable.builder(index).addIndexShard(indxShardRoutingTableBuilder); + routingTable = RoutingTable.builder().add(indexRoutingTableBuilder.build()).build(); + clusterStateBuilder = ClusterState.builder(cs); + clusterStateBuilder.routingTable(routingTable); + clusterState = clusterStateBuilder.build(); + manager.clusterChanged(event(clusterState)); + assertThat(manager.indexExists(), Matchers.equalTo(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(true)); + assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); + assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); + + // primary/index unavailable, replica/search available + // it is not currently possibly to have unassigned primaries with assigned replicas + } + private ClusterChangedEvent event(ClusterState clusterState) { return new ClusterChangedEvent("test-event", clusterState, EMPTY_CLUSTER_STATE); } @@ -419,7 +510,8 @@ public void testProcessClosedIndexState() { ); manager.clusterChanged(event(markShardsAvailable(indexAvailable))); assertThat(manager.indexExists(), is(true)); - assertThat(manager.isAvailable(), is(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(true)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), is(true)); // Now close it ClusterState.Builder indexClosed = createClusterState( @@ -436,19 +528,22 @@ public void testProcessClosedIndexState() { manager.clusterChanged(event(indexClosed.build())); assertThat(manager.indexExists(), is(true)); - assertThat(manager.isAvailable(), is(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), is(false)); } private void assertInitialState() { assertThat(manager.indexExists(), Matchers.equalTo(false)); - assertThat(manager.isAvailable(), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(false)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(false)); assertThat(manager.isStateRecovered(), Matchers.equalTo(false)); } private void assertIndexUpToDateButNotAvailable() { assertThat(manager.indexExists(), Matchers.equalTo(true)); - assertThat(manager.isAvailable(), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), Matchers.equalTo(false)); + assertThat(manager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS), Matchers.equalTo(false)); assertThat(manager.isMappingUpToDate(), Matchers.equalTo(true)); assertThat(manager.isStateRecovered(), Matchers.equalTo(true)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java index 82b7b312465d3..a15d8409fe2b4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityMocks.java @@ -86,9 +86,10 @@ public static SecurityIndexManager mockSecurityIndexManager(String alias, boolea return null; }).when(securityIndexManager).checkIndexVersionThenExecute(anyConsumer(), any(Runnable.class)); when(securityIndexManager.indexExists()).thenReturn(exists); - when(securityIndexManager.isAvailable()).thenReturn(available); + when(securityIndexManager.isAvailable(SecurityIndexManager.Availability.PRIMARY_SHARDS)).thenReturn(available); + when(securityIndexManager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS)).thenReturn(available); when(securityIndexManager.aliasName()).thenReturn(alias); - when(securityIndexManager.freeze()).thenReturn(securityIndexManager); + when(securityIndexManager.defensiveCopy()).thenReturn(securityIndexManager); return securityIndexManager; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java index 076ba1d6294b2..736f07582ac49 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java @@ -874,7 +874,7 @@ public void testTcpHandshakeTimeout() throws IOException { // A read call will execute the ssl handshake int byteRead = acceptedSocket.getInputStream().read(); assertEquals('E', byteRead); - doneLatch.await(); + safeAwait(doneLatch); } catch (Exception e) { throw new AssertionError(e); } finally { diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java index 16ee53d8cb7c8..4d275d87c6877 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.ShutdownShardMigrationStatus; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingNode; @@ -562,31 +563,22 @@ public void testNodeNotInCluster() { .nodes( DiscoveryNodes.builder() .add( - DiscoveryNode.createLocal( - Settings.builder() - .put(Settings.builder().build()) - .put(Node.NODE_NAME_SETTING.getKey(), SHUTTING_DOWN_NODE_ID) - .build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9200), - SHUTTING_DOWN_NODE_ID - ) + DiscoveryNodeUtils.builder(SHUTTING_DOWN_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), SHUTTING_DOWN_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9200)) + .build() ) .add( - DiscoveryNode.createLocal( - Settings.builder().put(Settings.builder().build()).put(Node.NODE_NAME_SETTING.getKey(), LIVE_NODE_ID).build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9201), - LIVE_NODE_ID - ) + DiscoveryNodeUtils.builder(LIVE_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), LIVE_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9201)) + .build() ) .add( - DiscoveryNode.createLocal( - Settings.builder() - .put(Settings.builder().build()) - .put(Node.NODE_NAME_SETTING.getKey(), OTHER_LIVE_NODE_ID) - .build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9202), - OTHER_LIVE_NODE_ID - ) + DiscoveryNodeUtils.builder(OTHER_LIVE_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), OTHER_LIVE_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9202)) + .build() ) ) .routingTable(routingTable.build()) @@ -749,26 +741,23 @@ private ClusterState createTestClusterState( Map indicesTable = indices.stream().collect(toMap(imd -> imd.getIndex().getName(), Function.identity())); DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder() .add( - DiscoveryNode.createLocal( - Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), LIVE_NODE_ID).build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9201), - LIVE_NODE_ID - ) + DiscoveryNodeUtils.builder(LIVE_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), LIVE_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9201)) + .build() ) .add( - DiscoveryNode.createLocal( - Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), OTHER_LIVE_NODE_ID).build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9202), - OTHER_LIVE_NODE_ID - ) + DiscoveryNodeUtils.builder(OTHER_LIVE_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), OTHER_LIVE_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9202)) + .build() ); if (shuttingDownNodeAlreadyLeft == false) { discoveryNodesBuilder.add( - DiscoveryNode.createLocal( - Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), SHUTTING_DOWN_NODE_ID).build(), - new TransportAddress(TransportAddress.META_ADDRESS, 9200), - SHUTTING_DOWN_NODE_ID - ) + DiscoveryNodeUtils.builder(SHUTTING_DOWN_NODE_ID) + .applySettings(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), SHUTTING_DOWN_NODE_ID).build()) + .address(new TransportAddress(TransportAddress.META_ADDRESS, 9200)) + .build() ); } diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java index 162d43e4d708d..3541edfa20c93 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.OperationRouting; @@ -503,7 +502,7 @@ public ClusterState createState(SnapshotLifecycleMetadata snapMeta) { public ClusterState createState(SnapshotLifecycleMetadata snapMeta, boolean localNodeMaster) { Metadata metadata = Metadata.builder().putCustom(SnapshotLifecycleMetadata.TYPE, snapMeta).build(); final DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder() - .add(DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9300), "local")) + .add(DiscoveryNodeUtils.create("local", new TransportAddress(TransportAddress.META_ADDRESS, 9300))) .add(DiscoveryNodeUtils.create("remote", new TransportAddress(TransportAddress.META_ADDRESS, 9301))) .localNodeId("local") .masterNodeId(localNodeMaster ? "local" : "remote"); diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java index 7df3d38a3e4b9..32cdcee280d6e 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -43,7 +44,9 @@ public void setLogging() throws IOException { { "persistent": { "logger.org.elasticsearch.xpack.core.indexing.AsyncTwoPhaseIndexer": "debug", - "logger.org.elasticsearch.xpack.transform": "trace" + "logger.org.elasticsearch.xpack.transform": "trace", + "logger.org.elasticsearch.xpack.transform.notifications": "debug", + "logger.org.elasticsearch.xpack.transform.transforms": "debug" } }"""); client().performRequest(settingsRequest); @@ -100,6 +103,16 @@ public void testTransformFeatureReset() throws Exception { // assert transforms are gone assertThat((Integer) getTransforms("_all").get("count"), equalTo(0)); + // assert transform documents are gone + Map transformIndicesContents = ESRestTestCase.entityAsMap( + adminClient().performRequest(new Request("GET", ".transform-*/_search")) + ); + assertThat( + "Indices contents were: " + transformIndicesContents, + XContentMapValues.extractValue(transformIndicesContents, "hits", "total", "value"), + is(equalTo(0)) + ); + // assert transform indices are gone assertThat(ESRestTestCase.entityAsMap(adminClient().performRequest(new Request("GET", ".transform-*"))), is(anEmptyMap())); } diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/BaseRunAsSuperuserCommandTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommandTests.java similarity index 96% rename from x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/BaseRunAsSuperuserCommandTests.java rename to x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommandTests.java index 07741cce0da8e..33fadccda079a 100644 --- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/enrollment/tool/BaseRunAsSuperuserCommandTests.java +++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/tool/BaseRunAsSuperuserCommandTests.java @@ -5,14 +5,13 @@ * 2.0. */ -package org.elasticsearch.xpack.security.enrollment.tool; +package org.elasticsearch.xpack.security.tool; import joptsimple.OptionSet; import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; -import org.elasticsearch.Version; import org.elasticsearch.cli.Command; import org.elasticsearch.cli.CommandTestCase; import org.elasticsearch.cli.ExitCodes; @@ -20,6 +19,7 @@ import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -31,7 +31,6 @@ import org.elasticsearch.xpack.core.security.CommandLineHttpClient; import org.elasticsearch.xpack.core.security.HttpResponse; import org.elasticsearch.xpack.core.security.authc.support.Hasher; -import org.elasticsearch.xpack.security.tool.BaseRunAsSuperuserCommand; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -192,12 +191,7 @@ public void testUnhealthyCluster() throws Exception { stringContainsInOrder( "Failed to determine the health of the cluster. Cluster health is currently RED.", "This means that some cluster data is unavailable and your cluster is not fully functional.", - "The cluster logs (https://www.elastic.co/guide/en/elasticsearch/reference/" - + Version.CURRENT.major - + "." - + Version.CURRENT.minor - + "/logging.html)" - + " might contain information/indications for the underlying cause" + "The cluster logs (" + ReferenceDocs.LOGGING + ")" + " might contain information/indications for the underlying cause" ) ); assertNoUsers();