Merge branch 'main' into 2024/11/06/interrupt-logging

docs/changelog/116348.yaml

@@ -0,0 +1,5 @@
+pr: 116348
+summary: "ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time"
+area: ES|QL
+type: enhancement
+issues: [ 114531 ]

docs/reference/settings/notification-settings.asciidoc

@@ -118,6 +118,17 @@ If you configure multiple email accounts, you must either configure this setting
 or specify the email account to use in the <<actions-email,`email`>> action. See
 <<configuring-email>>.
 
+`xpack.notification.email.recipient_allowlist`::
+(<<dynamic-cluster-setting,Dynamic>>)
+Specifies addresses to which emails are allowed to be sent.
+Emails with recipients (`To:`, `Cc:`, or `Bcc:`) outside of these patterns will be rejected and an
+error thrown. This setting defaults to `["*"]` which means all recipients are allowed.
+Simple globbing is supported, such as `list-*@company.com` in the list of allowed recipients.
+
+NOTE: This setting can't be used at the same time as `xpack.notification.email.account.domain_allowlist`
+and an error will be thrown if both are set at the same time. This setting can be used to specify domains
+to allow by using a wildcard pattern such as `*@company.com`.
+
 `xpack.notification.email.account`::
 Specifies account information for sending notifications via email. You
 can specify the following email account attributes:
@@ -129,6 +140,10 @@ Specifies domains to which emails are allowed to be sent. Emails with recipients
 `Bcc:`) outside of these domains will be rejected and an error thrown. This setting defaults to
 `["*"]` which means all domains are allowed. Simple globbing is supported, such as `*.company.com`
 in the list of allowed domains.
+
+NOTE: This setting can't be used at the same time as `xpack.notification.email.recipient_allowlist`
+and an error will be thrown if both are set at the same time.
+
 --
 [[email-account-attributes]]
 

modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java

@@ -112,7 +112,7 @@ public void testThrottleResponsesAreCountedInMetrics() throws IOException {
         blobContainer.blobExists(purpose, blobName);
 
         // Correct metrics are recorded
-        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics()
+        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES).expectMetrics()
             .withRequests(numThrottles + 1)
             .withThrottles(numThrottles)
             .withExceptions(numThrottles)
@@ -137,7 +137,7 @@ public void testRangeNotSatisfiedAreCountedInMetrics() throws IOException {
         assertThrows(RequestedRangeNotSatisfiedException.class, () -> blobContainer.readBlob(purpose, blobName));
 
         // Correct metrics are recorded
-        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB, repository).expectMetrics()
+        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB).expectMetrics()
             .withRequests(1)
             .withThrottles(0)
             .withExceptions(1)
@@ -170,7 +170,7 @@ public void testErrorResponsesAreCountedInMetrics() throws IOException {
         blobContainer.blobExists(purpose, blobName);
 
         // Correct metrics are recorded
-        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics()
+        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES).expectMetrics()
             .withRequests(numErrors + 1)
             .withThrottles(throttles.get())
             .withExceptions(numErrors)
@@ -191,7 +191,7 @@ public void testRequestFailuresAreCountedInMetrics() {
         assertThrows(IOException.class, () -> blobContainer.listBlobs(purpose));
 
         // Correct metrics are recorded
-        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.LIST_BLOBS, repository).expectMetrics()
+        metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.LIST_BLOBS).expectMetrics()
             .withRequests(4)
             .withThrottles(0)
             .withExceptions(4)
@@ -322,20 +322,14 @@ private void clearMetrics(String discoveryNode) {
             .forEach(TestTelemetryPlugin::resetMeter);
     }
 
-    private MetricsAsserter metricsAsserter(
-        String dataNodeName,
-        OperationPurpose operationPurpose,
-        AzureBlobStore.Operation operation,
-        String repository
-    ) {
-        return new MetricsAsserter(dataNodeName, operationPurpose, operation, repository);
+    private MetricsAsserter metricsAsserter(String dataNodeName, OperationPurpose operationPurpose, AzureBlobStore.Operation operation) {
+        return new MetricsAsserter(dataNodeName, operationPurpose, operation);
     }
 
     private class MetricsAsserter {
         private final String dataNodeName;
         private final OperationPurpose purpose;
         private final AzureBlobStore.Operation operation;
-        private final String repository;
 
         enum Result {
             Success,
@@ -361,11 +355,10 @@ List<Measurement> getMeasurements(TestTelemetryPlugin testTelemetryPlugin, Strin
             abstract List<Measurement> getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name);
         }
 
-        private MetricsAsserter(String dataNodeName, OperationPurpose purpose, AzureBlobStore.Operation operation, String repository) {
+        private MetricsAsserter(String dataNodeName, OperationPurpose purpose, AzureBlobStore.Operation operation) {
             this.dataNodeName = dataNodeName;
             this.purpose = purpose;
             this.operation = operation;
-            this.repository = repository;
         }
 
         private class Expectations {
@@ -458,7 +451,6 @@ private void assertMatchingMetricRecorded(MetricType metricType, String metricNa
                 .filter(
                     m -> m.attributes().get("operation").equals(operation.getKey())
                         && m.attributes().get("purpose").equals(purpose.getKey())
-                        && m.attributes().get("repo_name").equals(repository)
                         && m.attributes().get("repo_type").equals("azure")
                 )
                 .findFirst()
@@ -470,8 +462,6 @@ private void assertMatchingMetricRecorded(MetricType metricType, String metricNa
                             + operation.getKey()
                             + " and purpose="
                             + purpose.getKey()
-                            + " and repo_name="
-                            + repository
                             + " in "
                             + measurements
                     )

modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -402,10 +402,7 @@ public void testMetrics() throws Exception {
                 )
             );
             metrics.forEach(metric -> {
-                assertThat(
-                    metric.attributes(),
-                    allOf(hasEntry("repo_type", AzureRepository.TYPE), hasKey("repo_name"), hasKey("operation"), hasKey("purpose"))
-                );
+                assertThat(metric.attributes(), allOf(hasEntry("repo_type", AzureRepository.TYPE), hasKey("operation"), hasKey("purpose")));
                 final AzureBlobStore.Operation operation = AzureBlobStore.Operation.fromKey((String) metric.attributes().get("operation"));
                 final AzureBlobStore.StatsKey statsKey = new AzureBlobStore.StatsKey(
                     operation,

modules/repository-s3/build.gradle

@@ -12,6 +12,7 @@ import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
  */
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
@@ -48,6 +49,10 @@ dependencies {
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 
+  javaRestTestImplementation project(":test:framework")
+  javaRestTestImplementation project(':test:fixtures:s3-fixture')
+  javaRestTestImplementation project(':modules:repository-s3')
+
   yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
   internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 }

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -300,10 +300,7 @@ public void testMetrics() throws Exception {
                 )
             );
             metrics.forEach(metric -> {
-                assertThat(
-                    metric.attributes(),
-                    allOf(hasEntry("repo_type", S3Repository.TYPE), hasKey("repo_name"), hasKey("operation"), hasKey("purpose"))
-                );
+                assertThat(metric.attributes(), allOf(hasEntry("repo_type", S3Repository.TYPE), hasKey("operation"), hasKey("purpose")));
                 final S3BlobStore.Operation operation = S3BlobStore.Operation.parse((String) metric.attributes().get("operation"));
                 final S3BlobStore.StatsKey statsKey = new S3BlobStore.StatsKey(
                     operation,

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.s3.S3HttpFixture;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.MutableSettingsProvider;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RepositoryS3RestIT extends ESRestTestCase {
+
+    private static final String BUCKET = "RepositoryS3JavaRestTest-bucket";
+    private static final String BASE_PATH = "RepositoryS3JavaRestTest-base-path";
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+
+    private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .keystore(keystoreSettings)
+        .setting("s3.client.default.endpoint", s3Fixture::getAddress)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/116811")
+    public void testReloadCredentialsFromKeystore() throws IOException {
+        // Register repository (?verify=false because we don't have access to the blob store yet)
+        final var repositoryName = randomIdentifier();
+        registerRepository(
+            repositoryName,
+            S3Repository.TYPE,
+            false,
+            Settings.builder().put("bucket", BUCKET).put("base_path", BASE_PATH).build()
+        );
+        final var verifyRequest = new Request("POST", "/_snapshot/" + repositoryName + "/_verify");
+
+        // Set up initial credentials
+        final var accessKey1 = randomIdentifier();
+        s3Fixture.setAccessKey(accessKey1);
+        keystoreSettings.put("s3.client.default.access_key", accessKey1);
+        keystoreSettings.put("s3.client.default.secret_key", randomIdentifier());
+        cluster.updateStoredSecureSettings();
+        assertOK(client().performRequest(new Request("POST", "/_nodes/reload_secure_settings")));
+
+        // Check access using initial credentials
+        assertOK(client().performRequest(verifyRequest));
+
+        // Rotate credentials in blob store
+        final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomIdentifier);
+        s3Fixture.setAccessKey(accessKey2);
+
+        // Ensure that initial credentials now invalid
+        final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
+        assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
+        assertThat(
+            accessDeniedException2.getMessage(),
+            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+        );
+
+        // Set up refreshed credentials
+        keystoreSettings.put("s3.client.default.access_key", accessKey2);
+        cluster.updateStoredSecureSettings();
+        assertOK(client().performRequest(new Request("POST", "/_nodes/reload_secure_settings")));
+
+        // Check access using refreshed credentials
+        assertOK(client().performRequest(verifyRequest));
+    }
+
+}

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java

@@ -327,8 +327,6 @@ private Map<String, Object> metricAttributes(String action) {
         return Map.of(
             "repo_type",
             S3Repository.TYPE,
-            "repo_name",
-            blobStore.getRepositoryMetadata().name(),
             "operation",
             Operation.GET_OBJECT.getKey(),
             "purpose",

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java

@@ -1106,7 +1106,7 @@ private List<Measurement> getRetryHistogramMeasurements() {
     }
 
     private Map<String, Object> metricAttributes(String action) {
-        return Map.of("repo_type", "s3", "repo_name", "repository", "operation", "GetObject", "purpose", "Indices", "action", action);
+        return Map.of("repo_type", "s3", "operation", "GetObject", "purpose", "Indices", "action", action);
     }
 
     /**

modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java

@@ -210,7 +210,9 @@ public void testServerCloseConnectionMidStream() throws Exception {
             // terminate connection on server and wait resources are released
             handler.channel.request().getHttpChannel().close();
             assertBusy(() -> {
-                assertNull(handler.stream.buf());
+                // Cannot be simplified to assertNull.
+                // assertNull requires object to not fail on toString() method, but closing buffer can
+                assertTrue(handler.stream.buf() == null);
                 assertTrue(handler.streamClosed);
             });
         }