Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…add-kql-query

.buildkite/pipelines/periodic-packaging.template.yml

@@ -40,8 +40,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

.buildkite/pipelines/periodic-packaging.yml

@@ -345,8 +345,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp

.buildkite/pipelines/periodic-platform-support.yml

@@ -38,8 +38,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             GRADLE_TASK:
               - checkPart1

.buildkite/pipelines/pull-request/packaging-tests-windows-sample.yml

@@ -12,7 +12,7 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2019
+              - windows-2022
             PACKAGING_TASK:
               - default-windows-archive
         agents:

.buildkite/pipelines/pull-request/packaging-tests-windows.yml

@@ -10,8 +10,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
             PACKAGING_TASK:
               - default-windows-archive

docs/changelog/113713.yaml

@@ -0,0 +1,5 @@
+pr: 113713
+summary: Adding inference endpoint validation for `AzureAiStudioService`
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/114207.yaml

@@ -0,0 +1,15 @@
+pr: 114207
+summary: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting
+area: Allocation
+type: breaking
+issues: []
+breaking:
+  title: Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting
+  area: 'Cluster and node setting'
+  details: >-
+    Prior to 7.8, whenever a cluster had only a single data node, the watermarks would not be respected.
+    In order to change this in 7.8+ in a backwards compatible way, we introduced the 
+    `cluster.routing.allocation.disk.watermark.enable_for_single_data_node`
+    node setting. The setting was deprecated in 7.14 and was made to accept only true in 8.0
+  impact: No known end user impact
+  notable: false

docs/reference/connector/docs/connectors-ms-sql.asciidoc

@@ -45,7 +45,9 @@ include::_connectors-create-native.asciidoc[]
 To use this connector as a *managed connector*, use the *Connector* workflow.
 See <<es-native-connectors>>.
 
-Users require the `sysadmin` server role.
+Users require the `sysadmin` SQL Server role.
+Note that SQL Server Authentication is required.
+Windows Authentication is not supported.
 
 For additional operations, see <<es-connectors-usage>>.
 
@@ -75,10 +77,10 @@ Port::
 The port where the Microsoft SQL Server is hosted. Default value is `1433`.
 
 Username::
-The username of the account for Microsoft SQL Server.
+The username of the account for Microsoft SQL Server (SQL Server Authentication only).
 
 Password::
-The password of the account to be used for the Microsoft SQL Server.
+The password of the account to be used for the Microsoft SQL Server (SQL Server Authentication only).
 
 Database::
 Name of the Microsoft SQL Server database.
@@ -310,6 +312,8 @@ include::_connectors-create-client.asciidoc[]
 ===== Usage
 
 Users require the `sysadmin` server role.
+Note that SQL Server Authentication is required.
+Windows Authentication is not supported.
 
 To use this connector as a *self-managed connector*, see <<es-build-connector>>
 For additional usage operations, see <<es-connectors-usage>>.
@@ -350,10 +354,10 @@ Examples:
 The port where the Microsoft SQL Server is hosted. Default value is `9090`.
 
 `username`::
-The username of the account for Microsoft SQL Server.
+The username of the account for Microsoft SQL Server. (SQL Server Authentication only)
 
 `password`::
-The password of the account to be used for the Microsoft SQL Server.
+The password of the account to be used for the Microsoft SQL Server. (SQL Server Authentication only)
 
 `database`::
 Name of the Microsoft SQL Server database.

muted-tests.yml

@@ -79,9 +79,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/500_date_range/from, to, include_lower, include_upper deprecated}
   issue: https://github.com/elastic/elasticsearch/pull/113286
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {categorize.Categorize}
-  issue: https://github.com/elastic/elasticsearch/issues/113428
 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests
   method: testFieldMappings
   issue: https://github.com/elastic/elasticsearch/issues/113592
@@ -94,9 +91,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113722
-- class: org.elasticsearch.threadpool.SimpleThreadPoolIT
-  method: testThreadPoolMetrics
-  issue: https://github.com/elastic/elasticsearch/issues/108320
 - class: org.elasticsearch.kibana.KibanaThreadPoolIT
   method: testBlockedThreadPoolsRejectUserRequests
   issue: https://github.com/elastic/elasticsearch/issues/113939

qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -48,7 +48,6 @@
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
-@PackagingTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/116299")
 public class ArchiveTests extends PackagingTestCase {
 
     @BeforeClass

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java

@@ -335,7 +335,7 @@ public DesiredBalance compute(
                 nextReportTime = currentTime + timeWarningInterval;
             }
 
-            if (hasComputationConverged(hasChanges, i)) {
+            if (hasChanges == false && hasEnoughIterations(i)) {
                 logger.debug(
                     "Desired balance computation for [{}] converged after [{}] and [{}] iterations",
                     desiredBalanceInput.index(),
@@ -415,8 +415,8 @@ public DesiredBalance compute(
     }
 
     // visible for testing
-    boolean hasComputationConverged(boolean hasRoutingChanges, int currentIteration) {
-        return hasRoutingChanges == false;
+    boolean hasEnoughIterations(int currentIteration) {
+        return true;
     }
 
     private static Map<ShardId, ShardAssignment> collectShardAssignments(RoutingNodes routingNodes) {

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java

@@ -25,9 +25,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 
 import java.util.Map;
@@ -72,25 +70,6 @@ public class DiskThresholdDecider extends AllocationDecider {
 
     public static final String NAME = "disk_threshold";
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
-    public static final Setting<Boolean> ENABLE_FOR_SINGLE_DATA_NODE = Setting.boolSetting(
-        "cluster.routing.allocation.disk.watermark.enable_for_single_data_node",
-        true,
-        new Setting.Validator<>() {
-            @Override
-            public void validate(Boolean value) {
-                if (value == Boolean.FALSE) {
-                    throw new SettingsException(
-                        "setting [{}=false] is not allowed, only true is valid",
-                        ENABLE_FOR_SINGLE_DATA_NODE.getKey()
-                    );
-                }
-            }
-        },
-        Setting.Property.NodeScope,
-        Setting.Property.DeprecatedWarning
-    );
-
     public static final Setting<Boolean> SETTING_IGNORE_DISK_WATERMARKS = Setting.boolSetting(
         "index.routing.allocation.disk.watermark.ignore",
         false,
@@ -102,9 +81,6 @@ public void validate(Boolean value) {
 
     public DiskThresholdDecider(Settings settings, ClusterSettings clusterSettings) {
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
-        // get deprecation warnings.
-        boolean enabledForSingleDataNode = ENABLE_FOR_SINGLE_DATA_NODE.get(settings);
-        assert enabledForSingleDataNode;
     }
 
     /**

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -51,7 +51,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
@@ -272,7 +271,6 @@ public void apply(Settings value, Settings current, Settings previous) {
         ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING,
         ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING,
         ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING,
-        DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE,
         DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING,
         DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_MAX_HEADROOM_SETTING,
         DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING,

server/src/main/java/org/elasticsearch/inference/InferenceService.java

@@ -178,6 +178,15 @@ default Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) {
         return model;
     }
 
+    /**
+     * Update a chat completion model's max tokens if required. The default behaviour is to just return the model.
+     * @param model The original model without updated embedding details
+     * @return The model with updated chat completion details
+     */
+    default Model updateModelWithChatCompletionDetails(Model model) {
+        return model;
+    }
+
     /**
      * Defines the version required across all clusters to use this service
      * @return {@link TransportVersion} specifying the version

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java

@@ -408,8 +408,8 @@ public DesiredBalance compute(
                 }
 
                 @Override
-                boolean hasComputationConverged(boolean hasRoutingChanges, int currentIteration) {
-                    return super.hasComputationConverged(hasRoutingChanges, currentIteration) && currentIteration >= minIterations;
+                boolean hasEnoughIterations(int currentIteration) {
+                    return currentIteration >= minIterations;
                 }
             },
             reconcileAction,

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java

@@ -45,8 +45,6 @@
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
@@ -1070,9 +1068,6 @@ private void doTestWatermarksEnabledForSingleDataNode(boolean testMaxHeadroom) {
                     ByteSizeValue.ofGb(110).toString()
                 );
         }
-        if (randomBoolean()) {
-            builder = builder.put(DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.getKey(), true);
-        }
         Settings diskSettings = builder.build();
 
         final long totalBytes = testMaxHeadroom ? ByteSizeValue.ofGb(10000).getBytes() : 100;
@@ -1151,10 +1146,6 @@ private void doTestWatermarksEnabledForSingleDataNode(boolean testMaxHeadroom) {
                         + "on node, actual free: [20b], actual used: [80%]"
             )
         );
-
-        if (DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.exists(diskSettings)) {
-            assertSettingDeprecationsAndWarnings(new Setting<?>[] { DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE });
-        }
     }
 
     public void testWatermarksEnabledForSingleDataNodeWithPercentages() {
@@ -1165,25 +1156,6 @@ public void testWatermarksEnabledForSingleDataNodeWithMaxHeadroom() {
         doTestWatermarksEnabledForSingleDataNode(true);
     }
 
-    public void testSingleDataNodeDeprecationWarning() {
-        Settings settings = Settings.builder().put(DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.getKey(), false).build();
-
-        IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> new DiskThresholdDecider(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
-        );
-
-        assertThat(
-            e.getCause().getMessage(),
-            equalTo(
-                "setting [cluster.routing.allocation.disk.watermark.enable_for_single_data_node=false] is not allowed,"
-                    + " only true is valid"
-            )
-        );
-
-        assertSettingDeprecationsAndWarnings(new Setting<?>[] { DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE });
-    }
-
     private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom) {
         final long shardSizeInBytes = randomBoolean()
             ? (testMaxHeadroom ? ByteSizeValue.ofGb(99).getBytes() : 10L) // fits free space of node1

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/AutoscalingStorageIntegTestCase.java

@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.DiskUsageIntegTestCase;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
-import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -40,8 +39,7 @@ protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSetti
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), LOW_WATERMARK_BYTES + "b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), HIGH_WATERMARK_BYTES + "b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "0b")
-            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms")
-            .put(DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.getKey(), "true");
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms");
         return builder.build();
     }
 

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -42,7 +42,6 @@ private DeprecationChecks() {}
                 NodeDeprecationChecks::checkDataPathsList,
                 NodeDeprecationChecks::checkSharedDataPathSetting,
                 NodeDeprecationChecks::checkReservedPrefixedRealmNames,
-                NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting,
                 NodeDeprecationChecks::checkExporterUseIngestPipelineSettings,
                 NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting,
                 NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting,

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java

@@ -10,7 +10,6 @@
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
-import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -216,28 +215,6 @@ static DeprecationIssue checkReservedPrefixedRealmNames(
         }
     }
 
-    static DeprecationIssue checkSingleDataNodeWatermarkSetting(
-        final Settings settings,
-        final PluginsAndModules pluginsAndModules,
-        final ClusterState clusterState,
-        final XPackLicenseState licenseState
-    ) {
-        if (DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.exists(settings)) {
-            String key = DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.getKey();
-            return new DeprecationIssue(
-                DeprecationIssue.Level.CRITICAL,
-                String.format(Locale.ROOT, "setting [%s] is deprecated and will not be available in a future version", key),
-                "https://www.elastic.co/guide/en/elasticsearch/reference/7.14/"
-                    + "breaking-changes-7.14.html#deprecate-single-data-node-watermark",
-                String.format(Locale.ROOT, "found [%s] configured. Discontinue use of this setting.", key),
-                false,
-                null
-            );
-        }
-
-        return null;
-    }
-
     private static DeprecationIssue deprecatedAffixSetting(
         Setting.AffixSetting<?> deprecatedAffixSetting,
         String detailPattern,