Merge branch 'main' into fix_375

Signed-off-by: DumboJetEngine <[email protected]>

.github/actions/run-released-opensearch/action.yml

@@ -11,6 +11,9 @@ outputs:
   opensearch_url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.opensearch_url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.admin_password }}
 runs:
   using: composite
   steps:

.github/actions/start-opensearch/action.yml

@@ -8,6 +8,9 @@ outputs:
   opensearch_url:
     description: The URL where the OpenSearch node is accessible
     value: ${{ steps.opensearch.outputs.url }}
+  admin_password:
+    description: The initial admin password
+    value: ${{ steps.opensearch.outputs.password }}
 runs:
   using: composite
   steps:
@@ -24,30 +27,49 @@ runs:
         if [[ "$RUNNER_OS" == "macOS" ]]; then
           brew install -q coreutils
         fi
-        OPENSEARCH_HOME=$(realpath ./opensearch-*)
+        OPENSEARCH_HOME=$(realpath ./opensearch-[1-9]*)
+        CONFIG_DIR=$OPENSEARCH_HOME/config
+        CONFIG_FILE=$CONFIG_DIR/opensearch.yml
+        SECURITY_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
         OPENSEARCH_JAVA_OPTS="-Djava.net.preferIPv4Stack=true"
 
-        url="http://localhost:9200"
-        cp ./client/.ci/opensearch/opensearch.yml $OPENSEARCH_HOME/config/
+        URL="http://localhost:9200"
+        cp ./client/.ci/opensearch/opensearch.yml $CONFIG_FILE
 
         bash ./client/.ci/generate-certs.sh
+        
+        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin
 
-        if [[ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]]; then
+        if [[ -d "$SECURITY_DIR" ]]; then
           if [[ "$SECURED" == "true" ]]; then
-            bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s
-            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $OPENSEARCH_HOME/config/opensearch.yml
-            cp ./client/.ci/opensearch/*.pem $OPENSEARCH_HOME/config/
-            url="https://localhost:9200"
+            SECURITY_VERSION=$(cat $SECURITY_DIR/plugin-descriptor.properties | grep '^version=' | cut -d'=' -f 2)
+            SECURITY_VERSION_COMPONENTS=(${SECURITY_VERSION//./ })
+            SECURITY_MAJOR="${SECURITY_VERSION_COMPONENTS[0]}"
+            SECURITY_MINOR="${SECURITY_VERSION_COMPONENTS[1]}"
+        
+            if (( $SECURITY_MAJOR > 2 || ( $SECURITY_MAJOR == 2 && $SECURITY_MINOR >= 12 ) )); then
+              export OPENSEARCH_INITIAL_ADMIN_PASSWORD=$(LC_ALL=C tr -dc A-Za-z0-9 </dev/urandom | head -c 16)
+            fi
+        
+            bash "$SECURITY_DIR/tools/install_demo_configuration.sh" -y -i -s
+        
+            sed -i.bak -e 's/plugins.security.audit.type:.*/plugins.security.audit.type: log4j/' $CONFIG_FILE
+            cp ./client/.ci/opensearch/*.pem $CONFIG_DIR/
+            URL="https://localhost:9200"
           else
-            printf "\nplugins.security.disabled: true" >> $OPENSEARCH_HOME/config/opensearch.yml
+            printf "\nplugins.security.disabled: true" >> $CONFIG_FILE
           fi
         fi
 
         if [[ "$RUNNER_OS" == "macOS" ]]; then
-          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $OPENSEARCH_HOME/config/opensearch.yml
+          sed -i.bak -e 's/bootstrap.memory_lock:.*/bootstrap.memory_lock: false/' $CONFIG_FILE
         fi
 
-        echo "url=$url" >> $GITHUB_OUTPUT
+        {
+          echo "url=$URL"
+          echo "password=$OPENSEARCH_INITIAL_ADMIN_PASSWORD"
+        } | tee -a $GITHUB_OUTPUT
+        
         if [[ "$RUNNER_OS" == "Linux" ]]; then
           sudo swapoff -a
           sudo sysctl -w vm.swappiness=1
@@ -64,7 +86,7 @@ runs:
 
         for attempt in {1..20}; do
           sleep 5
-          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:admin $url; then
+          if curl -k -sS --cacert ./client/.ci/certs/root-ca.crt -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} $URL; then
             echo '=====> ready'
             exit 0
           fi

.github/workflows/integration-yaml-tests.yml

@@ -13,7 +13,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [ '1.2.4', '1.3.11', '2.2.0', '2.4.0', '2.6.0', '2.8.0', '2.9.0' ]
+        version:
+          - 2.11.1
+          - 2.10.0
+          - 2.8.0
+          - 2.6.0
+          - 2.4.1
+          - 2.2.1
+          - 2.0.1
+          - 1.3.14
+          - 1.2.4
+          - 1.1.0
     steps:
       - name: Checkout Client
         uses: actions/checkout@v3
@@ -99,13 +109,18 @@ jobs:
         uses: ./client/.github/actions/build-opensearch
         with:
           ref: ${{ matrix.opensearch_ref }}
+          security_plugin: true
 
       - name: Unpack OpenSearch
-        run: tar -xzf ${{ steps.opensearch_build.outputs.distribution }}
+        run: |
+          tar -xzf ${{ steps.opensearch_build.outputs.distribution }} \
+          && ./opensearch-*/bin/opensearch-plugin install --batch file://$(realpath ./opensearch-security/build/distributions/opensearch-security-*-SNAPSHOT.zip)
 
       - name: Start OpenSearch
         id: opensearch
         uses: ./client/.github/actions/start-opensearch
+        with:
+          secured: true
 
       - name: Run YAML tests
         working-directory: client
@@ -114,9 +129,12 @@ jobs:
             --project ./tests/Tests.YamlRunner/Tests.YamlRunner.fsproj \
             -- \
             --endpoint $OPENSEARCH_URL \
+            --auth-cert ./.ci/certs/kirk.p12 \
+            --auth-cert-pass kirk \
             --junit-output-file ./test-results.xml
         env:
           OPENSEARCH_URL: ${{ steps.opensearch.outputs.opensearch_url }}
+          ADMIN_PASS: ${{ steps.opensearch.outputs.admin_password }}
 
       - name: Save OpenSearch logs
         if: failure()

.github/workflows/integration.yml

@@ -17,17 +17,14 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.9.0
+          - 2.11.1
+          - 2.10.0
           - 2.8.0
-          - 2.7.0
           - 2.6.0
-          - 2.5.0
           - 2.4.1
-          - 2.3.0
           - 2.2.1
-          - 2.1.0
           - 2.0.1
-          - 1.3.11
+          - 1.3.14
           - 1.2.4
           - 1.1.0
 
@@ -50,7 +47,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
-      - run: "./build.sh integrate ${{ matrix.version }} readonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate ${{ matrix.version }} readonly,replicatedreadonly,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
 
@@ -62,7 +59,6 @@ jobs:
           path: client/build/output/*
 
   integration-opensearch-unreleased:
-    if: false # TODO: Temporarily disabled due to failures building & running OpenSearch from source, pending investigation & fixes (https://github.com/opensearch-project/opensearch-net/issues/268)
     name: Integration OpenSearch Unreleased
     runs-on: ubuntu-latest
     strategy:
@@ -101,7 +97,7 @@ jobs:
           knn_plugin: true
           plugins_output_directory: ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
 
-      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,replicatedreadonly,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
         env:

CHANGELOG.md

@@ -7,29 +7,35 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed naming of `ClusterManagerTimeout` and `MasterTimeout` properties from `*TimeSpanout` in the low-level client ([#332](https://github.com/opensearch-project/opensearch-net/pull/332))
 
 ### Added
-- Added `.Strict(...)`, `.Verbatim(...)`, `.Name(...)` methods on `QueryContainer` to help modify contained query attributes.
+- Added support for the `Cat.PitSegments` and `Cat.SegmentReplication` APIs ([#527](https://github.com/opensearch-project/opensearch-net/pull/527))
+- Added `.Strict(...)`, `.Verbatim(...)`, `.Name(...)` methods on `QueryContainer` to help modify contained query attributes ([#509](https://github.com/opensearch-project/opensearch-net/pull/509))
 
 ### Removed
 - Removed the `Features` API which is not supported by OpenSearch from the low-level client ([#331](https://github.com/opensearch-project/opensearch-net/pull/331))
 - Removed the deprecated low-level `IndexTemplateV2` APIs in favour of the `ComposableIndexTemplate` APIs ([#437](https://github.com/opensearch-project/opensearch-net/pull/437))
 
 ### Fixed
-- Fix `HttpConnection.ConvertHttpMethod` to support `Patch` method ([#489](https://github.com/opensearch-project/opensearch-net/pull/489))
+- Fixed `HttpConnection.ConvertHttpMethod` to support `Patch` method ([#489](https://github.com/opensearch-project/opensearch-net/pull/489))
 - Fixed `IEnumerable<int?>` property mapping. ([#503](https://github.com/opensearch-project/opensearch-net/pull/503))
 
 ### Dependencies
 - Bumps `Microsoft.CodeAnalysis.CSharp` from 4.2.0 to 4.6.0
-- Bumps `NSwag.Core.Yaml` from 13.19.0 to 13.20.0
+- Bumps `NSwag.Core.Yaml` from 13.19.0 to 14.0.2
 - Bumps `CSharpier.Core` from 0.25.0 to 0.26.7
 - Bumps `System.Diagnostics.DiagnosticSource` from 6.0.1 to 8.0.0
 - Bumps `Spectre.Console` from 0.47.0 to 0.48.0
 - Bumps `System.Text.Encodings.Web` from 7.0.0 to 8.0.0
 - Bumps `xunit.runner.visualstudio` from 2.5.4 to 2.5.6
-- Bumps `xunit` from 2.6.2 to 2.6.5
-- Bumps `Argu` from 6.1.1 to 6.1.4
+- Bumps `xunit` from 2.6.2 to 2.6.6
+- Bumps `Argu` from 6.1.1 to 6.1.5
 - Bumps `Microsoft.NET.Test.Sdk` from 17.7.2 to 17.8.0
 - Bumps `JetBrains.Annotations` from 2023.2.0 to 2023.3.0
 - Bumps `Bogus` from 34.0.2 to 35.3.0
+- Bumps `Octokit` from 9.0.0 to 9.1.0
+- Bumps `FSharp.Core` from 8.0.100 to 8.0.101
+- Bumps `Proc` from 0.6.2 to 0.8.1
+- Bumps `System.Text.Json` from 8.0.0 to 8.0.1
+- Bumps `Bullseye` from 4.2.1 to 5.0.0
 
 ## [1.6.0]
 ### Added
@@ -141,4 +147,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 [1.6.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.5.0...v1.6.0
 [1.5.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.4.0...v1.5.0
 [1.4.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.3.0...v1.4.0
-[1.3.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.2.0...v1.3.0
+[1.3.0]: https://github.com/opensearch-project/opensearch-net/compare/v1.2.0...v1.3.0

abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs

@@ -34,6 +34,7 @@
 using System.Security.Cryptography;
 using System.Text;
 using OpenSearch.OpenSearch.Managed;
+using OpenSearch.OpenSearch.Managed.Configuration;
 using OpenSearch.Stack.ArtifactsApi;
 
 namespace OpenSearch.OpenSearch.Ephemeral
@@ -59,6 +60,13 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo
 
 		protected EphemeralClusterComposer<TConfiguration> Composer { get; }
 
+		protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port)
+		{
+			base.ModifyNodeConfiguration(nodeConfiguration, port);
+
+			if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true");
+		}
+
 		public virtual ICollection<Uri> NodesUris(string hostName = null)
 		{
 			hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any()

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs

@@ -175,28 +175,35 @@ protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConso
 			ExecuteBinaryInternal(config, writer, binary, description, null, arguments);
 
 		protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, StartedHandler startedHandler, params string[] arguments) =>
-			ExecuteBinaryInternal(config, writer, binary, description, startedHandler, arguments);
+			string binary, string description, IDictionary<string, string> environmentVariables,
+			params string[] arguments) =>
+			ExecuteBinaryInternal(config, writer, binary, description, environmentVariables, arguments);
 
 		private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer,
-			string binary, string description, StartedHandler startedHandler, params string[] arguments)
+			string binary, string description, IDictionary<string, string> environmentVariables, params string[] arguments)
 		{
 			var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}";
 			writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}");
 
+			var environment = new Dictionary<string, string>
+			{
+				{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
+				{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}
+			};
+
+			if (environmentVariables != null)
+			{
+				foreach (var kvp in environmentVariables)
+					environment[kvp.Key] = kvp.Value;
+			}
+
 			var timeout = TimeSpan.FromSeconds(420);
 			var processStartArguments = new StartArguments(binary, arguments)
 			{
-				Environment = new Dictionary<string, string>
-				{
-					{config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath},
-					{"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome},
-				}
+				Environment = environment
 			};
 
-			var result = startedHandler != null
-				? Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter(), startedHandler)
-				: Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());
+			var result = Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter());
 
 			if (!result.Completed)
 				throw new Exception($"Timeout while executing {description} exceeded {timeout}");

abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs

@@ -26,10 +26,12 @@
 *  under the License.
 */
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using OpenSearch.OpenSearch.Managed.ConsoleWriters;
-using OpenSearch.Stack.ArtifactsApi;
+using OpenSearch.Stack.ArtifactsApi.Products;
+using SemanticVersioning;
 
 namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks
 {
@@ -38,30 +40,39 @@ public class InitialConfiguration : ClusterComposeTask
 		public override void Run(IEphemeralCluster<EphemeralClusterConfiguration> cluster)
 		{
 			var fs = cluster.FileSystem;
-			var configFile = Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml");
 
-			if (File.Exists(configFile) && File.ReadLines(configFile).Any(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#")))
-			{
-				cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} opensearch.yml already exists, skipping initial configuration");
+			var installConfigDir = Path.Combine(fs.OpenSearchHome, "config");
+			var installConfigFile = Path.Combine(installConfigDir, "opensearch.yml");
+			var pluginSecurity = Path.Combine(fs.OpenSearchHome, "plugins/opensearch-security");
+
+			if (!Directory.Exists(pluginSecurity))
 				return;
-			}
 
-			var securityInstallDemoConfigSubPath = "plugins/opensearch-security/tools/install_demo_configuration.sh";
-			var securityInstallDemoConfig = Path.Combine(fs.OpenSearchHome, securityInstallDemoConfigSubPath);
+			var isNewDemoScript = cluster.ClusterConfiguration.Version.BaseVersion() >= new Version(2, 12, 0);
+
+			const string securityInstallDemoConfigSubPath = "tools/install_demo_configuration.sh";
+			var securityInstallDemoConfig = Path.Combine(pluginSecurity, securityInstallDemoConfigSubPath);
 
 			cluster.Writer?.WriteDiagnostic($"{{{nameof(InitialConfiguration)}}} going to run [{securityInstallDemoConfigSubPath}]");
 
+			if (File.Exists(installConfigFile) && File.ReadLines(installConfigFile).Any(l => l.Contains("plugins.security"))) return;
+
+			var env = new Dictionary<string, string>();
+			var args = new List<string> { securityInstallDemoConfig, "-y", "-i" };
+
+			if (isNewDemoScript)
+			{
+				env.Add("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin");
+				args.Add("-t");
+			}
+
 			ExecuteBinary(
 				cluster.ClusterConfiguration,
 				cluster.Writer,
 				"/bin/bash",
 				"install security plugin demo configuration",
-				securityInstallDemoConfig,
-				"-y", "-i", "-s");
-
-			if (cluster.ClusterConfiguration.EnableSsl) return;
-
-			File.AppendAllText(configFile, "plugins.security.disabled: true");
+				env,
+				args.ToArray());
 		}
 	}
 }