Merge branch 'main' into readBytesReferece

iverase · Oct 14, 2023 · 0f9d76a · 0f9d76a
2 parents 3ed9aec + c00b626
commit 0f9d76a
Show file tree

Hide file tree

Showing 280 changed files with 5,822 additions and 1,952 deletions.
diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
@@ -2,7 +2,7 @@ steps:
   - group: platform-support-unix
     steps:
       - label: "{{matrix.image}} / platform-support-unix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
         timeout_in_minutes: 420
         matrix:
           setup:
@@ -46,7 +46,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: gcp
@@ -70,7 +69,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: aws

diff --git a/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap b/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap
@@ -3,183 +3,201 @@
 exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = `
 [
   {
-    "steps": [
-      {
-        "group": "bwc-snapshots",
-        "steps": [
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
-            },
-            "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
-            "env": {
-              "BWC_VERSION": "{{matrix.BWC_VERSION}}",
-            },
-            "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
-            "matrix": {
-              "setup": {
-                "BWC_VERSION": [
-                  "7.17.14",
-                  "8.10.3",
-                  "8.11.0",
-                ],
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
               },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
             },
-            "timeout_in_minutes": 300,
-          },
-        ],
-      },
-    ],
+          ],
+        },
+      ],
+    },
   },
   {
-    "env": {
-      "CUSTOM_ENV_VAR": "value",
-    },
-    "steps": [
-      {
-        "command": "echo 'hello world'",
-        "label": "test-step",
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
       },
-    ],
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
   },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = `
 [
   {
-    "steps": [
-      {
-        "agents": {
-          "buildDirectory": "/dev/shm/bk",
-          "image": "family/elasticsearch-ubuntu-2004",
-          "machineType": "custom-32-98304",
-          "provider": "gcp",
+    "name": "docs-check",
+    "pipeline": {
+      "steps": [
+        {
+          "agents": {
+            "buildDirectory": "/dev/shm/bk",
+            "image": "family/elasticsearch-ubuntu-2004",
+            "machineType": "custom-32-98304",
+            "provider": "gcp",
+          },
+          "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
+          "label": "docs-check",
+          "timeout_in_minutes": 300,
         },
-        "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check",
-        "label": "docs-check",
-        "timeout_in_minutes": 300,
-      },
-    ],
+      ],
+    },
   },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = `
 [
   {
-    "steps": [
-      {
-        "group": "bwc",
-        "steps": [
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
-            },
-            "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
-            "env": {
-              "BWC_VERSION": "7.0.0",
-            },
-            "key": "full-bwc:7_0_0",
-            "label": "7.0.0 / bwc",
-            "timeout_in_minutes": 300,
-          },
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
-            },
-            "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
-            "env": {
-              "BWC_VERSION": "7.0.1",
-            },
-            "key": "full-bwc:7_0_1",
-            "label": "7.0.1 / bwc",
-            "timeout_in_minutes": 300,
-          },
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
-            },
-            "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
-            "env": {
-              "BWC_VERSION": "7.1.0",
+    "name": "full-bwc",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest",
+              "env": {
+                "BWC_VERSION": "7.0.0",
+              },
+              "key": "full-bwc:7_0_0",
+              "label": "7.0.0 / bwc",
+              "timeout_in_minutes": 300,
             },
-            "key": "full-bwc:7_1_0",
-            "label": "7.1.0 / bwc",
-            "timeout_in_minutes": 300,
-          },
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest",
+              "env": {
+                "BWC_VERSION": "7.0.1",
+              },
+              "key": "full-bwc:7_0_1",
+              "label": "7.0.1 / bwc",
+              "timeout_in_minutes": 300,
             },
-            "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
-            "env": {
-              "BWC_VERSION": "8.10.0",
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest",
+              "env": {
+                "BWC_VERSION": "7.1.0",
+              },
+              "key": "full-bwc:7_1_0",
+              "label": "7.1.0 / bwc",
+              "timeout_in_minutes": 300,
             },
-            "key": "full-bwc:8_10_0",
-            "label": "8.10.0 / bwc",
-            "timeout_in_minutes": 300,
-          },
-          {
-            "agents": {
-              "buildDirectory": "/dev/shm/bk",
-              "image": "family/elasticsearch-ubuntu-2004",
-              "machineType": "custom-32-98304",
-              "provider": "gcp",
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest",
+              "env": {
+                "BWC_VERSION": "8.10.0",
+              },
+              "key": "full-bwc:8_10_0",
+              "label": "8.10.0 / bwc",
+              "timeout_in_minutes": 300,
             },
-            "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
-            "env": {
-              "BWC_VERSION": "8.11.0",
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest",
+              "env": {
+                "BWC_VERSION": "8.11.0",
+              },
+              "key": "full-bwc:8_11_0",
+              "label": "8.11.0 / bwc",
+              "timeout_in_minutes": 300,
             },
-            "key": "full-bwc:8_11_0",
-            "label": "8.11.0 / bwc",
-            "timeout_in_minutes": 300,
-          },
-        ],
-      },
-    ],
+          ],
+        },
+      ],
+    },
   },
   {
-    "env": {
-      "CUSTOM_ENV_VAR": "value",
-    },
-    "steps": [
-      {
-        "command": "echo 'hello world'",
-        "label": "test-step",
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
       },
-    ],
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
   },
 ]
 `;
 
 exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = `
 [
   {
-    "env": {
-      "CUSTOM_ENV_VAR": "value",
-    },
-    "steps": [
-      {
-        "command": "echo 'hello world'",
-        "label": "test-step",
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
       },
-    ],
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
   },
 ]
 `;
diff --git a/.buildkite/scripts/pull-request/pipeline.generate.ts b/.buildkite/scripts/pull-request/pipeline.generate.ts
@@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline";
 const pipelines = generatePipelines();
 
 for (const pipeline of pipelines) {
-  if (!process.env.CI) {
-    // Just for local debugging purposes
+  const yaml = stringify(pipeline.pipeline);
+
+  console.log(`--- Generated pipeline: ${pipeline.name}`);
+  console.log(yaml);
+
+  // Only do the pipeline upload if we're actually in CI
+  // This lets us run the tool locally and see the output
+  if (process.env.CI) {
     console.log("");
-    console.log(stringify(pipeline));
-  } else {
+    console.log("Uploading pipeline...");
+
     execSync(`buildkite-agent pipeline upload`, {
-      input: stringify(pipeline),
+      input: yaml,
       stdio: ["pipe", "inherit", "inherit"],
     });
   }