Merge branch 'main' into manage-agent-policies

elastic · Jul 2, 2024 · 0907247 · 0907247
2 parents c889881 + e73eb1d
commit 0907247
Show file tree

Hide file tree

Showing 126 changed files with 3,706 additions and 1,034 deletions.
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -3,7 +3,7 @@ steps:
     label: Build Kibana Artifacts
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: c2-standard-16
     timeout_in_minutes: 75
@@ -18,7 +18,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -34,7 +34,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -50,7 +50,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -66,7 +66,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -81,7 +81,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -96,7 +96,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 30
@@ -109,7 +109,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -127,7 +127,7 @@ steps:
       - exit_status: -1
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -154,7 +154,7 @@ steps:
     label: 'Publish Kibana Artifacts'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme

diff --git a/.buildkite/pipelines/artifacts_container_image.yml b/.buildkite/pipelines/artifacts_container_image.yml
@@ -3,7 +3,7 @@ steps:
     label: Build serverless container images
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-16
     timeout_in_minutes: 60
diff --git a/.buildkite/pipelines/artifacts_trigger.yml b/.buildkite/pipelines/artifacts_trigger.yml
@@ -3,7 +3,7 @@ steps:
     label: Trigger artifacts build
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 10
diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts
@@ -848,6 +848,7 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
       datastreamsDownsampling: `${ELASTICSEARCH_DOCS}downsampling.html`,
       installElasticAgent: `${FLEET_DOCS}install-fleet-managed-elastic-agent.html`,
       installElasticAgentStandalone: `${FLEET_DOCS}install-standalone-elastic-agent.html`,
+      grantESAccessToStandaloneAgents: `${FLEET_DOCS}grant-access-to-elasticsearch.html`,
       upgradeElasticAgent: `${FLEET_DOCS}upgrade-elastic-agent.html`,
       learnMoreBlog: `${ELASTIC_WEBSITE_URL}blog/elastic-agent-and-fleet-make-it-easier-to-integrate-your-systems-with-elastic`,
       apiKeysLearnMore: isServerless ? `${SERVERLESS_DOCS}api-keys` : `${KIBANA_DOCS}api-keys.html`,

diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts
@@ -537,6 +537,7 @@ export interface DocLinks {
     datastreamsDownsampling: string;
     installElasticAgent: string;
     installElasticAgentStandalone: string;
+    grantESAccessToStandaloneAgents: string;
     packageSignatures: string;
     upgradeElasticAgent: string;
     learnMoreBlog: string;

diff --git a/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts
@@ -17,15 +17,23 @@ describe('getESQLWithSafeLimit()', () => {
   });
 
   it('should add the limit', () => {
-    expect(getESQLWithSafeLimit(' from logs', LIMIT)).toBe('from logs \n| LIMIT 10000');
+    expect(getESQLWithSafeLimit('from logs', LIMIT)).toBe('from logs \n| LIMIT 10000');
     expect(getESQLWithSafeLimit('FROM logs* | LIMIT 5', LIMIT)).toBe(
-      'FROM logs* \n| LIMIT 10000| LIMIT 5'
+      'FROM logs* \n| LIMIT 10000 | LIMIT 5'
     );
     expect(getESQLWithSafeLimit('FROM logs* | SORT @timestamp | LIMIT 5', LIMIT)).toBe(
-      'FROM logs* |SORT @timestamp \n| LIMIT 10000| LIMIT 5'
+      'FROM logs* | SORT @timestamp \n| LIMIT 10000 | LIMIT 5'
     );
     expect(getESQLWithSafeLimit('from logs* | STATS MIN(a) BY b', LIMIT)).toBe(
-      'from logs* \n| LIMIT 10000| STATS MIN(a) BY b'
+      'from logs* \n| LIMIT 10000 | STATS MIN(a) BY b'
+    );
+
+    expect(getESQLWithSafeLimit('from logs* | STATS MIN(a) BY b | SORT b', LIMIT)).toBe(
+      'from logs* \n| LIMIT 10000 | STATS MIN(a) BY b | SORT b'
+    );
+
+    expect(getESQLWithSafeLimit('from logs* // | STATS MIN(a) BY b', LIMIT)).toBe(
+      'from logs* \n| LIMIT 10000 // | STATS MIN(a) BY b'
     );
   });
 });
diff --git a/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts
@@ -5,30 +5,33 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
 
 export function getESQLWithSafeLimit(esql: string, limit: number): string {
-  if (!esql.trim().toLowerCase().startsWith('from')) {
+  const { ast } = getAstAndSyntaxErrors(esql);
+  const sourceCommand = ast.find(({ name }) => ['from', 'metrics'].includes(name));
+  if (!sourceCommand) {
     return esql;
   }
-  const parts = esql.split('|');
 
-  if (!parts.length) {
-    return esql;
+  let sortCommandIndex = -1;
+  const sortCommand = ast.find(({ name }, index) => {
+    sortCommandIndex = index;
+    return name === 'sort';
+  });
+
+  if (!sortCommand || (sortCommand && sortCommandIndex !== 1)) {
+    const sourcePipeText = esql.substring(
+      sourceCommand.location.min,
+      sourceCommand.location.max + 1
+    );
+    return esql.replace(sourcePipeText, `${sourcePipeText} \n| LIMIT ${limit}`);
   }
 
-  const fromCommandIndex = 0;
-  const sortCommandIndex = 1;
-  const index =
-    parts.length > 1 && parts[1].trim().toLowerCase().startsWith('sort')
-      ? sortCommandIndex
-      : fromCommandIndex;
+  const sourceSortPipeText = esql.substring(
+    sourceCommand.location.min,
+    sortCommand.location.max + 1
+  );
 
-  return parts
-    .map((part, i) => {
-      if (i === index) {
-        return `${part.trim()} \n| LIMIT ${limit}`;
-      }
-      return part;
-    })
-    .join('|');
+  return esql.replace(sourceSortPipeText, `${sourceSortPipeText} \n| LIMIT ${limit}`);
 }
diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites>
-  <testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71">
+<testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
+  <testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts">
     <testcase name="maps app  maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378">
       <system-out>
         <![CDATA[[00:00:00]       │

diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/jest_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/jest_report.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9">
-  <testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts">
+<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9" command-line="node scripts/jest --config some/jest/config.ts">
+  <testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts">
     <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/>
     <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/>
     <testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060">

diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
-<testsuites>
-  <testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3">
+<testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js">
+  <testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js">
     <testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121">
       <system-out>
         <![CDATA[]]>

diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts
@@ -60,8 +60,8 @@ it('rewrites ftr reports with minimal changes', async () => {
     +++ ftr.xml
     @@ -1,53 +1,56 @@
      ‹?xml version="1.0" encoding="utf-8"?›
-     ‹testsuites›
-       ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
+     ‹testsuites name="ftr" timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
+       ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71" command-line="node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts"›
          ‹testcase name="maps app  maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
     -      ‹system-out›
     -        ‹![CDATA[[00:00:00]       │
@@ -155,7 +155,7 @@ it('rewrites jest reports with minimal changes', async () => {
     --- jest.xml
     +++ jest.xml
     @@ -3,13 +3,17 @@
-       ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"›
+       ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" command-line="node scripts/jest --config some/jest/config.ts"›
          ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/›
          ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/›
          ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"›
@@ -203,8 +203,8 @@ it('rewrites mocha reports with minimal changes', async () => {
     +++ mocha.xml
     @@ -1,13 +1,16 @@
      ‹?xml version="1.0" encoding="utf-8"?›
-     ‹testsuites›
-       ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
+     ‹testsuites command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
+       ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3" command-line="node scripts/functional_tests --config super-mocha-test.config.js"›
          ‹testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
     -      ‹system-out›
     -        ‹![CDATA[]]›

diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts
@@ -16,6 +16,7 @@ it('discovers failures in ftr report', async () => {
     Array [
       Object {
         "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
         "failure": "
             Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
     Wait timed out after 10055ms
@@ -37,6 +38,7 @@ it('discovers failures in ftr report', async () => {
       },
       Object {
         "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
         "failure": "
             { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
         at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
@@ -56,6 +58,7 @@ it('discovers failures in ftr report', async () => {
       },
       Object {
         "classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts",
+        "commandLine": "node scripts/functional_tests --config=x-pack/test/api_integration/apis/status/config.ts",
         "failure": "{ NoSuchSessionError: Tried to run command without establishing a connection
         at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
         at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
@@ -76,6 +79,7 @@ it('discovers failures in jest report', async () => {
     Array [
       Object {
         "classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
+        "commandLine": "node scripts/jest --config some/jest/config.ts",
         "failure": "
             TypeError: Cannot read property '0' of undefined
         at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
@@ -95,6 +99,7 @@ it('discovers failures in mocha report', async () => {
     Array [
       Object {
         "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
+        "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
         "failure": "
             Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
       <html>
@@ -117,6 +122,7 @@ it('discovers failures in mocha report', async () => {
       },
       Object {
         "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
+        "commandLine": "node scripts/functional_tests --config super-mocha-test.config.js",
         "failure": "
             TypeError: Cannot read property 'shutdown' of undefined
         at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)

diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts
@@ -16,6 +16,7 @@ export type TestFailure = FailedTestCase['$'] & {
   'system-out'?: string;
   githubIssue?: string;
   failureCount?: number;
+  commandLine?: string;
 };
 
 const getText = (node?: Array<string | { _: string }>) => {
@@ -71,19 +72,35 @@ const isLikelyIrrelevant = (name: string, failure: string) => {
 export function getFailures(report: TestReport) {
   const failures: TestFailure[] = [];
 
+  const commandLine = getCommandLineFromReport(report);
+
   for (const testCase of makeFailedTestCaseIter(report)) {
     const failure = getText(testCase.failure);
     const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure);
 
-    failures.push({
+    const failureObj = {
       // unwrap xml weirdness
       ...testCase.$,
       // Strip ANSI color characters
       failure,
       likelyIrrelevant,
       'system-out': getText(testCase['system-out']),
-    });
+      commandLine,
+    };
+
+    // cleaning up duplicates
+    delete failureObj['command-line'];
+
+    failures.push(failureObj);
   }
 
   return failures;
 }
+
+function getCommandLineFromReport(report: TestReport) {
+  if ('testsuites' in report) {
+    return report.testsuites?.testsuite?.[0]?.$['command-line'] || '';
+  } else {
+    return report.testsuite?.$['command-line'] || '';
+  }
+}
diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts
@@ -170,14 +170,23 @@ export async function reportFailuresToFile(
         <p><strong>${escape(failure.name)}</strong></p>
         <p>
           <small>
-            <strong>Failures in tracked branches</strong>: <span class="badge rounded-pill bg-danger">${
-              failure.failureCount || 0
-            }</span>
+            ${
+              failure.commandLine
+                ? `<div>
+                     <strong>Command Line</strong>:
+                     <pre>${escape(failure.commandLine)}</pre>
+                   </div>`
+                : ''
+            }
+            <div>
+                <strong>Failures in tracked branches</strong>:
+                    <span class="badge rounded-pill bg-danger">${failure.failureCount || 0}</span>
+            </div>
             ${
               failure.githubIssue
-                ? `<br /><a href="${escape(failure.githubIssue)}">${escape(
-                    failure.githubIssue
-                  )}</a>`
+                ? `<div>
+                     <a href="${escape(failure.githubIssue)}">${escape(failure.githubIssue)}</a>
+                   </div>`
                 : ''
             }
           </small>