Merge branch 'master' into url_decode

.github/ISSUE_TEMPLATE/Bug_report_security_solution.md

@@ -1,38 +1,38 @@
----
-name: Bug report for Security Solution
-about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them!
-title: '[Security Solution]'
-labels: 'Team: SecuritySolution'
----
-
-**Describe the bug:**
-
-**Kibana/Elasticsearch Stack version:**
-
-**Server OS version:**
-
-**Browser and Browser OS versions:**
-
-**Elastic Endpoint version:**
-
-**Original install method (e.g. download page, yum, from source, etc.):**
-
-**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):**
-
-**Steps to reproduce:**
-
-1.
-2.
-3.
-
-**Current behavior:**
-
-**Expected behavior:**
-
-**Screenshots (if relevant):**
-
-**Errors in browser console (if relevant):**
-
-**Provide logs and/or server output (if relevant):**
-
-**Any additional context (logs, chat logs, magical formulas, etc.):**
+---
+name: Bug report for Security Solution
+about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them!
+title: '[Security Solution]'
+labels: 'bug, Team: SecuritySolution'
+---
+
+**Describe the bug:**
+
+**Kibana/Elasticsearch Stack version:**
+
+**Server OS version:**
+
+**Browser and Browser OS versions:**
+
+**Elastic Endpoint version:**
+
+**Original install method (e.g. download page, yum, from source, etc.):**
+
+**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):**
+
+**Steps to reproduce:**
+
+1.
+2.
+3.
+
+**Current behavior:**
+
+**Expected behavior:**
+
+**Screenshots (if relevant):**
+
+**Errors in browser console (if relevant):**
+
+**Provide logs and/or server output (if relevant):**
+
+**Any additional context (logs, chat logs, magical formulas, etc.):**

.github/ISSUE_TEMPLATE/v8_breaking_change.md

@@ -2,7 +2,7 @@
 name: 8.0 Breaking change
 about: Breaking changes from 7.x -> 8.0
 title: "[Breaking change]"
-labels: Team:Elasticsearch UI, Feature:Upgrade Assistant
+labels: Team:Elasticsearch UI, Feature:Upgrade Assistant, Breaking Change
 assignees: ''
 
 ---
@@ -11,15 +11,16 @@ assignees: ''
 
 **Which release will ship the breaking change?**
 
-<!-- e.g., v7.6.2 -->
+8.0
 
 **Describe the change. How will it manifest to users?**
 
-**What percentage of users will be affected?**
+**How many users will be affected?**
 
-<!-- e.g., Roughly 75% will need to make changes to x. -->
+<!-- e.g., Based on telemetry data, roughly 75% of our users will need to make changes to x -->
+<!-- e.g., A majority of users will need to make changes to x. -->
 
-**What can users to do to address the change manually?**
+**What can users do to address the change manually?**
 
 <!-- If applicable, describe the manual workaround -->
 

packages/kbn-apm-config-loader/src/config.test.ts

@@ -28,6 +28,8 @@ import {
 
 import { ApmConfiguration } from './config';
 
+const initialEnv = { ...process.env };
+
 describe('ApmConfiguration', () => {
   beforeEach(() => {
     packageMock.raw = {
@@ -39,6 +41,7 @@ describe('ApmConfiguration', () => {
   });
 
   afterEach(() => {
+    process.env = { ...initialEnv };
     resetAllMocks();
   });
 
@@ -90,13 +93,16 @@ describe('ApmConfiguration', () => {
     let config = new ApmConfiguration(mockedRootDir, {}, false);
     expect(config.getConfig('serviceName')).toEqual(
       expect.objectContaining({
-        serverUrl: expect.any(String),
-        secretToken: expect.any(String),
+        breakdownMetrics: true,
       })
     );
 
     config = new ApmConfiguration(mockedRootDir, {}, true);
-    expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl');
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        breakdownMetrics: false,
+      })
+    );
   });
 
   it('loads the configuration from the kibana config file', () => {
@@ -156,4 +162,32 @@ describe('ApmConfiguration', () => {
       })
     );
   });
+
+  it('correctly sets environment', () => {
+    delete process.env.ELASTIC_APM_ENVIRONMENT;
+    delete process.env.NODE_ENV;
+
+    let config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'development',
+      })
+    );
+
+    process.env.NODE_ENV = 'production';
+    config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'production',
+      })
+    );
+
+    process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
+    config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'ci',
+      })
+    );
+  });
 });

packages/kbn-apm-config-loader/src/config.ts

@@ -26,32 +26,26 @@ import { readFileSync } from 'fs';
 import { ApmAgentConfig } from './types';
 
 const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => {
-  if (isDistributable) {
-    return {
-      active: false,
-      globalLabels: {},
-      // Do not use a centralized controlled config
-      centralConfig: false,
-      // Capture all exceptions that are not caught
-      logUncaughtExceptions: true,
-      // Can be performance intensive, disabling by default
-      breakdownMetrics: false,
-    };
-  }
-
+  // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html
   return {
-    active: false,
-    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
+    active: process.env.ELASTIC_APM_ACTIVE || false,
+    environment: process.env.ELASTIC_APM_ENVIRONMENT || process.env.NODE_ENV || 'development',
+
+    serverUrl: 'https://b1e3b4b4233e44cdad468c127d0af8d8.apm.europe-west1.gcp.cloud.es.io:443',
+
     // The secretToken below is intended to be hardcoded in this file even though
     // it makes it public. This is not a security/privacy issue. Normally we'd
     // instead disable the need for a secretToken in the APM Server config where
     // the data is transmitted to, but due to how it's being hosted, it's easier,
     // for now, to simply leave it in.
-    secretToken: 'R0Gjg46pE9K9wGestd',
+    secretToken: '2OyjjaI6RVkzx2O5CV',
+
+    logUncaughtExceptions: true,
     globalLabels: {},
-    breakdownMetrics: true,
     centralConfig: false,
-    logUncaughtExceptions: true,
+
+    // Can be performance intensive, disabling by default
+    breakdownMetrics: isDistributable ? false : true,
   };
 };
 
@@ -84,7 +78,8 @@ export class ApmConfiguration {
         getDefaultConfig(this.isDistributable),
         this.getConfigFromKibanaConfig(),
         this.getDevConfig(),
-        this.getDistConfig()
+        this.getDistConfig(),
+        this.getCIConfig()
       );
 
       const rev = this.getGitRev();
@@ -146,6 +141,21 @@ export class ApmConfiguration {
     };
   }
 
+  private getCIConfig(): ApmAgentConfig {
+    if (process.env.ELASTIC_APM_ENVIRONMENT !== 'ci') {
+      return {};
+    }
+
+    return {
+      globalLabels: {
+        branch: process.env.ghprbSourceBranch || '',
+        targetBranch: process.env.ghprbTargetBranch || '',
+        ciJobName: process.env.JOB_NAME || '',
+        ciBuildNumber: process.env.BUILD_NUMBER || '',
+      },
+    };
+  }
+
   private getGitRev() {
     if (this.isDistributable) {
       return this.pkgBuild.sha;

packages/kbn-optimizer/src/cli.ts

@@ -146,7 +146,11 @@ run(
     await lastValueFrom(update$.pipe(logOptimizerState(log, config)));
 
     if (updateLimits) {
-      updateBundleLimits(log, config);
+      updateBundleLimits({
+        log,
+        config,
+        dropMissing: !(focus || filter),
+      });
     }
   },
   {

packages/kbn-optimizer/src/limits.ts

@@ -79,10 +79,18 @@ export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerCo
   log.success('limits.yml file valid');
 }
 
-export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) {
+interface UpdateBundleLimitsOptions {
+  log: ToolingLog;
+  config: OptimizerConfig;
+  dropMissing: boolean;
+}
+
+export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) {
   const metrics = getMetrics(log, config);
 
-  const pageLoadAssetSize: NonNullable<Limits['pageLoadAssetSize']> = {};
+  const pageLoadAssetSize: NonNullable<Limits['pageLoadAssetSize']> = dropMissing
+    ? {}
+    : config.limits.pageLoadAssetSize ?? {};
 
   for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) {
     if (metric.group === 'page load bundle size') {

src/core/server/elasticsearch/client/client_config.test.ts

@@ -216,28 +216,14 @@ describe('parseClientOptions', () => {
         );
       });
 
-      it('adds auth to the nodes if both `username` and `password` are set', () => {
-        let options = parseClientOptions(
+      it('does not add auth to the nodes', () => {
+        const options = parseClientOptions(
           createConfig({
             username: 'user',
-            hosts: ['http://node-A:9200'],
-          }),
-          false
-        );
-        expect(options.nodes).toMatchInlineSnapshot(`
-                  Array [
-                    Object {
-                      "url": "http://node-a:9200/",
-                    },
-                  ]
-              `);
-
-        options = parseClientOptions(
-          createConfig({
             password: 'pass',
             hosts: ['http://node-A:9200'],
           }),
-          false
+          true
         );
         expect(options.nodes).toMatchInlineSnapshot(`
                   Array [
@@ -246,22 +232,6 @@ describe('parseClientOptions', () => {
                     },
                   ]
               `);
-
-        options = parseClientOptions(
-          createConfig({
-            username: 'user',
-            password: 'pass',
-            hosts: ['http://node-A:9200'],
-          }),
-          false
-        );
-        expect(options.nodes).toMatchInlineSnapshot(`
-                  Array [
-                    Object {
-                      "url": "http://user:pass@node-a:9200/",
-                    },
-                  ]
-              `);
       });
     });
     describe('when `scoped` is true', () => {

src/core/server/elasticsearch/client/client_config.ts

@@ -93,7 +93,7 @@ export function parseClientOptions(
     };
   }
 
-  clientOptions.nodes = config.hosts.map((host) => convertHost(host, !scoped, config));
+  clientOptions.nodes = config.hosts.map((host) => convertHost(host));
 
   if (config.ssl) {
     clientOptions.ssl = generateSslConfig(
@@ -140,18 +140,10 @@ const generateSslConfig = (
   return ssl;
 };
 
-const convertHost = (
-  host: string,
-  needAuth: boolean,
-  { username, password }: ElasticsearchClientConfig
-): NodeOptions => {
+const convertHost = (host: string): NodeOptions => {
   const url = new URL(host);
   const isHTTPS = url.protocol === 'https:';
   url.port = url.port || (isHTTPS ? '443' : '80');
-  if (needAuth && username && password) {
-    url.username = username;
-    url.password = password;
-  }
 
   return {
     url,

src/dev/ci_setup/setup_env.sh

@@ -24,6 +24,10 @@ export NODE_OPTIONS="$NODE_OPTIONS --max-old-space-size=4096"
 ###
 export FORCE_COLOR=1
 
+### APM tracking
+###
+export ELASTIC_APM_ENVIRONMENT=ci
+
 ###
 ### check that we seem to be in a kibana project
 ###