Merge branch '8.0' into cases-remove-rule-fields

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -9,3 +9,5 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
+    artifact_paths:
+      - "target/kibana-osquery/**/*"

.buildkite/scripts/steps/functional/osquery_cypress.sh

@@ -4,8 +4,9 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
+export BUILD_TS_REFS_DISABLE=false
 .buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
+node scripts/build_kibana_platform_plugins.js
 
 export JOB=kibana-osquery-cypress
 
@@ -16,5 +17,5 @@ cd "$XPACK_DIR"
 checks-reporter-with-killswitch "Osquery Cypress Tests" \
  node scripts/functional_tests \
    --debug --bail \
-   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
    --config test/osquery_cypress/cli_config.ts
+

docs/CHANGELOG.asciidoc

@@ -134,17 +134,19 @@ The `xpack.apm.autocreateApmIndexPattern` APM setting has been removed. For more
 *Impact* +
 To automatically create data views in APM, use `xpack.apm.autoCreateApmDataView`.
 ====
-      
+
 [discrete]
 [[deprecation-119494]]
-.Updates Fleet API responses for consistency
+.Updates Fleet API to improve consistency
 [%collapsible]
 ====
 *Details* +
-To make sure all Fleet API GET resposes return `items`, the following have been updated:
+The Fleet API has been updated to improve consistency:
 
-* `/api/fleet/enrollment-api-keys`
-* `/api/fleet/agents`
+* Hyphens are changed to underscores in some names.
+* The `pkgkey` path parameter in the packages endpoint is split.
+* The `response` and `list` properties are renamed to `items` or `item` in some
+responses.
 
 For more information, refer to {kibana-pull}119494[#119494].
 
@@ -157,24 +159,30 @@ When you upgrade to 8.0.0, use the following API changes:
 
 * Use `service_tokens` instead of `service-tokens`.
 
-* `check-permissions` is no longer supported.
-
 * Use `/epm/packages/{packageName}/{version}` instead of `/epm/packages/{pkgkey}`.
 
-* Use `items[]` or `item` instead of `response[]` in the following:
-
+* Use `items[]` instead of `response[]` in:
++
 [source,text]
 --
+/api/fleet/enrollment_api_keys
+/api/fleet/agents
 /epm/packages/
-/epm/packages/{pkgkey}
 /epm/categories
 /epm/packages/_bulk
 /epm/packages/limited
+/epm/packages/{packageName}/{version} <1>
 --
+<1> Use `items[]` when the verb is `POST` or `DELETE`. Use `item` when the verb
+is `GET` or `PUT`.
+
+For more information, refer to {fleet-guide}/fleet-api-docs.html[Fleet APIs].
+
 ====
 
-To review the depcrecations in previous versions, refer to the <<deprecations-8.0.0-alpha1,8.0.0-alpha1 release notes>>. 
-
+To review the deprecations in previous versions, refer to the <<deprecations-8.0.0-alpha1,8.0.0-alpha1 release notes>>. 
+
+
 [float]
 [[features-8.0.0-rc1]]
 === Features

docs/apm/troubleshooting.asciidoc

@@ -185,7 +185,7 @@ There are two things you can do to if you'd like to ensure a field is searchable
 1. Index your additional data as {apm-guide-ref}/metadata.html[labels] instead.
 These are dynamic by default, which means they will be indexed and become searchable and aggregatable.
 
-2. Use the {apm-guide-ref}/configuration-template.html[`append_fields`] feature. As an example,
+2. Use the `append_fields` feature. As an example,
 adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`:
 
 [source,yml]

docs/management/connectors/action-types/email.asciidoc

@@ -16,7 +16,7 @@ NOTE: For emails to have a footer with a link back to {kib}, set the <<server-pu
 Email connectors have the following configuration properties.
 
 Name::      The name of the connector. The name is used to identify a  connector in the management UI connector listing, or in the connector list when configuring an action.
-Sender::    The from address for all emails sent with this connector. This can be specified in `user@host-name` format or as `"human name <user@host-name>"` format. See the https://nodemailer.com/message/addresses/[Nodemailer address documentation] for more information.
+Sender::    The from address for all emails sent with this connector. This must be specified in `user@host-name` format. See the https://nodemailer.com/message/addresses/[Nodemailer address documentation] for more information.
 Service::   The name of the email service. If `service` is one of Nodemailer's https://nodemailer.com/smtp/well-known/[well-known email service providers], the `host`, `port`, and `secure` properties are defined with the default values and disabled for modification. If `service` is `MS Exchange Server`, the `host`, `port`, and `secure` properties are ignored and `tenantId`, `clientId`, `clientSecret` are required instead. If `service` is `other`, the `host` and `port` properties must be defined.
 Host::      Host name of the service provider. If you are using the <<action-settings, `xpack.actions.allowedHosts`>> setting, make sure this hostname is added to the allowed hosts.
 Port::      The port to connect to on the service provider.

docs/osquery/osquery.asciidoc

@@ -121,11 +121,18 @@ image::images/scheduled-pack.png[Shows queries in the pack and details about eac
 
 [float]
 [[osquery-manage-query]]
-== Edit saved queries
+== Save queries
 
-Add or edit saved queries from the *Saved queries* tab.
+You can save queries in two ways:
 
-. Go to the saved queries, then click **Add saved query** or the edit icon.
+* After running a live query, click the *Save for later* link.
+* From the *Saved queries* tab, click the **Add saved query** button.
+
+Once you save a query, you can only edit it from the *Saved queries* tab.
+
+To add or edit saved queries from the *Saved queries* tab:
+
+. Go to *Saved queries*, and then click **Add saved query** or the edit icon.
 . Provide the following fields:
 
 * The unique identifier.
@@ -148,7 +155,7 @@ Add or edit saved queries from the *Saved queries* tab.
 
 * From the *Test query* panel, select agents or groups to test the query, then click *Submit* to run a live query. Result columns with the image:images/mapped-icon.png[mapping] icon are mapped. Hover over the icon to see the mapped ECS field.
 
-. Click **Save query**.
+. Click *Save* or *Update*. 
 
 [float]
 [[osquery-map-fields]]
@@ -175,11 +182,7 @@ and the mapped ECS fields. For example, if you update a query to map `osquery.na
 
 ** **Static value**: Enter a static value. When the query runs, the ECS field is set to the value entered. For example, static fields can be used to apply `tags` or your preferred `event.category` to the query results. 
 
-. Map more fields, as needed.
-
-** To add a new row for additional fields to map, click the plus icon.
-
-** To remove any mapped rows, click the trash icon.
+. Map more fields, as needed. To remove any mapped rows, click the delete icon.
 
 . Save your changes.
 
@@ -314,7 +317,7 @@ While this allows you to use advanced Osquery functionality like pack discovery
 
 . Edit the *Osquery config* JSON field to apply your preferred Osquery configuration. Note the following:
 
-* The field may already have content if you have scheduled packs for this agent policy. To keep these packs scheduled, do not edit the `packs` section.
+* The field may already have content if you have scheduled packs for this agent policy. To keep these packs scheduled, do not remove the `packs` section.
 
 * Refer to the https://osquery.readthedocs.io/en/stable/[Osquery documentation] for configuration options. 
 
@@ -344,14 +347,12 @@ https://www.elastic.co/guide/en/fleet/master/upgrade-elastic-agent.html[upgrade
 
 [float]
 === Debug issues
-If you encounter issues with *Osquery Manager*, find the relevant logs for the {elastic-agent}
-and Osquerybeat in the installed agent directory, then adjust the agent path for your setup. 
-
-The relevant logs look similar to the following example paths:
+If you encounter issues with *Osquery Manager*, find the relevant logs for {elastic-agent}
+and Osquerybeat in the agent directory. Refer to the {fleet-guide}/installation-layout.html[Fleet Installation layout] to find the log file location for your OS.
 
 ```ts
-`/data/elastic-agent-054e22/logs/elastic-agent-json.log-*`
-`/data/elastic-agent-054e22/logs/default/osquerybeat-json.log`
+../data/elastic-agent-*/logs/elastic-agent-json.log-*
+../data/elastic-agent-*/logs/default/osquerybeat-json.log
 ```
 
 To get more details in the logs, change the agent logging level to debug:

packages/elastic-apm-synthtrace/src/scripts/examples/01_simple_trace.ts

@@ -12,14 +12,14 @@ import { getApmWriteTargets } from '../../lib/apm/utils/get_apm_write_targets';
 import { Scenario } from '../scenario';
 import { getCommonServices } from '../utils/get_common_services';
 
-const scenario: Scenario = async ({ target, logLevel }) => {
+const scenario: Scenario = async ({ target, logLevel, scenarioOpts }) => {
   const { client, logger } = getCommonServices({ target, logLevel });
   const writeTargets = await getApmWriteTargets({ client });
 
+  const { numServices = 3 } = scenarioOpts || {};
+
   return {
     generate: ({ from, to }) => {
-      const numServices = 3;
-
       const range = timerange(from, to);
 
       const transactionName = '240rpm/75% 1000ms';

packages/elastic-apm-synthtrace/src/scripts/run.ts

@@ -69,6 +69,12 @@ function options(y: Argv) {
       describe: 'Target to index',
       string: true,
     })
+    .option('scenarioOpts', {
+      describe: 'Options specific to the scenario',
+      coerce: (arg) => {
+        return arg as Record<string, any> | undefined;
+      },
+    })
     .conflicts('to', 'live');
 }
 

packages/elastic-apm-synthtrace/src/scripts/utils/parse_run_cli_flags.ts

@@ -47,7 +47,15 @@ export function parseRunCliFlags(flags: RunCliFlags) {
   }
 
   return {
-    ...pick(flags, 'target', 'workers', 'clientWorkers', 'batchSize', 'writeTarget'),
+    ...pick(
+      flags,
+      'target',
+      'workers',
+      'clientWorkers',
+      'batchSize',
+      'writeTarget',
+      'scenarioOpts'
+    ),
     intervalInMs,
     bucketSizeInMs,
     logLevel: parsedLogLevel,

packages/elastic-apm-synthtrace/src/scripts/utils/start_historical_data_upload.ts

@@ -24,6 +24,7 @@ export async function startHistoricalDataUpload({
   target,
   file,
   writeTarget,
+  scenarioOpts,
 }: RunOptions & { from: number; to: number }) {
   let requestedUntil: number = from;
 
@@ -57,6 +58,7 @@ export async function startHistoricalDataUpload({
       target,
       workers,
       writeTarget,
+      scenarioOpts,
     };
 
     const worker = new Worker(Path.join(__dirname, './upload_next_batch.js'), {

packages/elastic-apm-synthtrace/src/scripts/utils/start_live_data_upload.ts

@@ -24,6 +24,7 @@ export async function startLiveDataUpload({
   logLevel,
   workers,
   writeTarget,
+  scenarioOpts,
 }: RunOptions & { start: number }) {
   let queuedEvents: ElasticsearchOutput[] = [];
   let requestedUntil: number = start;
@@ -41,6 +42,7 @@ export async function startLiveDataUpload({
     target,
     workers,
     writeTarget,
+    scenarioOpts,
   });
 
   function uploadNextBatch() {

packages/elastic-apm-synthtrace/src/scripts/utils/upload_next_batch.ts

@@ -17,6 +17,7 @@ export interface WorkerData {
   bucketFrom: number;
   bucketTo: number;
   file: string;
+  scenarioOpts: Record<string, any> | undefined;
   logLevel: LogLevel;
   clientWorkers: number;
   batchSize: number;
@@ -39,6 +40,7 @@ const {
   workers,
   target,
   writeTarget,
+  scenarioOpts,
 } = workerData as WorkerData;
 
 async function uploadNextBatch() {
@@ -63,6 +65,7 @@ async function uploadNextBatch() {
     target,
     workers,
     writeTarget,
+    scenarioOpts,
   });
 
   const events = logger.perf('execute_scenario', () =>

src/core/server/elasticsearch/client/cluster_client.test.ts

@@ -144,13 +144,13 @@ describe('ClusterClient', () => {
       });
     });
 
-    it('creates a scoped facade with filtered auth headers', () => {
+    it('does not filter auth headers', () => {
       const config = createConfig({
         requestHeadersWhitelist: ['authorization'],
       });
       getAuthHeaders.mockReturnValue({
         authorization: 'auth',
-        other: 'nope',
+        other: 'yep',
       });
 
       const clusterClient = new ClusterClient(config, logger, 'custom-type', getAuthHeaders);
@@ -160,7 +160,12 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { ...DEFAULT_HEADERS, authorization: 'auth', 'x-opaque-id': expect.any(String) },
+        headers: {
+          ...DEFAULT_HEADERS,
+          authorization: 'auth',
+          other: 'yep',
+          'x-opaque-id': expect.any(String),
+        },
       });
     });
 
@@ -170,7 +175,7 @@ describe('ClusterClient', () => {
       });
       getAuthHeaders.mockReturnValue({
         authorization: 'auth',
-        other: 'nope',
+        other: 'yep',
       });
 
       const clusterClient = new ClusterClient(config, logger, 'custom-type', getAuthHeaders);
@@ -184,7 +189,12 @@ describe('ClusterClient', () => {
 
       expect(scopedClient.child).toHaveBeenCalledTimes(1);
       expect(scopedClient.child).toHaveBeenCalledWith({
-        headers: { ...DEFAULT_HEADERS, authorization: 'auth', 'x-opaque-id': expect.any(String) },
+        headers: {
+          ...DEFAULT_HEADERS,
+          authorization: 'auth',
+          other: 'yep',
+          'x-opaque-id': expect.any(String),
+        },
       });
     });
 

src/core/server/elasticsearch/client/cluster_client.ts

@@ -54,8 +54,6 @@ export interface ICustomClusterClient extends IClusterClient {
 export class ClusterClient implements ICustomClusterClient {
   public readonly asInternalUser: KibanaClient;
   private readonly rootScopedClient: KibanaClient;
-  private readonly allowListHeaders: string[];
-
   private isClosed = false;
 
   constructor(
@@ -72,8 +70,6 @@ export class ClusterClient implements ICustomClusterClient {
       getExecutionContext,
       scoped: true,
     });
-
-    this.allowListHeaders = ['x-opaque-id', ...this.config.requestHeadersWhitelist];
   }
 
   asScoped(request: ScopeableRequest) {
@@ -95,14 +91,15 @@ export class ClusterClient implements ICustomClusterClient {
   private getScopedHeaders(request: ScopeableRequest): Headers {
     let scopedHeaders: Headers;
     if (isRealRequest(request)) {
-      const requestHeaders = ensureRawRequest(request).headers;
+      const requestHeaders = ensureRawRequest(request).headers ?? {};
       const requestIdHeaders = isKibanaRequest(request) ? { 'x-opaque-id': request.id } : {};
-      const authHeaders = this.getAuthHeaders(request);
+      const authHeaders = this.getAuthHeaders(request) ?? {};
 
-      scopedHeaders = filterHeaders(
-        { ...requestHeaders, ...requestIdHeaders, ...authHeaders },
-        this.allowListHeaders
-      );
+      scopedHeaders = {
+        ...filterHeaders(requestHeaders, this.config.requestHeadersWhitelist),
+        ...requestIdHeaders,
+        ...authHeaders,
+      };
     } else {
       scopedHeaders = filterHeaders(request?.headers ?? {}, this.config.requestHeadersWhitelist);
     }

src/core/server/http/router/headers.ts

@@ -55,7 +55,7 @@ export function filterHeaders(
   headers: Headers,
   fieldsToKeep: string[],
   fieldsToExclude: string[] = []
-) {
+): Headers {
   const fieldsToExcludeNormalized = fieldsToExclude.map(normalizeHeaderField);
   // Normalize list of headers we want to allow in upstream request
   const fieldsToKeepNormalized = fieldsToKeep

src/plugins/discover/public/application/apps/context/context_app.tsx

@@ -152,7 +152,7 @@ export const ContextApp = ({ indexPattern, anchorId }: ContextAppProps) => {
           <EuiPage className={classNames({ dscDocsPage: !isLegacy })}>
             <EuiPageContent paddingSize="s" className="dscDocsContent">
               <EuiSpacer size="s" />
-              <EuiText>
+              <EuiText data-test-subj="contextDocumentSurroundingHeader">
                 <strong>
                   <FormattedMessage
                     id="discover.context.contextOfTitle"