Merge branch 'main' into Add-quality-issues-table-to-Logs-Flyout

.buildkite/scripts/common/deployment_credentials.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/vault_fns.sh
+
+## Usage
+# ./deployment_credentials.sh set <key-path> <key=value> <key=value> ...
+# ./deployment_credentials.sh unset <key-path>
+# ./deployment_credentials.sh print <key-path>
+
+if [[ "${1:-}" == "set" ]]; then
+  set_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "unset" ]]; then
+  unset_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "print" ]]; then
+  print_legacy_vault_read "${2}"
+else
+  echo "Unknown command: $1"
+  exit 1
+fi

.buildkite/scripts/common/vault_fns.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
-if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+# TODO: rewrite after https://github.com/elastic/kibana-operations/issues/15 is done
+export LEGACY_VAULT_ADDR="https://secrets.elastic.co:8200"
+if [[ "${VAULT_ADDR:-}" == "$LEGACY_VAULT_ADDR" ]]; then
   VAULT_PATH_PREFIX="secret/kibana-issues/dev"
   VAULT_KV_PREFIX="secret/kibana-issues/dev"
   IS_LEGACY_VAULT_ADDR=true
@@ -85,3 +86,56 @@ function get_vault_secret_id() {
 
     echo "$VAULT_SECRET_ID"
 }
+
+function set_in_legacy_vault() {
+  key_path=$1
+  shift
+  fields=("$@")
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  # shellcheck disable=SC2068
+  vault write -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path" ${fields[@]}
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function unset_in_legacy_vault() {
+  key_path=$1
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  vault delete -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path"
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function print_legacy_vault_read() {
+  key_path=$1
+
+  echo "vault read -address=$LEGACY_VAULT_ADDR secret/kibana-issues/dev/cloud-deploy/$key_path"
+}

.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh

@@ -15,7 +15,11 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true
 source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh
 
 echo "--- Running test script $1"
-TARGET_SCRIPT=$1 node .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/start_api_ftr_execution
+
+cd x-pack/test/security_solution_api_integration
+set +e
+
+TARGET_SCRIPT=$1 node ./scripts/mki_start_api_ftr_execution
 cmd_status=$?
 echo "Exit code with status: $cmd_status"
 exit $cmd_status

.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh

@@ -8,6 +8,13 @@ vault_get security-quality-gate/role-users data -format=json > .ftr/role_users.j
 vault_get security-quality-gate/role-users/sec-sol-auto-01 data -format=json > .ftr/sec-sol-auto-01.json
 vault_get security-quality-gate/role-users/sec-sol-auto-02 data -format=json > .ftr/sec-sol-auto-02.json
 vault_get security-quality-gate/role-users/sec-sol-auto-03 data -format=json > .ftr/sec-sol-auto-03.json
+vault_get security-quality-gate/role-users/sec-sol-auto-04 data -format=json > .ftr/sec-sol-auto-04.json
+vault_get security-quality-gate/role-users/sec-sol-auto-05 data -format=json > .ftr/sec-sol-auto-05.json
+vault_get security-quality-gate/role-users/sec-sol-auto-06 data -format=json > .ftr/sec-sol-auto-06.json
+vault_get security-quality-gate/role-users/sec-sol-auto-07 data -format=json > .ftr/sec-sol-auto-07.json
+vault_get security-quality-gate/role-users/sec-sol-auto-08 data -format=json > .ftr/sec-sol-auto-08.json
+vault_get security-quality-gate/role-users/sec-sol-auto-09 data -format=json > .ftr/sec-sol-auto-09.json
+vault_get security-quality-gate/role-users/sec-sol-auto-10 data -format=json > .ftr/sec-sol-auto-10.json
 
 # The vault entries relevant to QA Cloud
 export CLOUD_QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key)

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -82,7 +82,9 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
 
   echo "Writing to vault..."
 
-  vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  set_in_legacy_vault "$CLOUD_DEPLOYMENT_NAME" \
+    username="$CLOUD_DEPLOYMENT_USERNAME" \
+    password="$CLOUD_DEPLOYMENT_PASSWORD"
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -114,6 +116,7 @@ else
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
+VAULT_READ_COMMAND=$(print_legacy_vault_read "$CLOUD_DEPLOYMENT_NAME")
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
@@ -125,9 +128,7 @@ Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL
 
 Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-Credentials: \`vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
-
-(Stored in the production vault: VAULT_ADDR=https://vault-ci-prod.elastic.dev, more info: https://docs.elastic.dev/ci/using-secrets)
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -7,12 +7,10 @@
  */
 
 import { execSync } from 'child_process';
+import { getKibanaDir } from '#pipeline-utils';
 
 const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
 const { deployments } = JSON.parse(deploymentsListJson);
-const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
-  ? 'secret/kibana-issues/dev'
-  : 'secret/ci/elastic-kibana';
 
 const prDeployments = deployments.filter((deployment: any) =>
   deployment.name.startsWith('kibana-pr-')
@@ -70,7 +68,9 @@ for (const deployment of deploymentsToPurge) {
   console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
   try {
     execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
-    execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${deployment.name}`, {
+      cwd: getKibanaDir(),
       stdio: 'inherit',
     });
   } catch (ex) {

.buildkite/scripts/steps/cloud/purge_projects.ts

@@ -8,6 +8,7 @@
 
 import { execSync } from 'child_process';
 import axios from 'axios';
+import { getKibanaDir } from '#pipeline-utils';
 
 async function getPrProjects() {
   const match = /^(keep.?)?kibana-pr-([0-9]+)-(elasticsearch|security|observability)$/;
@@ -43,12 +44,19 @@ async function getPrProjects() {
 async function deleteProject({
   type,
   id,
+  name,
 }: {
   type: 'elasticsearch' | 'observability' | 'security';
   id: number;
+  name: string;
 }) {
   try {
     await projectRequest.delete(`/api/v1/serverless/projects/${type}/${id}`);
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${name}`, {
+      cwd: getKibanaDir(),
+      stdio: 'inherit',
+    });
   } catch (e) {
     if (e.isAxiosError) {
       const message =
@@ -61,7 +69,7 @@ async function deleteProject({
 
 async function purgeProjects() {
   const prProjects = await getPrProjects();
-  const projectsToPurge = [];
+  const projectsToPurge: typeof prProjects = [];
   for (const project of prProjects) {
     const NOW = new Date().getTime() / 1000;
     const DAY_IN_SECONDS = 60 * 60 * 24;

.buildkite/scripts/steps/serverless/deploy.sh

@@ -88,7 +88,10 @@ deploy() {
 
     echo "Write to vault..."
 
-    vault_kv_set "cloud-deploy/$VAULT_KEY_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+    set_in_legacy_vault "$VAULT_KEY_NAME" \
+     username="$PROJECT_USERNAME" \
+     password="$PROJECT_PASSWORD" \
+     id="$PROJECT_ID"
 
   else
     echo "Updating project..."
@@ -109,16 +112,16 @@ deploy() {
   PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
   PROJECT_ELASTICSEARCH_URL=$(jq -r '.endpoints.elasticsearch' $PROJECT_INFO_LOGS)
 
+  VAULT_READ_COMMAND=$(print_legacy_vault_read "$VAULT_KEY_NAME")
+
   cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
 ### $PROJECT_TYPE_LABEL Deployment
 
 Kibana: $PROJECT_KIBANA_LOGIN_URL
 
 Elasticsearch: $PROJECT_ELASTICSEARCH_URL
 
-Credentials: \`vault kv get $VAULT_KV_PREFIX/cloud-deploy/$VAULT_KEY_NAME\`
-
-(Stored in the production vault: VAULT_ADDR=https://vault-ci-prod.elastic.dev, more info: https://docs.elastic.dev/ci/using-secrets)
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_IMAGE\`
 EOF

.github/CODEOWNERS

@@ -1445,6 +1445,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/plugins/security_solution/public/detections/components/alerts_info @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/flyout/document_details @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/flyout/shared @elastic/security-threat-hunting-investigations
+/x-pack/plugins/security_solution/public/notes @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/resolver @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/threat_intelligence @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/timelines @elastic/security-threat-hunting-investigations

examples/controls_example/public/react_controls/control_renderer.tsx

@@ -59,7 +59,7 @@ export const ControlRenderer = <
           return fullApi;
         };
 
-        const { rawState: initialState } = parentApi.getSerializedStateForChild(uuid);
+        const { rawState: initialState } = parentApi.getSerializedStateForChild(uuid) ?? {};
 
         const { api, Component } = factory.buildControl(
           initialState as unknown as StateType,

examples/embeddable_examples/public/react_embeddables/saved_book/saved_book_react_embeddable.tsx

@@ -35,7 +35,7 @@ import {
 const bookSerializedStateIsByReference = (
   state?: BookSerializedState
 ): state is BookByReferenceSerializedState => {
-  return Boolean(state && (state as BookByReferenceSerializedState).savedBookId !== undefined);
+  return Boolean(state && (state as BookByReferenceSerializedState).savedBookId);
 };
 
 export const getSavedBookEmbeddableFactory = (core: CoreStart) => {
@@ -86,7 +86,7 @@ export const getSavedBookEmbeddableFactory = (core: CoreStart) => {
               defaultMessage: 'book',
             }),
           serializeState: async () => {
-            if (savedBookId$.value === undefined) {
+            if (!Boolean(savedBookId$.value)) {
               // if this book is currently by value, we serialize the entire state.
               const bookByValueState: BookByValueSerializedState = {
                 attributes: serializeBookAttributes(bookAttributesManager),
@@ -97,7 +97,7 @@ export const getSavedBookEmbeddableFactory = (core: CoreStart) => {
 
             // if this book is currently by reference, we serialize the reference and write to the external store.
             const bookByReferenceState: BookByReferenceSerializedState = {
-              savedBookId: savedBookId$.value,
+              savedBookId: savedBookId$.value!,
               ...serializeTitles(),
             };
 
@@ -123,6 +123,11 @@ export const getSavedBookEmbeddableFactory = (core: CoreStart) => {
           unlinkFromLibrary: () => {
             savedBookId$.next(undefined);
           },
+          getByValueRuntimeSnapshot: () => {
+            const snapshot = api.snapshotRuntimeState();
+            delete snapshot.savedBookId;
+            return snapshot;
+          },
         },
         {
           savedBookId: [savedBookId$, (val) => savedBookId$.next(val)],

examples/embeddable_examples/public/react_embeddables/saved_book/types.ts

@@ -47,4 +47,4 @@ export interface BookRuntimeState
 
 export type BookApi = DefaultEmbeddableApi<BookSerializedState, BookRuntimeState> &
   HasEditCapabilities &
-  HasInPlaceLibraryTransforms;
+  HasInPlaceLibraryTransforms<BookRuntimeState>;

packages/presentation/presentation_containers/interfaces/child_state.ts

@@ -9,7 +9,9 @@
 import { SerializedPanelState } from './serialized_state';
 
 export interface HasSerializedChildState<SerializedState extends object = object> {
-  getSerializedStateForChild: (childId: string) => SerializedPanelState<SerializedState>;
+  getSerializedStateForChild: (
+    childId: string
+  ) => SerializedPanelState<SerializedState> | undefined;
 }
 
 export interface HasRuntimeChildState<RuntimeState extends object = object> {

packages/presentation/presentation_containers/interfaces/serialized_state.ts

@@ -32,8 +32,8 @@ export const apiHasSerializableState = (api: unknown | null): api is HasSerializ
 
 export interface HasSnapshottableState<RuntimeState extends object = object> {
   /**
-   * Serializes all runtime state exactly as it appears. This could be used
-   * to rehydrate a component's state without needing to deserialize it.
+   * Serializes all runtime state exactly as it appears. This can be used
+   * to rehydrate a component's state without needing to serialize then deserialize it.
    */
   snapshotRuntimeState: () => RuntimeState;
 }

packages/presentation/presentation_publishing/index.ts

@@ -16,8 +16,8 @@ export interface EmbeddableApiContext {
 
 export {
   getInitialValuesFromComparators,
-  runComparators,
   getUnchangingComparator,
+  runComparators,
   type ComparatorDefinition,
   type ComparatorFunction,
   type StateComparators,
@@ -35,22 +35,22 @@ export {
   type SerializedTimeRange,
 } from './interfaces/fetch/initialize_time_range';
 export {
-  apiPublishesPartialUnifiedSearch,
   apiPublishesFilters,
+  apiPublishesPartialUnifiedSearch,
   apiPublishesTimeRange,
   apiPublishesUnifiedSearch,
   apiPublishesWritableUnifiedSearch,
   useSearchApi,
-  type PublishesTimeRange,
   type PublishesFilters,
+  type PublishesTimeRange,
+  type PublishesTimeslice,
   type PublishesUnifiedSearch,
   type PublishesWritableUnifiedSearch,
-  type PublishesTimeslice,
 } from './interfaces/fetch/publishes_unified_search';
 export {
   apiHasAppContext,
-  type HasAppContext,
   type EmbeddableAppContext,
+  type HasAppContext,
 } from './interfaces/has_app_context';
 export {
   apiHasDisableTriggers,
@@ -63,9 +63,9 @@ export {
   type HasExecutionContext,
 } from './interfaces/has_execution_context';
 export {
+  apiHasInPlaceLibraryTransforms,
   apiHasLegacyLibraryTransforms,
   apiHasLibraryTransforms,
-  apiHasInPlaceLibraryTransforms,
   type HasInPlaceLibraryTransforms,
   type HasLegacyLibraryTransforms,
   type HasLibraryTransforms,
@@ -130,7 +130,11 @@ export {
   type PublishesPanelTitle,
   type PublishesWritablePanelTitle,
 } from './interfaces/titles/publishes_panel_title';
-export { initializeTitles, type SerializedTitles } from './interfaces/titles/titles_api';
+export {
+  initializeTitles,
+  stateHasTitles,
+  type SerializedTitles,
+} from './interfaces/titles/titles_api';
 export {
   useBatchedOptionalPublishingSubjects,
   useBatchedPublishingSubjects,