Merge branch 'main' into eui-upgrade-47.0.0

docs/developer/contributing/development-functional-tests.asciidoc

@@ -105,7 +105,9 @@ There are also command line flags for `--bail` and `--grep`, which behave just l
 
 Logging can also be customized with `--quiet`, `--debug`, or `--verbose` flags.
 
-Use the `--help` flag for more options.
+There are also options like `--include` to run only the tests defined in a single file or set of files.
+
+Run `node scripts/functional_test_runner --help` to see all available options.
 
 
 [discrete]

docs/management/connectors/action-types/slack.asciidoc

@@ -76,5 +76,3 @@ URL, set up an an **Incoming Webhook Integration** through the Slack console:
 image::images/slack-add-webhook-integration.png[]
 . Click *Add Incoming Webhook Integration*.
 . Copy the generated webhook URL so you can paste it into your Slack connector form.
-+
-image::images/slack-copy-webhook-url.png[]

src/core/server/deprecations/deprecations_registry.test.ts

@@ -65,6 +65,25 @@ describe('DeprecationsRegistry', () => {
       ]);
     });
 
+    it('rejects deprecations when reaching the timeout', async () => {
+      const deprecationsRegistry = new DeprecationsRegistry({ timeout: 100 });
+      const mockContext = {} as unknown as GetDeprecationsContext;
+      const deprecationsConfigA = {
+        getDeprecations: jest.fn().mockReturnValue(new Promise(() => {})),
+      };
+      deprecationsRegistry.registerDeprecations(deprecationsConfigA);
+      const deprecations = await deprecationsRegistry.getDeprecations(mockContext);
+      expect(deprecations).toStrictEqual([
+        {
+          status: 'rejected',
+          reason: expect.any(Error),
+        },
+      ]);
+      expect((deprecations[0] as PromiseRejectedResult).reason.message).toEqual(
+        'Deprecations did not resolve in 10sec.'
+      );
+    });
+
     it('passes dependencies to registered getDeprecations function', async () => {
       const deprecationsRegistry = new DeprecationsRegistry();
       const mockContext = {} as unknown as GetDeprecationsContext;

src/core/server/deprecations/deprecations_registry.ts

@@ -6,15 +6,23 @@
  * Side Public License, v 1.
  */
 
+import { withTimeout, isPromise } from '@kbn/std';
 import type {
   DeprecationsDetails,
   RegisterDeprecationsConfig,
   GetDeprecationsContext,
 } from './types';
 
+const MsInSec = 1000;
+
 export class DeprecationsRegistry {
+  private readonly timeout: number;
   private readonly deprecationContexts: RegisterDeprecationsConfig[] = [];
 
+  constructor({ timeout = 10 * MsInSec }: { timeout?: number } = {}) {
+    this.timeout = timeout;
+  }
+
   public registerDeprecations = (deprecationContext: RegisterDeprecationsConfig) => {
     if (typeof deprecationContext.getDeprecations !== 'function') {
       throw new Error(`getDeprecations must be a function in registerDeprecations(context)`);
@@ -27,9 +35,21 @@ export class DeprecationsRegistry {
     dependencies: GetDeprecationsContext
   ): Promise<Array<PromiseSettledResult<DeprecationsDetails[]>>> => {
     return await Promise.allSettled(
-      this.deprecationContexts.map(
-        async (deprecationContext) => await deprecationContext.getDeprecations(dependencies)
-      )
+      this.deprecationContexts.map(async (deprecationContext) => {
+        const maybePromise = deprecationContext.getDeprecations(dependencies);
+        if (isPromise(maybePromise)) {
+          const resultOrTimeout = await withTimeout({
+            promise: maybePromise,
+            timeoutMs: this.timeout,
+          });
+          if (resultOrTimeout.timedout) {
+            throw new Error('Deprecations did not resolve in 10sec.');
+          }
+          return resultOrTimeout.value;
+        } else {
+          return maybePromise;
+        }
+      })
     );
   };
 }

x-pack/plugins/alerting/server/saved_objects/migrations.test.ts

@@ -2063,31 +2063,172 @@ describe('successful migrations', () => {
         { params: { outputIndex: 'output-index', type: 'query' }, alertTypeId: 'not.siem.signals' },
         true
       );
-      expect(migration800(alert, migrationContext).attributes.alertTypeId).toEqual(
-        'not.siem.signals'
-      );
-      expect(migration800(alert, migrationContext).attributes.enabled).toEqual(true);
-      expect(migration800(alert, migrationContext).attributes.params.outputIndex).toEqual(
-        'output-index'
-      );
+      const migratedAlert = migration800(alert, migrationContext);
+      expect(migratedAlert.attributes.alertTypeId).toEqual('not.siem.signals');
+      expect(migratedAlert.attributes.enabled).toEqual(true);
+      expect(migratedAlert.attributes.tags).toEqual(['foo']);
+      expect(migratedAlert.attributes.params.outputIndex).toEqual('output-index');
     });
 
     test.each(Object.keys(ruleTypeMappings) as RuleType[])(
-      'Changes AAD rule params accordingly if rule is a siem.signals %p rule',
+      'changes AAD rule params accordingly if rule is a siem.signals %p rule',
       (ruleType) => {
         const migration800 = getMigrations(encryptedSavedObjectsSetup, isPreconfigured)['8.0.0'];
         const alert = getMockData(
           { params: { outputIndex: 'output-index', type: ruleType }, alertTypeId: 'siem.signals' },
           true
         );
-        expect(migration800(alert, migrationContext).attributes.alertTypeId).toEqual(
-          ruleTypeMappings[ruleType]
-        );
-        expect(migration800(alert, migrationContext).attributes.enabled).toEqual(false);
-        expect(migration800(alert, migrationContext).attributes.params.outputIndex).toEqual('');
+        const migratedAlert = migration800(alert, migrationContext);
+        expect(migratedAlert.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+        expect(migratedAlert.attributes.enabled).toEqual(false);
+        expect(migratedAlert.attributes.tags).toEqual(['foo']);
+        expect(migratedAlert.attributes.params.outputIndex).toEqual('');
       }
     );
 
+    describe('8.0.1', () => {
+      describe.each(Object.keys(ruleTypeMappings) as RuleType[])(
+        'auto_disabled %p rule tags',
+        (ruleType) => {
+          const alert717Enabled = getMockData(
+            {
+              params: { outputIndex: 'output-index', type: ruleType },
+              alertTypeId: 'siem.signals',
+              enabled: true,
+              scheduledTaskId: 'abcd',
+            },
+            true
+          );
+          const alert717Disabled = getMockData(
+            {
+              params: { outputIndex: 'output-index', type: ruleType },
+              alertTypeId: 'siem.signals',
+              enabled: false,
+            },
+            true
+          );
+          const alert800 = getMockData(
+            {
+              params: { outputIndex: '', type: ruleType },
+              alertTypeId: ruleTypeMappings[ruleType],
+              enabled: false,
+              scheduledTaskId: 'abcd',
+            },
+            true
+          );
+
+          test('Does not update rule tags if rule has already been enabled', () => {
+            const migrations = getMigrations(encryptedSavedObjectsSetup, isPreconfigured);
+            const migration800 = migrations['8.0.0'];
+            const migration801 = migrations['8.0.1'];
+
+            // migrate to 8.0.0
+            const migratedAlert800 = migration800(alert717Enabled, migrationContext);
+            expect(migratedAlert800.attributes.enabled).toEqual(false);
+
+            // reenable rule
+            migratedAlert800.attributes.enabled = true;
+
+            // migrate to 8.0.1
+            const migratedAlert801 = migration801(migratedAlert800, migrationContext);
+
+            expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+            expect(migratedAlert801.attributes.enabled).toEqual(true);
+            expect(migratedAlert801.attributes.params.outputIndex).toEqual('');
+
+            // tags not updated
+            expect(migratedAlert801.attributes.tags).toEqual(['foo']);
+          });
+
+          test('Does not update rule tags if rule was already disabled before upgrading to 8.0', () => {
+            const migrations = getMigrations(encryptedSavedObjectsSetup, isPreconfigured);
+            const migration800 = migrations['8.0.0'];
+            const migration801 = migrations['8.0.1'];
+
+            // migrate to 8.0.0
+            const migratedAlert800 = migration800(alert717Disabled, migrationContext);
+            expect(migratedAlert800.attributes.enabled).toEqual(false);
+
+            // migrate to 8.0.1
+            const migratedAlert801 = migration801(migratedAlert800, migrationContext);
+
+            expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+            expect(migratedAlert801.attributes.enabled).toEqual(false);
+            expect(migratedAlert801.attributes.params.outputIndex).toEqual('');
+
+            // tags not updated
+            expect(migratedAlert801.attributes.tags).toEqual(['foo']);
+          });
+
+          test('Updates rule tags if rule was auto-disabled in 8.0 upgrade and not reenabled', () => {
+            const migrations = getMigrations(encryptedSavedObjectsSetup, isPreconfigured);
+            const migration800 = migrations['8.0.0'];
+            const migration801 = migrations['8.0.1'];
+
+            // migrate to 8.0.0
+            const migratedAlert800 = migration800(alert717Enabled, migrationContext);
+            expect(migratedAlert800.attributes.enabled).toEqual(false);
+
+            // migrate to 8.0.1
+            const migratedAlert801 = migration801(migratedAlert800, migrationContext);
+
+            expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+            expect(migratedAlert801.attributes.enabled).toEqual(false);
+            expect(migratedAlert801.attributes.params.outputIndex).toEqual('');
+
+            // tags updated
+            expect(migratedAlert801.attributes.tags).toEqual(['foo', 'auto_disabled_8.0']);
+          });
+
+          test('Updates rule tags correctly if tags are undefined', () => {
+            const migrations = getMigrations(encryptedSavedObjectsSetup, isPreconfigured);
+            const migration801 = migrations['8.0.1'];
+
+            const alert = {
+              ...alert800,
+              attributes: {
+                ...alert800.attributes,
+                tags: undefined,
+              },
+            };
+
+            // migrate to 8.0.1
+            const migratedAlert801 = migration801(alert, migrationContext);
+
+            expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+            expect(migratedAlert801.attributes.enabled).toEqual(false);
+            expect(migratedAlert801.attributes.params.outputIndex).toEqual('');
+
+            // tags updated
+            expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']);
+          });
+
+          test('Updates rule tags correctly if tags are null', () => {
+            const migrations = getMigrations(encryptedSavedObjectsSetup, isPreconfigured);
+            const migration801 = migrations['8.0.1'];
+
+            const alert = {
+              ...alert800,
+              attributes: {
+                ...alert800.attributes,
+                tags: null,
+              },
+            };
+
+            // migrate to 8.0.1
+            const migratedAlert801 = migration801(alert, migrationContext);
+
+            expect(migratedAlert801.attributes.alertTypeId).toEqual(ruleTypeMappings[ruleType]);
+            expect(migratedAlert801.attributes.enabled).toEqual(false);
+            expect(migratedAlert801.attributes.params.outputIndex).toEqual('');
+
+            // tags updated
+            expect(migratedAlert801.attributes.tags).toEqual(['auto_disabled_8.0']);
+          });
+        }
+      );
+    });
+
     describe('Metrics Inventory Threshold rule', () => {
       test('Migrates incorrect action group spelling', () => {
         const migration800 = getMigrations(encryptedSavedObjectsSetup, isPreconfigured)['8.0.0'];

x-pack/plugins/alerting/server/saved_objects/migrations.ts

@@ -58,6 +58,9 @@ export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc<RawRu
 export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc<RawRule>): boolean =>
   doc.attributes.alertTypeId === 'siem.signals';
 
+export const isDetectionEngineAADRuleType = (doc: SavedObjectUnsanitizedDoc<RawRule>): boolean =>
+  (Object.values(ruleTypeMappings) as string[]).includes(doc.attributes.alertTypeId);
+
 /**
  * Returns true if the alert type is that of "siem.notifications" which is a legacy notification system that was deprecated in 7.16.0
  * in favor of using the newer alerting notifications system.
@@ -136,6 +139,12 @@ export function getMigrations(
     )
   );
 
+  const migrationRules801 = createEsoMigration(
+    encryptedSavedObjects,
+    (doc: SavedObjectUnsanitizedDoc<RawRule>): doc is SavedObjectUnsanitizedDoc<RawRule> => true,
+    pipeMigrations(addSecuritySolutionAADRuleTypeTags)
+  );
+
   return {
     '7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'),
     '7.11.0': executeMigrationWithErrorHandling(migrationAlertUpdatedAtAndNotifyWhen, '7.11.0'),
@@ -145,6 +154,7 @@ export function getMigrations(
     '7.15.0': executeMigrationWithErrorHandling(migrationSecurityRules715, '7.15.0'),
     '7.16.0': executeMigrationWithErrorHandling(migrateRules716, '7.16.0'),
     '8.0.0': executeMigrationWithErrorHandling(migrationRules800, '8.0.0'),
+    '8.0.1': executeMigrationWithErrorHandling(migrationRules801, '8.0.1'),
   };
 }
 
@@ -672,6 +682,28 @@ function addSecuritySolutionAADRuleTypes(
     : doc;
 }
 
+function addSecuritySolutionAADRuleTypeTags(
+  doc: SavedObjectUnsanitizedDoc<RawRule>
+): SavedObjectUnsanitizedDoc<RawRule> {
+  const ruleType = doc.attributes.params.type;
+  return isDetectionEngineAADRuleType(doc) && isRuleType(ruleType)
+    ? {
+        ...doc,
+        attributes: {
+          ...doc.attributes,
+          // If the rule is disabled at this point, then the rule has not been re-enabled after
+          // running the 8.0.0 migrations. If `doc.attributes.scheduledTaskId` exists, then the
+          // rule was enabled prior to running the migration. Thus we know we should add the
+          // tag to indicate it was auto-disabled.
+          tags:
+            !doc.attributes.enabled && doc.attributes.scheduledTaskId
+              ? [...(doc.attributes.tags ?? []), 'auto_disabled_8.0']
+              : doc.attributes.tags ?? [],
+        },
+      }
+    : doc;
+}
+
 function addThreatIndicatorPathToThreatMatchRules(
   doc: SavedObjectUnsanitizedDoc<RawRule>
 ): SavedObjectUnsanitizedDoc<RawRule> {

x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts

@@ -13,7 +13,7 @@ import { packageRegistryPort } from './ftr_config';
 import { FtrProviderContext } from './ftr_provider_context';
 
 export const dockerImage =
-  'docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61';
+  'docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b';
 
 async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) {
   const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -107,7 +107,13 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
   // should an uploaded package be able to specify `internal`?
-  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal'>;
+  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
+
+export interface BundledPackage {
+  name: string;
+  version: string;
+  buffer: Buffer;
+}
 
 export type RegistryPackage = PackageSpecManifest &
   Partial<RegistryOverridesToOptional> &

x-pack/plugins/fleet/server/errors/index.ts

@@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {}
 export class AgentReassignmentError extends IngestManagerError {}
 export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {}
 export class PackagePolicyValidationError extends IngestManagerError {}
+export class BundledPackageNotFoundError extends IngestManagerError {}
 export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError {
   constructor(message = 'Cannot perform that action') {
     super(

x-pack/plugins/fleet/server/integration_tests/docker_registry_helper.ts

@@ -24,7 +24,7 @@ export function useDockerRegistry() {
 
   let dockerProcess: ChildProcess | undefined;
   async function startDockerRegistryServer() {
-    const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:de952debe048d903fc73e8a4472bb48bb95028d440cba852f21b863d47020c61`;
+    const dockerImage = `docker.elastic.co/package-registry/distribution@sha256:c5bf8e058727de72e561b228f4b254a14a6f880e582190d01bd5ff74318e1d0b`;
 
     const args = ['run', '--rm', '-p', `${packageRegistryPort}:8080`, dockerImage];
 

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -34,7 +34,7 @@ export async function getFullAgentPolicy(
   options?: { standalone: boolean }
 ): Promise<FullAgentPolicy | null> {
   let agentPolicy;
-  const standalone = options?.standalone;
+  const standalone = options?.standalone ?? false;
 
   try {
     agentPolicy = await agentPolicyService.get(soClient, id);