Merge branch 'main' into poc_decouple_consumers_feature_ids

.buildkite/scripts/copy_es_snapshot_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# If cached snapshots are baked into the agent, copy them into our workspace first
+# We are doing this rather than simply changing the ES base path because many workers
+#   run with the workspace mounted in memory or on a local ssd
+cacheDir="$ES_CACHE_DIR/cache"
+if [[ -d "$cacheDir" ]]; then
+  mkdir -p .es/cache
+  echo "--- Copying ES snapshot cache"
+  echo "Copying cached snapshots from $cacheDir to .es/cache"
+  cp -R "$cacheDir"/* .es/cache/
+fi

.buildkite/scripts/steps/functional/common.sh

@@ -8,17 +8,7 @@ source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
-
-# If cached snapshots are baked into the agent, copy them into our workspace first
-# We are doing this rather than simply changing the ES base path because many workers
-#   run with the workspace mounted in memory or on a local ssd
-cacheDir="$ES_CACHE_DIR/cache"
-if [[ -d "$cacheDir" ]]; then
-  mkdir -p .es/cache
-  echo "--- Copying ES snapshot cache"
-  echo "Copying cached snapshots from $cacheDir to .es/cache"
-  cp -R "$cacheDir"/* .es/cache/
-fi
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 is_test_execution_step
 

.buildkite/scripts/steps/test/jest_integration.sh

@@ -7,6 +7,7 @@ source .buildkite/scripts/common/util.sh
 is_test_execution_step
 
 .buildkite/scripts/bootstrap.sh
+.buildkite/scripts/copy_es_snapshot_cache.sh
 
 echo '--- Jest Integration Tests'
 .buildkite/scripts/steps/test/jest_parallel.sh jest.integration.config.js

packages/kbn-es-query/index.ts

@@ -128,6 +128,7 @@ export {
   getDataViewFieldSubtypeNested,
   isDataViewFieldSubtypeMulti,
   isDataViewFieldSubtypeNested,
+  isCCSRemoteIndexName,
 } from './src/utils';
 
 export type { ExecutionContextSearch } from './src/expressions/types';

packages/kbn-es-query/src/utils.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isCCSRemoteIndexName } from './utils';
+
+describe('util tests', () => {
+  describe('isCCSRemoteIndexName', () => {
+    it('should not validate empty string', () => {
+      expect(isCCSRemoteIndexName('')).toBe(false);
+    });
+
+    it('should not validate date math expression', () => {
+      expect(isCCSRemoteIndexName('<logstash-{now/d-2d}>')).toBe(false);
+    });
+
+    it('should not validate date math expression with negation', () => {
+      expect(isCCSRemoteIndexName('-<logstash-{now/d-2d}>')).toBe(false);
+    });
+
+    it('should not validate invalid prefix', () => {
+      expect(isCCSRemoteIndexName(':logstash-{now/d-2d}')).toBe(false);
+    });
+
+    it('should validate CCS pattern', () => {
+      expect(isCCSRemoteIndexName('*:logstash-{now/d-2d}')).toBe(true);
+    });
+  });
+});

packages/kbn-es-query/src/utils.ts

@@ -36,3 +36,22 @@ export function isDataViewFieldSubtypeMulti(field: HasSubtype) {
 export function getDataViewFieldSubtypeMulti(field: HasSubtype) {
   return isDataViewFieldSubtypeMulti(field) ? (field.subType as IFieldSubTypeMulti) : undefined;
 }
+
+/**
+ * Check whether the index expression represents a remote index (CCS) or not.
+ * The index name is assumed to be individual index (no commas) but can contain `-`, wildcards,
+ * datemath, remote cluster name and any other syntax permissible in index expression component.
+ *
+ * 2024/10/11 Implementation taken from https://github.com/smalyshev/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+ *
+ * @param indexExpression
+ */
+export function isCCSRemoteIndexName(indexExpression: string): boolean {
+  if (indexExpression === '' || indexExpression[0] === '<' || indexExpression.startsWith('-<')) {
+    // This is date math, but even if it is not, the remote can't start with '<'.
+    // Thus, whatever it is, this is definitely not a remote index.
+    return false;
+  }
+  // Note remote index name also can not start with ':'
+  return indexExpression.indexOf(':') > 0;
+}

packages/kbn-esql-ast/src/parser/__tests__/from.test.ts

@@ -187,5 +187,21 @@ describe('FROM', () => {
 
       expect(errors.length > 0).toBe(true);
     });
+
+    it('when open square bracket "[" is entered', () => {
+      const text = 'FROM kibana_sample_data_logs [';
+      const { errors } = parse(text);
+
+      expect(errors.length > 0).toBe(true);
+      expect(errors[0].message.toLowerCase().includes('metadata')).toBe(true);
+    });
+
+    it('when close square bracket "]" is entered', () => {
+      const text = 'FROM kibana_sample_data_logs []';
+      const { errors } = parse(text);
+
+      expect(errors.length > 0).toBe(true);
+      expect(errors[0].message.toLowerCase().includes('metadata')).toBe(true);
+    });
   });
 });

packages/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts

@@ -120,7 +120,7 @@ export class ESQLAstBuilderListener implements ESQLParserListener {
     const metadataContext = ctx.metadata();
     const metadataContent =
       metadataContext?.deprecated_metadata()?.metadataOption() || metadataContext?.metadataOption();
-    if (metadataContent) {
+    if (metadataContent && metadataContent.METADATA()) {
       const option = createOption(
         metadataContent.METADATA().getText().toLowerCase(),
         metadataContent

test/functional/apps/discover/group3/_doc_viewer.ts

@@ -625,7 +625,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           );
           expect(role).to.be('dialog');
           expect(tabindex).to.be('0');
-          expect(await find.existsByCssSelector(`#${describedBy}`)).to.be(true);
+          expect(await find.existsByCssSelector(`[id="${describedBy}"]`)).to.be(true);
           expect(noFocusLock).to.be('true');
           // overlay flyout
           await reduceScreenWidth();
@@ -635,7 +635,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           noFocusLock = await testSubjects.getAttribute('docViewerFlyout', 'data-no-focus-lock');
           expect(role).to.be('dialog');
           expect(tabindex).to.be('0');
-          expect(await find.existsByCssSelector(`#${describedBy}`)).to.be(true);
+          expect(await find.existsByCssSelector(`[id="${describedBy}"]`)).to.be(true);
           expect(noFocusLock).to.be(null);
         });
       });

x-pack/plugins/alerting/common/routes/framework/apis/health/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { healthFrameworkResponseSchema } from './schemas/latest';
+export type { HealthFrameworkResponse } from './types/latest';
+
+export {
+  healthFrameworkResponseBodySchema as healthFrameworkResponseBodySchemaV1,
+  healthFrameworkResponseSchema as healthFrameworkResponseSchemaV1,
+} from './schemas/v1';
+export type { HealthFrameworkResponseBody as HealthFrameworkResponseBodyV1 } from './types/v1';

x-pack/plugins/alerting/common/routes/framework/apis/health/schemas/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/plugins/alerting/common/routes/framework/apis/health/schemas/v1.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+const alertingFrameworkHealthSchemaObject = {
+  status: schema.oneOf([schema.literal('ok'), schema.literal('warn'), schema.literal('error')]),
+  timestamp: schema.string(),
+};
+
+export const healthFrameworkResponseBodySchema = schema.object({
+  is_sufficiently_secure: schema.boolean({
+    meta: {
+      description: 'If false, security is enabled but TLS is not.',
+    },
+  }),
+  has_permanent_encryption_key: schema.boolean({
+    meta: {
+      description: 'If false, the encryption key is not set',
+    },
+  }),
+  alerting_framework_health: schema.object(
+    {
+      decryption_health: schema.object(alertingFrameworkHealthSchemaObject, {
+        meta: { description: 'The timestamp and status of the alert decryption.' },
+      }),
+      execution_health: schema.object(alertingFrameworkHealthSchemaObject, {
+        meta: { description: 'The timestamp and status of the alert execution.' },
+      }),
+      read_health: schema.object(alertingFrameworkHealthSchemaObject, {
+        meta: { description: 'The timestamp and status of the alert reading events.' },
+      }),
+    },
+    {
+      meta: {
+        description:
+          'Three substates identify the health of the alerting framework: decryptionHealth, executionHealth, and readHealth.',
+      },
+    }
+  ),
+});
+
+export const healthFrameworkResponseSchema = schema.object({
+  body: healthFrameworkResponseBodySchema,
+});

x-pack/plugins/alerting/common/routes/framework/apis/health/types/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/plugins/alerting/common/routes/framework/apis/health/types/v1.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { TypeOf } from '@kbn/config-schema';
+
+import { healthFrameworkResponseBodySchemaV1, healthFrameworkResponseSchemaV1 } from '..';
+
+export type HealthFrameworkResponseBody = TypeOf<typeof healthFrameworkResponseBodySchemaV1>;
+export type HealthFrameworkResponse = TypeOf<typeof healthFrameworkResponseSchemaV1>;

x-pack/plugins/alerting/server/routes/health.test.ts → x-pack/plugins/alerting/server/routes/framework/apis/health/health.test.ts

@@ -8,18 +8,18 @@
 import { healthRoute } from './health';
 import { httpServiceMock } from '@kbn/core/server/mocks';
 import { HealthStatus } from '@kbn/alerting-types';
-import { mockHandlerArguments } from './_mock_handler_arguments';
-import { verifyApiAccess } from '../lib/license_api_access';
-import { licenseStateMock } from '../lib/license_state.mock';
+import { mockHandlerArguments } from '../../../_mock_handler_arguments';
+import { verifyApiAccess } from '../../../../lib/license_api_access';
+import { licenseStateMock } from '../../../../lib/license_state.mock';
 import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
-import { rulesClientMock } from '../rules_client.mock';
-import { alertsMock } from '../mocks';
-import { RecoveredActionGroup } from '../../common';
-import { RegistryAlertTypeWithAuth } from '../authorization';
+import { rulesClientMock } from '../../../../rules_client.mock';
+import { alertsMock } from '../../../../mocks';
+import { RecoveredActionGroup } from '../../../../../common';
+import { RegistryAlertTypeWithAuth } from '../../../../authorization';
 
 const rulesClient = rulesClientMock.create();
 
-jest.mock('../lib/license_api_access', () => ({
+jest.mock('../../../../lib/license_api_access', () => ({
   verifyApiAccess: jest.fn(),
 }));
 

x-pack/plugins/alerting/server/routes/health.ts → x-pack/plugins/alerting/server/routes/framework/apis/health/health.ts

@@ -7,30 +7,16 @@
 
 import { IRouter } from '@kbn/core/server';
 import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server';
-import { ILicenseState } from '../lib';
-import { RewriteResponseCase, verifyAccessAndContext } from './lib';
+import { healthFrameworkResponseSchemaV1 } from '../../../../../common/routes/framework/apis/health';
+import { ILicenseState } from '../../../../lib';
+import { verifyAccessAndContext } from '../../../lib';
 import {
   AlertingRequestHandlerContext,
   BASE_ALERTING_API_PATH,
   AlertingFrameworkHealth,
-} from '../types';
-import { getSecurityHealth } from '../lib/get_security_health';
-
-const rewriteBodyRes: RewriteResponseCase<AlertingFrameworkHealth> = ({
-  isSufficientlySecure,
-  hasPermanentEncryptionKey,
-  alertingFrameworkHealth,
-  ...rest
-}) => ({
-  ...rest,
-  is_sufficiently_secure: isSufficientlySecure,
-  has_permanent_encryption_key: hasPermanentEncryptionKey,
-  alerting_framework_health: {
-    decryption_health: alertingFrameworkHealth.decryptionHealth,
-    execution_health: alertingFrameworkHealth.executionHealth,
-    read_health: alertingFrameworkHealth.readHealth,
-  },
-});
+} from '../../../../types';
+import { getSecurityHealth } from '../../../../lib/get_security_health';
+import { transformHealthBodyResponse } from './transforms/transform_health_response/v1';
 
 export const healthRoute = (
   router: IRouter<AlertingRequestHandlerContext>,
@@ -45,7 +31,18 @@ export const healthRoute = (
         summary: `Get the alerting framework health`,
         tags: ['oas-tag:alerting'],
       },
-      validate: false,
+      validate: {
+        request: {},
+        response: {
+          200: {
+            body: () => healthFrameworkResponseSchemaV1,
+            description: 'Indicates a successful call.',
+          },
+          401: {
+            description: 'Authorization information is missing or invalid.',
+          },
+        },
+      },
     },
     router.handleLegacyErrors(
       verifyAccessAndContext(licenseState, async function (context, req, res) {
@@ -69,7 +66,7 @@ export const healthRoute = (
             };
 
             return res.ok({
-              body: rewriteBodyRes(frameworkHealth),
+              body: transformHealthBodyResponse(frameworkHealth),
             });
           } else {
             return res.forbidden({

x-pack/plugins/alerting/server/routes/framework/apis/health/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { healthRoute } from './health';

x-pack/plugins/alerting/server/routes/framework/apis/health/transforms/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { transformHealthBodyResponse } from './transform_health_response/latest';
+export { transformHealthBodyResponse as transformHealthBodyResponseV1 } from './transform_health_response/v1';

x-pack/plugins/alerting/server/routes/framework/apis/health/transforms/transform_health_response/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/plugins/alerting/server/routes/framework/apis/health/transforms/transform_health_response/v1.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AlertingFrameworkHealth } from '../../../../../../types';
+import type { HealthFrameworkResponseBodyV1 } from '../../../../../../../common/routes/framework/apis/health';
+
+export const transformHealthBodyResponse = (
+  frameworkHealth: AlertingFrameworkHealth
+): HealthFrameworkResponseBodyV1 => ({
+  is_sufficiently_secure: frameworkHealth.isSufficientlySecure,
+  has_permanent_encryption_key: frameworkHealth.hasPermanentEncryptionKey,
+  alerting_framework_health: {
+    decryption_health: frameworkHealth.alertingFrameworkHealth.decryptionHealth,
+    execution_health: frameworkHealth.alertingFrameworkHealth.executionHealth,
+    read_health: frameworkHealth.alertingFrameworkHealth.readHealth,
+  },
+});