Merge branch 'kbn-104081-split-multiple-indices-logic' into kbn-10408…

…1-split-multiple-indices-relocate

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -76,19 +76,19 @@ buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image-aarch64.tar.g
 buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"
 cd -
 
+# This part is related with updating the configuration of kibana-controller,
+# so that new stack instances contain the latest and greatest image of kibana,
+# and the respective stack components of course.
 echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
-
   cat << EOF | buildkite-agent pipeline upload
 steps:
-  - trigger: k8s-gitops-update-image-tag
+  - trigger: serverless-gitops-update-stack-image-tag
     async: true
     label: ":argo: Update image tag for Kibana"
     branches: main
     build:
       env:
-        MODE: sed
-        TARGET_FILE: kibana-controller.yaml
         IMAGE_TAG: "git-$GIT_ABBREV_COMMIT"
         SERVICE: kibana-controller
         NAMESPACE: kibana-ci

.github/CODEOWNERS

@@ -459,6 +459,7 @@ x-pack/packages/ml/local_storage @elastic/ml-ui
 x-pack/packages/ml/nested_property @elastic/ml-ui
 x-pack/plugins/ml @elastic/ml-ui
 x-pack/packages/ml/query_utils @elastic/ml-ui
+x-pack/packages/ml/random_sampler_utils @elastic/ml-ui
 x-pack/packages/ml/route_utils @elastic/ml-ui
 x-pack/packages/ml/string_hash @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui

docs/settings/apm-settings.asciidoc

@@ -101,4 +101,7 @@ Matcher for all source map indices. Defaults to `apm-*`.
 `xpack.apm.autoCreateApmDataView` {ess-icon}::
 Set to `false` to disable the automatic creation of the APM data view when the APM app is opened. Defaults to `true`.
 
+`xpack.apm.latestAgentVersionsUrl` {ess-icon}::
+Specifies the URL of a self hosted file that contains latest agent versions. Defaults to `https://apm-agent-versions.elastic.co/versions.json`. Set to `''` to disable requesting latest agent versions.
+
 // end::general-apm-settings[]

package.json

@@ -474,6 +474,7 @@
     "@kbn/ml-nested-property": "link:x-pack/packages/ml/nested_property",
     "@kbn/ml-plugin": "link:x-pack/plugins/ml",
     "@kbn/ml-query-utils": "link:x-pack/packages/ml/query_utils",
+    "@kbn/ml-random-sampler-utils": "link:x-pack/packages/ml/random_sampler_utils",
     "@kbn/ml-route-utils": "link:x-pack/packages/ml/route_utils",
     "@kbn/ml-string-hash": "link:x-pack/packages/ml/string_hash",
     "@kbn/ml-trained-models-utils": "link:x-pack/packages/ml/trained_models_utils",

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts

@@ -20,17 +20,13 @@ import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
 import { lastValueFrom } from 'rxjs';
 import { runResilientMigrator } from './run_resilient_migrator';
 
-jest.mock('./run_resilient_migrator', () => ({
-  runResilientMigrator: jest.fn((options) => {
-    return {
-      status: 'migrated',
-      // TODO check the actual value of the MigrationResult for these fields
-      destIndex: `${options.indexPrefix}_8.2.3_001`,
-      sourceIndex: `${options.indexPrefix}_8.2.2_001`,
-      elapsedMs: 28,
-    };
-  }),
-}));
+jest.mock('./run_resilient_migrator', () => {
+  const actual = jest.requireActual('./run_resilient_migrator');
+
+  return {
+    runResilientMigrator: jest.fn(actual.runResilientMigrator),
+  };
+});
 
 jest.mock('./document_migrator', () => {
   return {
@@ -89,7 +85,7 @@ describe('KibanaMigrator', () => {
         },
         {
           name: 'bmap',
-          indexPattern: 'other-index',
+          indexPattern: '.other-index',
           mappings: {
             properties: { field: { type: 'text' } },
           },
@@ -134,8 +130,22 @@ describe('KibanaMigrator', () => {
       );
     });
 
-    // eslint-disable-next-line jest/no-focused-tests
-    fit('only runs migrations once if called multiple times', async () => {
+    it('only runs migrations once if called multiple times', async () => {
+      const successfulRun: typeof runResilientMigrator = ({ indexPrefix }) =>
+        Promise.resolve({
+          sourceIndex: indexPrefix,
+          destIndex: indexPrefix,
+          elapsedMs: 28,
+          status: 'migrated',
+        });
+      const mockRunResilientMigrator = runResilientMigrator as jest.MockedFunction<
+        typeof runResilientMigrator
+      >;
+
+      mockRunResilientMigrator.mockImplementationOnce(successfulRun);
+      mockRunResilientMigrator.mockImplementationOnce(successfulRun);
+      mockRunResilientMigrator.mockImplementationOnce(successfulRun);
+      mockRunResilientMigrator.mockImplementationOnce(successfulRun);
       const options = mockOptions();
       options.client.indices.get.mockResponse({}, { statusCode: 200 });
       options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, {
@@ -157,7 +167,35 @@ describe('KibanaMigrator', () => {
       await migrator.runMigrations();
 
       // indices.get is called twice during a single migration
-      expect(options.client.indices.get).toHaveBeenCalledTimes(2);
+      expect(runResilientMigrator).toHaveBeenCalledTimes(4);
+      expect(runResilientMigrator).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          indexPrefix: '.my-index',
+          mustRelocateDocuments: true,
+        })
+      );
+      expect(runResilientMigrator).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          indexPrefix: '.other-index',
+          mustRelocateDocuments: true,
+        })
+      );
+      expect(runResilientMigrator).toHaveBeenNthCalledWith(
+        3,
+        expect.objectContaining({
+          indexPrefix: '.my-task-index',
+          mustRelocateDocuments: false,
+        })
+      );
+      expect(runResilientMigrator).toHaveBeenNthCalledWith(
+        4,
+        expect.objectContaining({
+          indexPrefix: '.my-complementary-index',
+          mustRelocateDocuments: true,
+        })
+      );
     });
 
     it('emits results on getMigratorResult$()', async () => {
@@ -179,12 +217,12 @@ describe('KibanaMigrator', () => {
         status: 'migrated',
       });
       expect(result![1]).toMatchObject({
-        destIndex: 'other-index_8.2.3_001',
+        destIndex: '.other-index_8.2.3_001',
         elapsedMs: expect.any(Number),
         status: 'patched',
       });
     });
-    it('rejects when the migration state machine terminates in a FATAL state', () => {
+    it('rejects when the migration state machine terminates in a FATAL state', async () => {
       const options = mockV2MigrationOptions();
       options.client.indices.get.mockResponse(
         {
@@ -244,34 +282,37 @@ describe('KibanaMigrator', () => {
           const migrator = new KibanaMigrator(options);
           migrator.prepareMigrations();
           const results = await migrator.runMigrations();
-          expect(results).toMatchInlineSnapshot(`
-            Array [
-              Object {
-                "destIndex": ".my-index_8.2.3_001",
-                "elapsedMs": 28,
-                "sourceIndex": ".my-index_8.2.2_001",
-                "status": "migrated",
-              },
-              Object {
-                "destIndex": ".other-index_8.2.3_001",
-                "elapsedMs": 28,
-                "sourceIndex": ".other-index_8.2.2_001",
-                "status": "migrated",
-              },
-              Object {
-                "destIndex": ".my-task-index_8.2.3_001",
-                "elapsedMs": 28,
-                "sourceIndex": ".my-task-index_8.2.2_001",
-                "status": "migrated",
-              },
-              Object {
-                "destIndex": ".my-complementary-index_8.2.3_001",
-                "elapsedMs": 28,
-                "sourceIndex": ".my-complementary-index_8.2.2_001",
-                "status": "migrated",
-              },
-            ]
-          `);
+
+          expect(results.length).toEqual(4);
+          expect(results[0]).toEqual(
+            expect.objectContaining({
+              sourceIndex: '.my-index_pre8.2.3_001',
+              destIndex: '.my-index_8.2.3_001',
+              elapsedMs: expect.any(Number),
+              status: 'migrated',
+            })
+          );
+          expect(results[1]).toEqual(
+            expect.objectContaining({
+              destIndex: '.other-index_8.2.3_001',
+              elapsedMs: expect.any(Number),
+              status: 'patched',
+            })
+          );
+          expect(results[2]).toEqual(
+            expect.objectContaining({
+              destIndex: '.my-task-index_8.2.3_001',
+              elapsedMs: expect.any(Number),
+              status: 'patched',
+            })
+          );
+          expect(results[3]).toEqual(
+            expect.objectContaining({
+              destIndex: '.my-complementary-index_8.2.3_001',
+              elapsedMs: expect.any(Number),
+              status: 'patched',
+            })
+          );
 
           expect(runResilientMigrator).toHaveBeenCalledTimes(4);
           expect(runResilientMigrator).toHaveBeenNthCalledWith(
@@ -472,7 +513,7 @@ const mockOptions = () => {
         name: 'testtype2',
         hidden: false,
         namespaceType: 'single',
-        // We are moving 'testtype2' from '.my-index' to '.my-other-index'
+        // We are moving 'testtype2' from '.my-index' to '.other-index'
         indexPattern: '.other-index',
         mappings: {
           properties: {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts

@@ -142,7 +142,7 @@ export class KibanaMigrator implements IKibanaMigrator {
   private async runMigrationsInternal(): Promise<MigrationResult[]> {
     const migrationAlgorithm = this.soMigrationsConfig.algorithm;
     if (migrationAlgorithm === 'zdt') {
-      return this.runMigrationZdt();
+      return await this.runMigrationZdt();
     } else {
       return await this.runMigrationV2();
     }

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts

@@ -65,6 +65,7 @@ export function mergeMigrationMappingPropertyHashes(
   return {
     ...targetMappings,
     _meta: {
+      ...targetMappings._meta,
       migrationMappingPropertyHashes: {
         ...indexMappings._meta?.migrationMappingPropertyHashes,
         ...targetMappings._meta?.migrationMappingPropertyHashes,

packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts

@@ -141,6 +141,8 @@ export type ApmFields = Fields<{
       values: number[];
       counts: number[];
     };
+    'transaction.result': string;
+    'transaction.sampled': boolean;
     'service.environment': string;
     'service.framework.name': string;
     'service.framework.version': string;
@@ -164,8 +166,6 @@ export type ApmFields = Fields<{
     'span.self_time.sum.us': number;
     'span.subtype': string;
     'span.type': string;
-    'transaction.result': string;
-    'transaction.sampled': true;
     'span.links': Array<{
       trace: { id: string };
       span: { id: string };

packages/kbn-apm-synthtrace-client/src/lib/apm/transaction.ts

@@ -32,6 +32,7 @@ export class Transaction extends BaseSpan {
       error.fields['trace.id'] = this.fields['trace.id'];
       error.fields['transaction.id'] = this.fields['transaction.id'];
       error.fields['transaction.type'] = this.fields['transaction.type'];
+      error.fields['transaction.sampled'] = this.fields['transaction.sampled'];
     });
 
     return this;
@@ -43,6 +44,7 @@ export class Transaction extends BaseSpan {
       error.fields['transaction.id'] = this.fields['transaction.id'];
       error.fields['transaction.name'] = this.fields['transaction.name'];
       error.fields['transaction.type'] = this.fields['transaction.type'];
+      error.fields['transaction.sampled'] = this.fields['transaction.sampled'];
     });
 
     this._errors.push(...errors);
@@ -56,7 +58,10 @@ export class Transaction extends BaseSpan {
   }
 
   sample(sampled: boolean = true) {
-    this._sampled = sampled;
+    this._sampled = this.fields['transaction.sampled'] = sampled;
+    this._errors.forEach((error) => {
+      error.fields['transaction.sampled'] = sampled;
+    });
     return this;
   }
 

packages/kbn-doc-links/src/get_doc_links.ts

@@ -137,6 +137,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => {
       crawlerOverview: `${ENTERPRISE_SEARCH_DOCS}crawler.html`,
       deployTrainedModels: `${MACHINE_LEARNING_DOCS}ml-nlp-deploy-models.html`,
       documentLevelSecurity: `${ELASTICSEARCH_DOCS}document-level-security.html`,
+      elser: `${MACHINE_LEARNING_DOCS}ml-nlp-elser.html`,
       engines: `${ENTERPRISE_SEARCH_DOCS}engines.html`,
       ingestPipelines: `${ENTERPRISE_SEARCH_DOCS}ingest-pipelines.html`,
       languageAnalyzers: `${ELASTICSEARCH_DOCS}analysis-lang-analyzer.html`,

packages/kbn-doc-links/src/types.ts

@@ -122,6 +122,7 @@ export interface DocLinks {
     readonly crawlerOverview: string;
     readonly deployTrainedModels: string;
     readonly documentLevelSecurity: string;
+    readonly elser: string;
     readonly engines: string;
     readonly ingestPipelines: string;
     readonly languageAnalyzers: string;

packages/kbn-slo-schema/src/rest_specs/slo.ts

@@ -54,7 +54,7 @@ const getSLOParamsSchema = t.type({
 });
 
 const sortDirectionSchema = t.union([t.literal('asc'), t.literal('desc')]);
-const sortBySchema = t.union([t.literal('name'), t.literal('indicatorType')]);
+const sortBySchema = t.union([t.literal('creationTime'), t.literal('indicatorType')]);
 
 const findSLOParamsSchema = t.partial({
   query: t.partial({

packages/kbn-slo-schema/src/schema/indicators.ts

@@ -50,17 +50,13 @@ const apmTransactionErrorRateIndicatorSchema = t.type({
 const kqlCustomIndicatorTypeSchema = t.literal('sli.kql.custom');
 const kqlCustomIndicatorSchema = t.type({
   type: kqlCustomIndicatorTypeSchema,
-  params: t.intersection([
-    t.type({
-      index: t.string,
-      filter: t.string,
-      good: t.string,
-      total: t.string,
-    }),
-    t.partial({
-      timestampField: t.string,
-    }),
-  ]),
+  params: t.type({
+    index: t.string,
+    filter: t.string,
+    good: t.string,
+    total: t.string,
+    timestampField: t.string,
+  }),
 });
 
 const indicatorDataSchema = t.type({

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -57,7 +57,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
       Object {
         "action": "6cfc277ed3211639e37546ac625f4a68f2494215",
         "action_task_params": "5f419caba96dd8c77d0f94013e71d43890e3d5d6",
-        "alert": "1e4cd6941f1eb39c729c646e91fbfb9700de84b9",
+        "alert": "7bec97d7775a025ecf36a33baf17386b9e7b4c3c",
         "api_key_pending_invalidation": "16e7bcf8e78764102d7f525542d5b616809a21ee",
         "apm-indices": "d19dd7fb51f2d2cbc1f8769481721e0953f9a6d2",
         "apm-server-schema": "1d42f17eff9ec6c16d3a9324d9539e2d123d0a9a",