Merge branch '7.17' of https://github.com/elastic/kibana into ua/rein…

…dex_error_handling

docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md

@@ -0,0 +1,21 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md)
+
+## mergeSavedObjectMigrationMaps variable
+
+Merges two saved object migration maps.
+
+If there is a migration for a given version on only one of the maps, that migration function will be used:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '4.5.6': g }<!-- -->) -<!-- -->&gt; { '1.2.3': f, '4.5.6': g }
+
+If there is a migration for a given version on both maps, the migrations will be composed:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '1.2.3': g }<!-- -->) -<!-- -->&gt; { '1.2.3': (doc, context) =<!-- -->&gt; f(g(doc, context), context) }
+
+<b>Signature:</b>
+
+```typescript
+mergeSavedObjectMigrationMaps: (map1: SavedObjectMigrationMap, map2: SavedObjectMigrationMap) => SavedObjectMigrationMap
+```

package.json

@@ -447,6 +447,7 @@
     "@emotion/babel-preset-css-prop": "^11.2.0",
     "@emotion/jest": "^11.3.0",
     "@istanbuljs/schema": "^0.1.2",
+    "@jest/console": "^26.6.2",
     "@jest/reporters": "^26.6.2",
     "@kbn/babel-code-parser": "link:bazel-bin/packages/kbn-babel-code-parser",
     "@kbn/babel-preset": "link:bazel-bin/packages/kbn-babel-preset",

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -11,16 +11,28 @@ import Os from 'os';
 import Fs from 'fs';
 import Path from 'path';
 import crypto from 'crypto';
+
 import execa from 'execa';
-import Axios from 'axios';
+import Axios, { AxiosRequestConfig } from 'axios';
 // @ts-expect-error not "public", but necessary to prevent Jest shimming from breaking things
 import httpAdapter from 'axios/lib/adapters/http';
 
 import { ToolingLog } from '../tooling_log';
 import { parseConfig, Config } from './ci_stats_config';
+import type { CiStatsTestGroupInfo, CiStatsTestRun } from './ci_stats_test_group_types';
 
 const BASE_URL = 'https://ci-stats.kibana.dev';
 
+/** Container for metadata that can be attached to different ci-stats objects */
+export interface CiStatsMetadata {
+  /**
+   * Arbitrary key-value pairs which can be attached to CiStatsTiming and CiStatsMetric
+   * objects stored in the ci-stats service
+   */
+  [key: string]: string | string[] | number | boolean | undefined;
+}
+
+/** A ci-stats metric record */
 export interface CiStatsMetric {
   /** Top-level categorization for the metric, e.g. "page load bundle size" */
   group: string;
@@ -40,13 +52,7 @@ export interface CiStatsMetric {
   meta?: CiStatsMetadata;
 }
 
-export interface CiStatsMetadata {
-  /**
-   * Arbitrary key-value pairs which can be attached to CiStatsTiming and CiStatsMetric
-   * objects stored in the ci-stats service
-   */
-  [key: string]: string | string[] | number | boolean | undefined;
-}
+/** A ci-stats timing event */
 export interface CiStatsTiming {
   /** Top-level categorization for the timing, e.g. "scripts/foo", process type, etc. */
   group: string;
@@ -58,13 +64,7 @@ export interface CiStatsTiming {
   meta?: CiStatsMetadata;
 }
 
-interface ReqOptions {
-  auth: boolean;
-  path: string;
-  body: any;
-  bodyDesc: string;
-}
-
+/** Options for reporting timings to ci-stats */
 export interface TimingsOptions {
   /** list of timings to record */
   timings: CiStatsTiming[];
@@ -74,10 +74,41 @@ export interface TimingsOptions {
   kibanaUuid?: string | null;
 }
 
+/** Options for reporting metrics to ci-stats */
 export interface MetricsOptions {
   /** Default metadata to add to each metric */
   defaultMeta?: CiStatsMetadata;
 }
+
+/** Options for reporting tests to ci-stats */
+export interface CiStatsReportTestsOptions {
+  /**
+   * Information about the group of tests that were run
+   */
+  group: CiStatsTestGroupInfo;
+  /**
+   * Information about each test that ran, including failure information
+   */
+  testRuns: CiStatsTestRun[];
+}
+
+/* @internal */
+interface ReportTestsResponse {
+  buildId: string;
+  groupId: string;
+  testRunCount: number;
+}
+
+/* @internal */
+interface ReqOptions {
+  auth: boolean;
+  path: string;
+  body: any;
+  bodyDesc: string;
+  query?: AxiosRequestConfig['params'];
+}
+
+/** Object that helps report data to the ci-stats service */
 export class CiStatsReporter {
   /**
    * Create a CiStatsReporter by inspecting the ENV for the necessary config
@@ -86,7 +117,7 @@ export class CiStatsReporter {
     return new CiStatsReporter(parseConfig(log), log);
   }
 
-  constructor(private config: Config | undefined, private log: ToolingLog) {}
+  constructor(private readonly config: Config | undefined, private readonly log: ToolingLog) {}
 
   /**
    * Determine if CI_STATS is explicitly disabled by the environment. To determine
@@ -165,7 +196,7 @@ export class CiStatsReporter {
 
     this.log.debug('CIStatsReporter committerHash: %s', defaultMeta.committerHash);
 
-    return await this.req({
+    return !!(await this.req({
       auth: !!buildId,
       path: '/v1/timings',
       body: {
@@ -175,7 +206,7 @@ export class CiStatsReporter {
         timings,
       },
       bodyDesc: timings.length === 1 ? `${timings.length} timing` : `${timings.length} timings`,
-    });
+    }));
   }
 
   /**
@@ -188,12 +219,11 @@ export class CiStatsReporter {
     }
 
     const buildId = this.config?.buildId;
-
     if (!buildId) {
-      throw new Error(`CiStatsReporter can't be authorized without a buildId`);
+      throw new Error(`metrics can't be reported without a buildId`);
     }
 
-    return await this.req({
+    return !!(await this.req({
       auth: true,
       path: '/v1/metrics',
       body: {
@@ -204,6 +234,30 @@ export class CiStatsReporter {
       bodyDesc: `metrics: ${metrics
         .map(({ group, id, value }) => `[${group}/${id}=${value}]`)
         .join(' ')}`,
+    }));
+  }
+
+  /**
+   * Send test reports to ci-stats
+   */
+  async reportTests({ group, testRuns }: CiStatsReportTestsOptions) {
+    if (!this.config?.buildId || !this.config?.apiToken) {
+      throw new Error(
+        'unable to report tests unless buildId is configured and auth config available'
+      );
+    }
+
+    return await this.req<ReportTestsResponse>({
+      auth: true,
+      path: '/v1/test_group',
+      query: {
+        buildId: this.config?.buildId,
+      },
+      bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`,
+      body: [
+        JSON.stringify({ group }),
+        ...testRuns.map((testRun) => JSON.stringify({ testRun })),
+      ].join('\n'),
     });
   }
 
@@ -241,7 +295,7 @@ export class CiStatsReporter {
     }
   }
 
-  private async req({ auth, body, bodyDesc, path }: ReqOptions) {
+  private async req<T>({ auth, body, bodyDesc, path, query }: ReqOptions) {
     let attempt = 0;
     const maxAttempts = 5;
 
@@ -251,23 +305,24 @@ export class CiStatsReporter {
         Authorization: `token ${this.config.apiToken}`,
       };
     } else if (auth) {
-      throw new Error('this.req() shouldnt be called with auth=true if this.config is defined');
+      throw new Error('this.req() shouldnt be called with auth=true if this.config is not defined');
     }
 
     while (true) {
       attempt += 1;
 
       try {
-        await Axios.request({
+        const resp = await Axios.request<T>({
           method: 'POST',
           url: path,
           baseURL: BASE_URL,
           headers,
           data: body,
+          params: query,
           adapter: httpAdapter,
         });
 
-        return true;
+        return resp.data;
       } catch (error) {
         if (!error?.request) {
           // not an axios error, must be a usage error that we should notify user about

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_test_group_types.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { CiStatsMetadata } from './ci_stats_reporter';
+
+export type CiStatsTestResult = 'fail' | 'pass' | 'skip';
+export type CiStatsTestType =
+  | 'after all hook'
+  | 'after each hook'
+  | 'before all hook'
+  | 'before each hook'
+  | 'test';
+
+export interface CiStatsTestRun {
+  /**
+   * ISO-8601 formatted datetime representing when the tests started running
+   */
+  startTime: string;
+  /**
+   * Duration of the tests in milliseconds
+   */
+  durationMs: number;
+  /**
+   * A sequence number, this is used to order the tests in a specific test run
+   */
+  seq: number;
+  /**
+   * The type of this "test run", usually this is just "test" but when reporting issues in hooks it can be set to the type of hook
+   */
+  type: CiStatsTestType;
+  /**
+   * "fail", "pass" or "skip", the result of the tests
+   */
+  result: CiStatsTestResult;
+  /**
+   * The list of suite names containing this test, the first being the outermost suite
+   */
+  suites: string[];
+  /**
+   * The name of this specific test run
+   */
+  name: string;
+  /**
+   * Relative path from the root of the repo contianing this test
+   */
+  file: string;
+  /**
+   * Error message if the test failed
+   */
+  error?: string;
+  /**
+   * Debug output/stdout produced by the test
+   */
+  stdout?: string;
+  /**
+   * Screenshots captured during the test run
+   */
+  screenshots?: Array<{
+    name: string;
+    base64Png: string;
+  }>;
+}
+
+export interface CiStatsTestGroupInfo {
+  /**
+   * ISO-8601 formatted datetime representing when the group of tests started running
+   */
+  startTime: string;
+  /**
+   * The number of miliseconds that the tests ran for
+   */
+  durationMs: number;
+  /**
+   * The type of tests run in this group, any value is valid but test groups are groupped by type in the UI so use something consistent
+   */
+  type: string;
+  /**
+   * The name of this specific group (within the "type")
+   */
+  name: string;
+  /**
+   * Arbitrary metadata associated with this group. We currently look for a ciGroup metadata property for highlighting that when appropriate
+   */
+  meta: CiStatsMetadata;
+}

packages/kbn-dev-utils/src/ci_stats_reporter/index.ts

@@ -10,3 +10,4 @@ export * from './ci_stats_reporter';
 export type { Config } from './ci_stats_config';
 export * from './ship_ci_stats_cli';
 export { getTimeReporter } from './report_time';
+export * from './ci_stats_test_group_types';