Merge remote-tracking branch 'upstream/master' into Diana/57813-vega

package.json

@@ -87,6 +87,7 @@
     "**/@types/hoist-non-react-statics": "^3.3.1",
     "**/@types/chai": "^4.2.11",
     "**/cypress/@types/lodash": "^4.14.155",
+    "**/cypress/lodash": "^4.15.19",
     "**/typescript": "3.9.5",
     "**/graphql-toolkit/lodash": "^4.17.15",
     "**/hoist-non-react-statics": "^3.3.2",

src/plugins/data/common/field_formats/converters/url.ts

@@ -30,7 +30,7 @@ import {
 } from '../types';
 
 const templateMatchRE = /{{([\s\S]+?)}}/g;
-const whitelistUrlSchemes = ['http://', 'https://'];
+const allowedUrlSchemes = ['http://', 'https://'];
 
 const URL_TYPES = [
   {
@@ -161,7 +161,7 @@ export class UrlFormat extends FieldFormat {
 
         return this.generateImgHtml(url, imageLabel);
       default:
-        const inWhitelist = whitelistUrlSchemes.some((scheme) => url.indexOf(scheme) === 0);
+        const inWhitelist = allowedUrlSchemes.some((scheme) => url.indexOf(scheme) === 0);
         if (!inWhitelist && !parsedUrl) {
           return url;
         }

src/plugins/kibana_utils/public/state_management/url/hash_unhash_url.test.ts

@@ -116,7 +116,7 @@ describe('hash unhash url', () => {
         expect(mockStorage.length).toBe(3);
       });
 
-      it('hashes only whitelisted properties', () => {
+      it('hashes only allow-listed properties', () => {
         const stateParamKey1 = '_g';
         const stateParamValue1 = '(yes:!t)';
         const stateParamKey2 = '_a';
@@ -227,7 +227,7 @@ describe('hash unhash url', () => {
         );
       });
 
-      it('unhashes only whitelisted properties', () => {
+      it('un-hashes only allow-listed properties', () => {
         const stateParamKey1 = '_g';
         const stateParamValueHashed1 = 'h@4e60e02';
         const state1 = { yes: true };

src/plugins/kibana_utils/public/state_management/url/hash_unhash_url.ts

@@ -35,7 +35,7 @@ export const hashUrl = createQueryReplacer(hashQuery);
 
 // naive hack, but this allows to decouple these utils from AppState, GlobalState for now
 // when removing AppState, GlobalState and migrating to IState containers,
-// need to make sure that apps explicitly passing this whitelist to hash
+// need to make sure that apps explicitly passing this allow-list to hash
 const __HACK_HARDCODED_LEGACY_HASHABLE_PARAMS = ['_g', '_a', '_s'];
 function createQueryMapper(queryParamMapper: (q: string) => string | null) {
   return (

test/functional/apps/discover/_doc_navigation.js

@@ -29,7 +29,7 @@ export default function ({ getService, getPageObjects }) {
   const retry = getService('retry');
 
   // Flaky: https://github.com/elastic/kibana/issues/71216
-  describe.skip('doc link in discover', function contextSize() {
+  describe('doc link in discover', function contextSize() {
     beforeEach(async function () {
       log.debug('load kibana index with default index pattern');
       await esArchiver.loadIfNeeded('discover');
@@ -63,20 +63,28 @@ export default function ({ getService, getPageObjects }) {
       await filterBar.addFilter('agent', 'is', 'Missing/Fields');
       await PageObjects.discover.waitUntilSearchingHasFinished();
 
-      // navigate to the doc view
-      await docTable.clickRowToggle({ rowIndex: 0 });
+      await retry.try(async () => {
+        // navigate to the doc view
+        await docTable.clickRowToggle({ rowIndex: 0 });
 
-      const details = await docTable.getDetailsRow();
-      await docTable.addInclusiveFilter(details, 'referer');
-      await PageObjects.discover.waitUntilSearchingHasFinished();
+        const details = await docTable.getDetailsRow();
+        await docTable.addInclusiveFilter(details, 'referer');
+        await PageObjects.discover.waitUntilSearchingHasFinished();
 
-      const hasInclusiveFilter = await filterBar.hasFilter('referer', 'exists', true, false, true);
-      expect(hasInclusiveFilter).to.be(true);
+        const hasInclusiveFilter = await filterBar.hasFilter(
+          'referer',
+          'exists',
+          true,
+          false,
+          true
+        );
+        expect(hasInclusiveFilter).to.be(true);
 
-      await docTable.removeInclusiveFilter(details, 'referer');
-      await PageObjects.discover.waitUntilSearchingHasFinished();
-      const hasExcludeFilter = await filterBar.hasFilter('referer', 'exists', true, false, false);
-      expect(hasExcludeFilter).to.be(true);
+        await docTable.removeInclusiveFilter(details, 'referer');
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        const hasExcludeFilter = await filterBar.hasFilter('referer', 'exists', true, false, false);
+        expect(hasExcludeFilter).to.be(true);
+      });
     });
   });
 }

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts

@@ -23,6 +23,7 @@ interface Aggregation {
     buckets: Array<{
       aggregatedValue: { value: number; values?: Array<{ key: number; value: number }> };
       doc_count: number;
+      key_as_string: string;
     }>;
   };
 }
@@ -57,17 +58,18 @@ export const evaluateAlert = (
       );
       const { threshold, comparator } = criterion;
       const comparisonFunction = comparatorMap[comparator];
-      return mapValues(currentValues, (values: number | number[] | null) => {
-        if (isTooManyBucketsPreviewException(values)) throw values;
+      return mapValues(currentValues, (points: any[] | typeof NaN | null) => {
+        if (isTooManyBucketsPreviewException(points)) throw points;
         return {
           ...criterion,
           metric: criterion.metric ?? DOCUMENT_COUNT_I18N,
-          currentValue: Array.isArray(values) ? last(values) : NaN,
-          shouldFire: Array.isArray(values)
-            ? values.map((value) => comparisonFunction(value, threshold))
+          currentValue: Array.isArray(points) ? last(points)?.value : NaN,
+          timestamp: Array.isArray(points) ? last(points)?.key : NaN,
+          shouldFire: Array.isArray(points)
+            ? points.map((point) => comparisonFunction(point.value, threshold))
             : [false],
-          isNoData: values === null,
-          isError: isNaN(values),
+          isNoData: points === null,
+          isError: isNaN(points),
         };
       });
     })
@@ -157,17 +159,20 @@ const getValuesFromAggregations = (
     const { buckets } = aggregations.aggregatedIntervals;
     if (!buckets.length) return null; // No Data state
     if (aggType === Aggregators.COUNT) {
-      return buckets.map((bucket) => bucket.doc_count);
+      return buckets.map((bucket) => ({ key: bucket.key_as_string, value: bucket.doc_count }));
     }
     if (aggType === Aggregators.P95 || aggType === Aggregators.P99) {
       return buckets.map((bucket) => {
         const values = bucket.aggregatedValue?.values || [];
         const firstValue = first(values);
         if (!firstValue) return null;
-        return firstValue.value;
+        return { key: bucket.key_as_string, value: firstValue.value };
       });
     }
-    return buckets.map((bucket) => bucket.aggregatedValue.value);
+    return buckets.map((bucket) => ({
+      key: bucket.key_as_string,
+      value: bucket.aggregatedValue.value,
+    }));
   } catch (e) {
     return NaN; // Error state
   }

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts

@@ -56,4 +56,26 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
       );
     });
   });
+
+  describe('handles time', () => {
+    const end = new Date('2020-07-08T22:07:27.235Z').valueOf();
+    const timerange = {
+      end,
+      start: end - 5 * 60 * 1000,
+    };
+    const searchBody = getElasticsearchMetricQuery(
+      expressionParams,
+      timefield,
+      undefined,
+      undefined,
+      timerange
+    );
+    test('by rounding timestamps to the nearest timeUnit', () => {
+      const rangeFilter = searchBody.query.bool.filter.find((filter) =>
+        filter.hasOwnProperty('range')
+      )?.range[timefield];
+      expect(rangeFilter?.lte).toBe(1594246020000);
+      expect(rangeFilter?.gte).toBe(1594245720000);
+    });
+  });
 });

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -3,9 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 import { networkTraffic } from '../../../../../common/inventory_models/shared/metrics/snapshot/network_traffic';
 import { MetricExpressionParams, Aggregators } from '../types';
 import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds';
+import { roundTimestamp } from '../../../../utils/round_timestamp';
 import { getDateHistogramOffset } from '../../../snapshot/query_helpers';
 import { createPercentileAggregation } from './create_percentile_aggregation';
 
@@ -34,12 +36,15 @@ export const getElasticsearchMetricQuery = (
   const interval = `${timeSize}${timeUnit}`;
   const intervalAsSeconds = getIntervalInSeconds(interval);
 
-  const to = timeframe ? timeframe.end : Date.now();
+  const to = roundTimestamp(timeframe ? timeframe.end : Date.now(), timeUnit);
   // We need enough data for 5 buckets worth of data. We also need
   // to convert the intervalAsSeconds to milliseconds.
   const minimumFrom = to - intervalAsSeconds * 1000 * MINIMUM_BUCKETS;
 
-  const from = timeframe && timeframe.start <= minimumFrom ? timeframe.start : minimumFrom;
+  const from = roundTimestamp(
+    timeframe && timeframe.start <= minimumFrom ? timeframe.start : minimumFrom,
+    timeUnit
+  );
 
   const offset = getDateHistogramOffset(from, interval);
 

x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts

@@ -94,12 +94,14 @@ describe('The metric threshold alert type', () => {
       expect(getState(instanceID).alertState).toBe(AlertStates.OK);
     });
     test('reports expected values to the action context', async () => {
+      const now = 1577858400000;
       await execute(Comparator.GT, [0.75]);
       const { action } = mostRecentAction(instanceID);
       expect(action.group).toBe('*');
       expect(action.reason).toContain('current value is 1');
       expect(action.reason).toContain('threshold of 0.75');
       expect(action.reason).toContain('test.metric.1');
+      expect(action.timestamp).toBe(new Date(now).toISOString());
     });
   });
 

x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts

@@ -76,11 +76,13 @@ export const createMetricThresholdExecutor = (libs: InfraBackendLibs, alertId: s
         }
       }
       if (reason) {
+        const firstResult = first(alertResults);
+        const timestamp = (firstResult && firstResult[group].timestamp) ?? moment().toISOString();
         alertInstance.scheduleActions(FIRED_ACTIONS.id, {
           group,
           alertState: stateToAlertMessage[nextState],
           reason,
-          timestamp: moment().toISOString(),
+          timestamp,
           value: mapToConditionsLookup(alertResults, (result) => result[group].currentValue),
           threshold: mapToConditionsLookup(criteria, (c) => c.threshold),
           metric: mapToConditionsLookup(criteria, (c) => c.metric),

x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts

@@ -12,6 +12,7 @@ const bucketsA = [
   {
     doc_count: 3,
     aggregatedValue: { value: 1.0, values: [{ key: 95.0, value: 1.0 }] },
+    key_as_string: new Date(1577858400000).toISOString(),
   },
 ];
 

x-pack/plugins/infra/server/utils/round_timestamp.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Unit } from '@elastic/datemath';
+import moment from 'moment';
+
+export const roundTimestamp = (timestamp: number, unit: Unit) => {
+  const floor = moment(timestamp).startOf(unit).valueOf();
+  const ceil = moment(timestamp).add(1, unit).startOf(unit).valueOf();
+  if (Math.abs(timestamp - floor) <= Math.abs(timestamp - ceil)) return floor;
+  return ceil;
+};

x-pack/plugins/ingest_manager/common/mocks.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { NewPackageConfig, PackageConfig } from './types/models/package_config';
+
+export const createNewPackageConfigMock = () => {
+  return {
+    name: 'endpoint-1',
+    description: '',
+    namespace: 'default',
+    enabled: true,
+    config_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
+    output_id: '',
+    package: {
+      name: 'endpoint',
+      title: 'Elastic Endpoint',
+      version: '0.9.0',
+    },
+    inputs: [],
+  } as NewPackageConfig;
+};
+
+export const createPackageConfigMock = () => {
+  const newPackageConfig = createNewPackageConfigMock();
+  return {
+    ...newPackageConfig,
+    id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1',
+    version: 'abcd',
+    revision: 1,
+    updated_at: '2020-06-25T16:03:38.159292',
+    updated_by: 'kibana',
+    created_at: '2020-06-25T16:03:38.159292',
+    created_by: 'kibana',
+    inputs: [
+      {
+        config: {},
+      },
+    ],
+  } as PackageConfig;
+};

x-pack/plugins/security_solution/common/types/timeline/index.ts

@@ -4,12 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-/* eslint-disable @typescript-eslint/no-empty-interface */
+/* eslint-disable @typescript-eslint/camelcase, @typescript-eslint/no-empty-interface */
 
 import * as runtimeTypes from 'io-ts';
 import { SavedObjectsClient } from 'kibana/server';
 
-import { unionWithNullType } from '../../utility_types';
+import { stringEnum, unionWithNullType } from '../../utility_types';
 import { NoteSavedObject, NoteSavedObjectToReturnRuntimeType } from './note';
 import { PinnedEventToReturnSavedObjectRuntimeType, PinnedEventSavedObject } from './pinned_event';
 
@@ -164,6 +164,24 @@ export type TimelineStatusLiteralWithNull = runtimeTypes.TypeOf<
   typeof TimelineStatusLiteralWithNullRt
 >;
 
+export enum RowRendererId {
+  auditd = 'auditd',
+  auditd_file = 'auditd_file',
+  netflow = 'netflow',
+  plain = 'plain',
+  suricata = 'suricata',
+  system = 'system',
+  system_dns = 'system_dns',
+  system_endgame_process = 'system_endgame_process',
+  system_file = 'system_file',
+  system_fim = 'system_fim',
+  system_security_event = 'system_security_event',
+  system_socket = 'system_socket',
+  zeek = 'zeek',
+}
+
+export const RowRendererIdRuntimeType = stringEnum(RowRendererId, 'RowRendererId');
+
 /**
  * Timeline template type
  */
@@ -211,6 +229,7 @@ export const SavedTimelineRuntimeType = runtimeTypes.partial({
   dataProviders: unionWithNullType(runtimeTypes.array(SavedDataProviderRuntimeType)),
   description: unionWithNullType(runtimeTypes.string),
   eventType: unionWithNullType(runtimeTypes.string),
+  excludedRowRendererIds: unionWithNullType(runtimeTypes.array(RowRendererIdRuntimeType)),
   favorite: unionWithNullType(runtimeTypes.array(SavedFavoriteRuntimeType)),
   filters: unionWithNullType(runtimeTypes.array(SavedFilterRuntimeType)),
   kqlMode: unionWithNullType(runtimeTypes.string),

x-pack/plugins/security_solution/common/utility_types.ts

@@ -15,3 +15,14 @@ export interface DescriptionList {
 
 export const unionWithNullType = <T extends runtimeTypes.Mixed>(type: T) =>
   runtimeTypes.union([type, runtimeTypes.null]);
+
+export const stringEnum = <T>(enumObj: T, enumName = 'enum') =>
+  new runtimeTypes.Type<T[keyof T], string>(
+    enumName,
+    (u): u is T[keyof T] => Object.values(enumObj).includes(u),
+    (u, c) =>
+      Object.values(enumObj).includes(u)
+        ? runtimeTypes.success(u as T[keyof T])
+        : runtimeTypes.failure(u, c),
+    (a) => (a as unknown) as string
+  );

x-pack/plugins/security_solution/cypress/screens/hosts/events.ts

@@ -7,7 +7,7 @@
 export const CLOSE_MODAL = '[data-test-subj="modal-inspect-close"]';
 
 export const EVENTS_VIEWER_FIELDS_BUTTON =
-  '[data-test-subj="events-viewer-panel"] [data-test-subj="show-field-browser-gear"]';
+  '[data-test-subj="events-viewer-panel"] [data-test-subj="show-field-browser"]';
 
 export const EVENTS_VIEWER_PANEL = '[data-test-subj="events-viewer-panel"]';