Merge branch '8.16' into backport/8.16/pr-196531

elastic · Oct 18, 2024 · cb15d93 · cb15d93
2 parents d040dc7 + 9ad35b8
commit cb15d93
Show file tree

Hide file tree

Showing 61 changed files with 643 additions and 389 deletions.
diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml
@@ -39,6 +39,9 @@ disabled:
   - x-pack/test/fleet_cypress/config.space_awareness.ts
   - x-pack/test/fleet_cypress/visual_config.ts
 
+  # Default http2 config to use for performance journeys
+  - x-pack/performance/configs/http2_config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -345,6 +348,7 @@ enabled:
   - x-pack/test/usage_collection/config.ts
   - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts
+  - x-pack/performance/journeys_e2e/ecommerce_dashboard_http2.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys_e2e/flight_dashboard.ts
   - x-pack/performance/journeys_e2e/login.ts
@@ -358,6 +362,7 @@ enabled:
   - x-pack/performance/journeys_e2e/web_logs_dashboard_esql.ts
   - x-pack/performance/journeys_e2e/web_logs_dashboard_dataview.ts
   - x-pack/performance/journeys_e2e/data_stress_test_lens.ts
+  - x-pack/performance/journeys_e2e/data_stress_test_lens_http2.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys_e2e/dashboard_listing_page.ts

diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc
@@ -237,6 +237,14 @@ you make the necessary updates after you upgrade to 8.15.0.
 The Uptime app is already hidden from Kibana when there is no recent Heartbeat data, but will be completely removed in 9.0.0. You should migrate to Synthetics as an alternative. For more details, refer to the {observability-guide}/uptime-intro.html[Uptime documentation].
 ====
 
+[discrete]
+.<<search-sessions,Search sessions>> are deprecated in 8.15.0 and will be removed in a future version.
+[%collapsible]
+====
+*Details* +
+Search sessions are now deprecated and will be removed in a future version. By default, queries that take longer than 10 minutes (the default for the advanced setting `search:timeout`) will be canceled. To allow queries to run longer, consider increasing `search:timeout` or setting it to `0` which will allow queries to continue running as long as a user is waiting on-screen for results.
+====
+
 [float]
 [[breaking-changes-8.15.0]]
 === Breaking changes

diff --git a/docs/upgrade-notes.asciidoc b/docs/upgrade-notes.asciidoc
@@ -1481,6 +1481,18 @@ The following rule action variables have been deprecated. Use the recommended va
 For more information, refer to ({kibana-pull}161136[#161136]).
 ====
 
+// Discover
+
+[discrete]
+[[deprecation-search-sessions]]
+.[Discover] <<search-sessions,Search sessions>> are deprecated in 8.15.0 and will be removed in a future version. (8.15)
+[%collapsible]
+====
+*Details* +
+Search sessions are now deprecated and will be removed in a future version. By default, queries that take longer than 10 minutes (the default for the advanced setting `search:timeout`) will be canceled. To allow queries to run longer, consider increasing `search:timeout` or setting it to `0` which will allow queries to continue running as long as a user is waiting on-screen for results.
+====
+
+
 // General settings
 
 [discrete]

diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/functions.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/functions.test.ts
@@ -260,7 +260,7 @@ describe('function validation', () => {
         // date
         await expectErrors('FROM a_index | EVAL TEST(NOW())', []);
         await expectErrors('FROM a_index | EVAL TEST(1.)', [
-          'Argument of [test] must be [date], found value [1] type [decimal]',
+          'Argument of [test] must be [date], found value [1.] type [decimal]',
         ]);
       });
     });

diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.ts
@@ -112,7 +112,7 @@ function validateFunctionLiteralArg(
           values: {
             name: astFunction.name,
             argType: argDef.type as string,
-            value: typeof actualArg.value === 'number' ? actualArg.value : String(actualArg.value),
+            value: actualArg.text,
             givenType: actualArg.literalType,
           },
           locations: actualArg.location,

diff --git a/packages/serverless/settings/observability_project/index.ts b/packages/serverless/settings/observability_project/index.ts
@@ -37,5 +37,4 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [
   settings.OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING,
   settings.OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN,
   settings.OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID,
-  settings.OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS,
 ];
diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -123,7 +123,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "5e95e539826a40ad08fd0c1d161da0a4d86ffc6d",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
         "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91",
-        "ingest-package-policies": "dc2af447c335215be2d6f7b7b8d437d05d6a1188",
+        "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22",
         "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
         "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",

diff --git a/src/dev/performance/run_performance_cli.ts b/src/dev/performance/run_performance_cli.ts
@@ -47,6 +47,7 @@ const journeyTargetGroups: JourneyTargetGroups = {
   maps: ['ecommerce_dashboard_map_only'],
   ml: ['aiops_log_rate_analysis', 'many_fields_transform', 'tsdb_logs_data_visualizer'],
   esql: ['many_fields_discover_esql', 'web_logs_dashboard_esql'],
+  http2: ['data_stress_test_lens_http2', 'ecommerce_dashboard_http2'],
 };
 
 const readFilesRecursively = (dir: string, callback: Function) => {

diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/cumulative_sum.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/cumulative_sum.ts
@@ -33,7 +33,7 @@ export const convertToCumulativeSumColumns = (
   // lens supports cumulative sum for count and sum as quick function
   // and everything else as formula
   if (subFunctionMetric.type !== 'count' && pipelineAgg.name !== 'sum') {
-    const metaValue = Number(meta?.replace(']', ''));
+    const metaValue = Number(meta?.replace(/\]/g, ''));
     formula = getPipelineSeriesFormula(metric, metrics, subFunctionMetric, {
       metaValue,
       reducedTimeRange,

diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/formula.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/formula.ts
@@ -66,7 +66,7 @@ const convertFormulaScriptForPercentileAggs = (
 ) => {
   variables.forEach((variable) => {
     const [_, meta] = variable?.field?.split('[') ?? [];
-    const metaValue = Number(meta?.replace(']', ''));
+    const metaValue = Number(meta?.replace(/\]/g, ''));
     if (!metaValue) {
       return;
     }
@@ -163,7 +163,7 @@ export const convertOtherAggsToFormulaColumn = (
   const metric = metrics[metrics.length - 1];
   const [fieldId, meta] = metric?.field?.split('[') ?? [];
   const subFunctionMetric = metrics.find(({ id }) => id === fieldId);
-  const metaValue = meta ? Number(meta?.replace(']', '')) : undefined;
+  const metaValue = meta ? Number(meta?.replace(/\]/g, '')) : undefined;
 
   if (!subFunctionMetric) {
     return null;

diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/parent_pipeline.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/convert/parent_pipeline.ts
@@ -235,7 +235,7 @@ const convertMovingAvgOrDerivativeToColumns = (
   if (!pipelineAgg) {
     return null;
   }
-  const metaValue = Number(meta?.replace(']', ''));
+  const metaValue = Number(meta?.replace(/\]/g, ''));
   const subMetricField = subFunctionMetric.field;
   const [nestedFieldId, _] = subMetricField?.split('[') ?? [];
   // support nested aggs with formula

diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/metrics/filter_ratio_formula.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/metrics/filter_ratio_formula.ts
@@ -14,7 +14,7 @@ import { addAdditionalArgs } from '.';
 import { AdditionalArgs } from '../../types';
 
 const escapeQuotes = (str: string) => {
-  return str?.replace(/'/g, "\\'");
+  return str?.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
 };
 
 const constructFilterRationFormula = (

diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/metrics/metrics_helpers.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/metrics/metrics_helpers.ts
@@ -113,7 +113,7 @@ export const getFormulaEquivalent = (
       }
 
       return getPipelineSeriesFormula(currentMetric, metrics, subFunctionMetric, {
-        metaValue: nestedMetaValue ? Number(nestedMetaValue?.replace(']', '')) : undefined,
+        metaValue: nestedMetaValue ? Number(nestedMetaValue?.replace(/\]/g, '')) : undefined,
         reducedTimeRange,
         timeShift,
       });

diff --git a/x-pack/packages/ml/data_grid/hooks/use_data_grid.tsx b/x-pack/packages/ml/data_grid/hooks/use_data_grid.tsx
@@ -14,7 +14,7 @@ import { ES_CLIENT_TOTAL_HITS_RELATION } from '@kbn/ml-query-utils';
 import { INDEX_STATUS } from '../lib/common';
 import type { ChartData } from '../lib/field_histograms';
 import { ColumnChart } from '../components/column_chart';
-import { COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLD, INIT_MAX_COLUMNS } from '../lib/common';
+import { MAX_ROW_COUNT, INIT_MAX_COLUMNS } from '../lib/common';
 import type {
   ChartsVisible,
   ColumnId,
@@ -62,6 +62,11 @@ export const useDataGrid = (
 
   const { rowCount, rowCountRelation } = rowCountInfo;
 
+  const setLimitedRowCountInfo = useCallback((info: RowCountInfo) => {
+    const limitedRowCount = Math.min(info.rowCount, MAX_ROW_COUNT);
+    setRowCountInfo({ rowCount: limitedRowCount, rowCountRelation: info.rowCountRelation });
+  }, []);
+
   const toggleChartVisibility = () => {
     if (chartsVisible !== undefined) {
       setChartsVisible(!chartsVisible);
@@ -161,10 +166,7 @@ export const useDataGrid = (
   // we decide whether to show or hide the charts by default.
   useEffect(() => {
     if (chartsVisible === undefined && rowCount > 0 && rowCountRelation !== undefined) {
-      setChartsVisible(
-        rowCount <= COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLD &&
-          rowCountRelation !== ES_CLIENT_TOTAL_HITS_RELATION.GTE
-      );
+      setChartsVisible(rowCountRelation !== ES_CLIENT_TOTAL_HITS_RELATION.GTE);
     }
   }, [chartsVisible, rowCount, rowCountRelation]);
 
@@ -189,7 +191,7 @@ export const useDataGrid = (
       setErrorMessage,
       setNoDataMessage,
       setPagination,
-      setRowCountInfo,
+      setRowCountInfo: setLimitedRowCountInfo,
       setSortingColumns,
       setStatus,
       setTableItems,

diff --git a/x-pack/packages/ml/data_grid/lib/common.ts b/x-pack/packages/ml/data_grid/lib/common.ts
@@ -42,9 +42,9 @@ import type { DataGridItem, IndexPagination, RenderCellValue } from './types';
 export const INIT_MAX_COLUMNS = 10;
 
 /**
- * The default threshold value for the number of rows at which the column chart visibility is set to true.
+ * The default maximum row count value, set to 10000 due to ES limitations.
  */
-export const COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLD = 10000;
+export const MAX_ROW_COUNT = 10000;
 
 /**
  * Enum for index status

diff --git a/x-pack/packages/ml/data_grid/lib/types.ts b/x-pack/packages/ml/data_grid/lib/types.ts
@@ -250,7 +250,7 @@ export interface UseDataGridReturnType {
   /**
    * Setter function for the row count info.
    */
-  setRowCountInfo: Dispatch<SetStateAction<RowCountInfo>>;
+  setRowCountInfo: (info: RowCountInfo) => void;
   /**
    * Setter function for the sorting columns.
    */

diff --git a/x-pack/performance/configs/http2_config.ts b/x-pack/performance/configs/http2_config.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FtrConfigProviderContext } from '@kbn/test';
+import { configureHTTP2 } from '@kbn/test-suites-src/common/configure_http2';
+
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xpackFunctionalConfig = await readConfigFile(
+    require.resolve('@kbn/test-suites-xpack/functional/config.base')
+  );
+
+  return configureHTTP2({
+    ...xpackFunctionalConfig.getAll(),
+  });
+}
diff --git a/x-pack/performance/journeys_e2e/data_stress_test_lens_http2.ts b/x-pack/performance/journeys_e2e/data_stress_test_lens_http2.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+
+export const journey = new Journey({
+  kbnArchives: ['test/functional/fixtures/kbn_archiver/stress_test'],
+  esArchives: ['test/functional/fixtures/es_archiver/stress_test'],
+  ftrConfigPath: 'x-pack/performance/configs/http2_config.ts',
+}).step('Go to dashboard', async ({ page, kbnUrl, kibanaServer, kibanaPage }) => {
+  await kibanaServer.uiSettings.update({ 'histogram:maxBars': 100 });
+  await page.goto(kbnUrl.get(`/app/dashboards#/view/92b143a0-2e9c-11ed-b1b6-a504560b392c`));
+  await kibanaPage.waitForVisualizations({ count: 1 });
+});
diff --git a/x-pack/performance/journeys_e2e/ecommerce_dashboard_http2.ts b/x-pack/performance/journeys_e2e/ecommerce_dashboard_http2.ts
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+import { subj } from '@kbn/test-subj-selector';
+
+export const journey = new Journey({
+  esArchives: ['x-pack/performance/es_archives/sample_data_ecommerce'],
+  kbnArchives: ['x-pack/performance/kbn_archives/ecommerce_no_map_dashboard'],
+  ftrConfigPath: 'x-pack/performance/configs/http2_config.ts',
+})
+
+  .step('Go to Dashboards Page', async ({ page, kbnUrl, kibanaPage }) => {
+    await page.goto(kbnUrl.get(`/app/dashboards`));
+    await kibanaPage.waitForListViewTable();
+  })
+
+  .step('Go to Ecommerce Dashboard', async ({ page, kibanaPage }) => {
+    await page.click(subj('dashboardListingTitleLink-[eCommerce]-Revenue-Dashboard'));
+    await kibanaPage.waitForVisualizations({ count: 13 });
+  });
diff --git a/x-pack/performance/tsconfig.json b/x-pack/performance/tsconfig.json
@@ -22,5 +22,6 @@
     "@kbn/expect",
     "@kbn/dev-utils",
     "@kbn/apm-synthtrace-client",
+    "@kbn/test-suites-src",
   ]
 }
diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -101,7 +101,6 @@ import {
   migratePackagePolicySetRequiresRootToV8150,
 } from './migrations/to_v8_15_0';
 import { backfillAgentPolicyToV4 } from './model_versions/agent_policy_v4';
-import { packagePolicyV15AdvancedFieldsForEndpointV816 } from './model_versions/security_solution/v15_advanced_package_policy_fields';
 
 /*
  * Saved object types and mappings
@@ -751,14 +750,6 @@ export const getSavedObjectTypes = (
             },
           ],
         },
-        '15': {
-          changes: [
-            {
-              type: 'data_backfill',
-              backfillFn: packagePolicyV15AdvancedFieldsForEndpointV816,
-            },
-          ],
-        },
       },
       migrations: {
         '7.10.0': migratePackagePolicyToV7100,