Merge branch 'main' into fix/x-pack-test-types

.github/CODEOWNERS

@@ -502,6 +502,7 @@ x-pack/plugins/metrics_data_access @elastic/infra-monitoring-ui
 x-pack/packages/ml/agg_utils @elastic/ml-ui
 x-pack/packages/ml/anomaly_utils @elastic/ml-ui
 x-pack/packages/ml/category_validator @elastic/ml-ui
+x-pack/packages/ml/chi2test @elastic/ml-ui
 x-pack/packages/ml/data_frame_analytics_utils @elastic/ml-ui
 x-pack/packages/ml/data_grid @elastic/ml-ui
 x-pack/packages/ml/date_picker @elastic/ml-ui

config/serverless.oblt.yml

@@ -12,7 +12,6 @@ xpack.legacy_uptime.enabled: false
 xpack.serverless.observability.enabled: true
 
 ## Configure plugins
-xpack.infra.logs.app_target: discover
 
 ## Set the home route
 uiSettings.overrides.defaultRoute: /app/observability/landing

package.json

@@ -523,6 +523,7 @@
     "@kbn/ml-agg-utils": "link:x-pack/packages/ml/agg_utils",
     "@kbn/ml-anomaly-utils": "link:x-pack/packages/ml/anomaly_utils",
     "@kbn/ml-category-validator": "link:x-pack/packages/ml/category_validator",
+    "@kbn/ml-chi2test": "link:x-pack/packages/ml/chi2test",
     "@kbn/ml-data-frame-analytics-utils": "link:x-pack/packages/ml/data_frame_analytics_utils",
     "@kbn/ml-data-grid": "link:x-pack/packages/ml/data_grid",
     "@kbn/ml-date-picker": "link:x-pack/packages/ml/date_picker",

packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts

@@ -33,6 +33,11 @@ describe('sql query helpers', () => {
       expect(flag).toBe(false);
     });
 
+    it('should return false for an undefined query', () => {
+      const flag = isOfAggregateQueryType(undefined);
+      expect(flag).toBe(false);
+    });
+
     it('should return true for an Aggregate type query', () => {
       const flag = isOfAggregateQueryType({ sql: 'SELECT * FROM foo' });
       expect(flag).toBe(true);

packages/kbn-es-query/src/es_query/es_aggregate_query.ts

@@ -18,7 +18,7 @@ export function isOfQueryType(arg?: Query | AggregateQuery): arg is Query {
 // currently only supports the sql query type
 // should be enhanced to support other query types
 export function isOfAggregateQueryType(
-  query: AggregateQuery | Query | { [key: string]: any }
+  query?: AggregateQuery | Query | { [key: string]: any }
 ): query is AggregateQuery {
   return Boolean(query && ('sql' in query || 'esql' in query));
 }

src/plugins/dashboard/public/dashboard_actions/add_to_library_action.test.tsx

@@ -21,6 +21,7 @@ import {
   CONTACT_CARD_EMBEDDABLE,
 } from '@kbn/embeddable-plugin/public/lib/test_samples/embeddables';
 import { embeddablePluginMock } from '@kbn/embeddable-plugin/public/mocks';
+import { type Query, type AggregateQuery, Filter } from '@kbn/es-query';
 
 import { buildMockDashboard } from '../mocks';
 import { pluginServices } from '../services/plugin_services';
@@ -82,6 +83,18 @@ test('Add to library is incompatible with Error Embeddables', async () => {
   expect(await action.isCompatible({ embeddable: errorEmbeddable })).toBe(false);
 });
 
+test('Add to library is incompatible with ES|QL Embeddables', async () => {
+  const action = new AddToLibraryAction();
+  const mockGetFilters = jest.fn(async () => [] as Filter[]);
+  const mockGetQuery = jest.fn(async () => undefined as Query | AggregateQuery | undefined);
+  const filterableEmbeddable = embeddablePluginMock.mockFilterableEmbeddable(embeddable, {
+    getFilters: () => mockGetFilters(),
+    getQuery: () => mockGetQuery(),
+  });
+  mockGetQuery.mockResolvedValue({ esql: 'from logstash-* | limit 10' } as AggregateQuery);
+  expect(await action.isCompatible({ embeddable: filterableEmbeddable })).toBe(false);
+});
+
 test('Add to library is incompatible on visualize embeddable without visualize save permissions', async () => {
   pluginServices.getServices().application.capabilities = {
     ...defaultCapabilities,

src/plugins/dashboard/public/dashboard_actions/add_to_library_action.tsx

@@ -14,9 +14,10 @@ import {
   PanelNotFoundError,
   type EmbeddableInput,
   isReferenceOrValueEmbeddable,
+  isFilterableEmbeddable,
 } from '@kbn/embeddable-plugin/public';
 import { Action, IncompatibleActionError } from '@kbn/ui-actions-plugin/public';
-
+import { type AggregateQuery } from '@kbn/es-query';
 import { DashboardPanelState } from '../../common';
 import { pluginServices } from '../services/plugin_services';
 import { dashboardAddToLibraryActionStrings } from './_dashboard_actions_strings';
@@ -61,7 +62,10 @@ export class AddToLibraryAction implements Action<AddToLibraryActionContext> {
     // TODO: Fix this, potentially by adding a 'canSave' function to embeddable interface
     const { maps, visualize } = this.applicationCapabilities;
     const canSave = embeddable.type === 'map' ? maps.save : visualize.save;
-
+    const { isOfAggregateQueryType } = await import('@kbn/es-query');
+    const query = isFilterableEmbeddable(embeddable) && (await embeddable.getQuery());
+    // Textbased panels (i.e. ES|QL, SQL) should not save to library
+    const isTextBasedEmbeddable = isOfAggregateQueryType(query as AggregateQuery);
     return Boolean(
       canSave &&
         !isErrorEmbeddable(embeddable) &&
@@ -70,7 +74,8 @@ export class AddToLibraryAction implements Action<AddToLibraryActionContext> {
         embeddable.getRoot().isContainer &&
         embeddable.getRoot().type === DASHBOARD_CONTAINER_TYPE &&
         isReferenceOrValueEmbeddable(embeddable) &&
-        !embeddable.inputIsRefType(embeddable.getInput())
+        !embeddable.inputIsRefType(embeddable.getInput()) &&
+        !isTextBasedEmbeddable
     );
   }
 

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -250,11 +250,6 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.index_management.dev.enableIndexDetailsPage (boolean)',
         'xpack.index_management.enableIndexStats (any)',
         'xpack.infra.sources.default.fields.message (array)',
-        /**
-         * xpack.infra.logs is conditional and will resolve to an object of properties
-         * - xpack.infra.logs.app_target (string)
-         */
-        'xpack.infra.logs (any)',
         'xpack.license_management.ui.enabled (boolean)',
         'xpack.maps.preserveDrawingBuffer (boolean)',
         'xpack.maps.showMapsInspectorAdapter (boolean)',

tsconfig.base.json

@@ -998,6 +998,8 @@
       "@kbn/ml-anomaly-utils/*": ["x-pack/packages/ml/anomaly_utils/*"],
       "@kbn/ml-category-validator": ["x-pack/packages/ml/category_validator"],
       "@kbn/ml-category-validator/*": ["x-pack/packages/ml/category_validator/*"],
+      "@kbn/ml-chi2test": ["x-pack/packages/ml/chi2test"],
+      "@kbn/ml-chi2test/*": ["x-pack/packages/ml/chi2test/*"],
       "@kbn/ml-data-frame-analytics-utils": ["x-pack/packages/ml/data_frame_analytics_utils"],
       "@kbn/ml-data-frame-analytics-utils/*": ["x-pack/packages/ml/data_frame_analytics_utils/*"],
       "@kbn/ml-data-grid": ["x-pack/packages/ml/data_grid"],

x-pack/packages/ml/chi2test/README.md

@@ -0,0 +1,4 @@
+# @kbn/ml-chi2test
+
+`computeChi2PValue` computes the p-value for how similar the datasets are.
+Returned value ranges from 0 to 1, with 1 meaning the datasets are identical.

x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.test.ts → x-pack/packages/ml/chi2test/compute_chi_2_pvalue.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { computeChi2PValue } from './data_drift_utils';
-import { Histogram } from './types';
+import { computeChi2PValue } from './compute_chi_2_pvalue';
+import type { Histogram } from './types';
 
 describe('computeChi2PValue()', () => {
   test('should return close to 1 if datasets are both empty or nearly identical', () => {
@@ -83,6 +83,6 @@ describe('computeChi2PValue()', () => {
         percentage: 1,
       },
     ];
-    expect(computeChi2PValue(referenceTerms, comparisonTerms)).toStrictEqual(0);
+    expect(computeChi2PValue(referenceTerms, comparisonTerms)).toStrictEqual(0.000001);
   });
 });

x-pack/packages/ml/chi2test/compute_chi_2_pvalue.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { criticalTableLookup } from './critical_table_lookup';
+import type { Histogram } from './types';
+
+/**
+ * Compute the p-value for how similar the datasets are.
+ * Returned value ranges from 0 to 1, with 1 meaning the datasets are identical.
+ *
+ * @param {Histogram[]} normalizedBaselineTerms - An array of normalized baseline terms (Histogram objects).
+ * @param {Histogram[]} normalizedDriftedTerms - An array of normalized drifted terms (Histogram objects).
+ * @returns {number} The p-value indicating the similarity of the datasets.
+ */
+export const computeChi2PValue = (
+  normalizedBaselineTerms: Histogram[],
+  normalizedDriftedTerms: Histogram[]
+) => {
+  // Get all unique keys from both arrays
+  const allKeys: string[] = Array.from(
+    new Set([
+      ...normalizedBaselineTerms.map((term) => term.key.toString()),
+      ...normalizedDriftedTerms.map((term) => term.key.toString()),
+    ])
+  ).slice(0, 100);
+
+  // Calculate the chi-squared statistic and degrees of freedom
+  let chiSquared: number = 0;
+  const degreesOfFreedom: number = allKeys.length - 1;
+
+  if (degreesOfFreedom === 0) return 1;
+
+  allKeys.forEach((key) => {
+    const baselineTerm = normalizedBaselineTerms.find((term) => term.key === key);
+    const driftedTerm = normalizedDriftedTerms.find((term) => term.key === key);
+
+    const observed: number = driftedTerm?.percentage ?? 0;
+    const expected: number = baselineTerm?.percentage ?? 0;
+    chiSquared += Math.pow(observed - expected, 2) / (expected > 0 ? expected : 1e-6); // Prevent divide by zero
+  });
+
+  // Use the criticalTableLookup function to determine the p-value
+  return criticalTableLookup(chiSquared, degreesOfFreedom);
+};