Merge branch 'main' into split-quality-gate-pipeline-init

.buildkite/scripts/steps/esql_grammar_sync.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 synchronize_lexer_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4"
-  destination_file="./packages/kbn-monaco/src/esql/antlr/esql_lexer.g4"
+  destination_file="./packages/kbn-esql-ast/src/antlr/esql_lexer.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file" 
@@ -27,7 +27,7 @@ synchronize_lexer_grammar () {
 synchronize_parser_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4"
-  destination_file="./packages/kbn-monaco/src/esql/antlr/esql_parser.g4"
+  destination_file="./packages/kbn-esql-ast/src/antlr/esql_parser.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file" 
@@ -105,7 +105,7 @@ main () {
   .buildkite/scripts/bootstrap.sh
 
   # Build ANTLR stuff
-  cd ./packages/kbn-monaco/src  
+  cd ./packages/kbn-esql-ast/src  
   yarn build:antlr4:esql 
 
   # Make a commit

.eslintignore

@@ -39,6 +39,7 @@ snapshots.js
 /packages/kbn-ui-framework/dist
 /packages/kbn-flot-charts/lib
 /packages/kbn-monaco/src/**/antlr
+/packages/kbn-esql-ast/src/**/antlr
 
 # Bazel
 /bazel-*

.github/CODEOWNERS

@@ -393,7 +393,10 @@ packages/kbn-eslint-plugin-imports @elastic/kibana-operations
 packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team
 examples/eso_model_version_example @elastic/kibana-security
 x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security
+packages/kbn-esql-ast @elastic/kibana-visualizations
 packages/kbn-esql-utils @elastic/kibana-esql
+packages/kbn-esql-validation-autocomplete @elastic/kibana-visualizations
+examples/esql_validation_example @elastic/kibana-visualizations
 packages/kbn-event-annotation-common @elastic/kibana-visualizations
 packages/kbn-event-annotation-components @elastic/kibana-visualizations
 src/plugins/event_annotation_listing @elastic/kibana-visualizations

.i18nrc.json

@@ -81,6 +81,7 @@
     "interpreter": "src/legacy/core_plugins/interpreter",
     "imageEmbeddable": "src/plugins/image_embeddable",
     "kbn": "src/legacy/core_plugins/kibana",
+    "kbn-esql-validation-autocomplete": "packages/kbn-esql-validation-autocomplete/src",
     "kbnConfig": "packages/kbn-config/src",
     "kbnDocViews": "src/legacy/core_plugins/kbn_doc_views",
     "kibana_react": "src/legacy/core_plugins/kibana_react",

examples/esql_validation_example/README.md

@@ -0,0 +1,9 @@
+# esql_validation_example
+
+Examples of the ES|QL validation service.
+
+To run this example, start kibana with the `--run-examples` flag.
+
+```bash
+yarn start --run-examples
+```

examples/esql_validation_example/kibana.jsonc

@@ -0,0 +1,17 @@
+{
+    "type": "plugin",
+    "id": "@kbn/esql-validation-example-plugin",
+    "owner": "@elastic/kibana-visualizations",
+    "plugin": {
+      "id": "esqlValidationExample",
+      "server": false,
+      "browser": true,
+      "configPath": [
+        "esql_validation_example"
+      ],
+      "requiredPlugins": [
+        "dataViews",
+        "developerExamples"
+      ]
+    }
+  }

examples/esql_validation_example/public/app.tsx

@@ -0,0 +1,178 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React, { useEffect, useMemo, useState } from 'react';
+import {
+  EuiPage,
+  EuiPageBody,
+  EuiPageSection,
+  EuiPageHeader,
+  EuiSpacer,
+  EuiFieldText,
+  EuiCheckboxGroup,
+  EuiFormRow,
+  EuiSwitch,
+  EuiForm,
+  EuiCallOut,
+} from '@elastic/eui';
+
+import type { CoreStart } from '@kbn/core/public';
+
+import { ESQLCallbacks, validateQuery } from '@kbn/esql-validation-autocomplete';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import type { StartDependencies } from './plugin';
+import { CodeSnippet } from './code_snippet';
+
+export const App = (props: { core: CoreStart; plugins: StartDependencies }) => {
+  const [currentErrors, setErrors] = useState<string[]>([]);
+  const [currentWarnings, setWarnings] = useState<string[]>([]);
+  const [currentQuery, setQuery] = useState(
+    'from index1 | eval var0 = round(numberField, 2) | stats by stringField'
+  );
+
+  const [ignoreErrors, setIgnoreErrors] = useState(false);
+  const [callbacksEnabled, setCallbacksEnabled] = useState<
+    Record<'sources' | 'fields' | 'policies' | 'metaFields', boolean>
+  >({
+    sources: false,
+    fields: true,
+    policies: true,
+    metaFields: true,
+  });
+
+  const callbacks: ESQLCallbacks = useMemo(
+    () => ({
+      getSources: callbacksEnabled.sources
+        ? async () =>
+            ['index1', 'anotherIndex', 'dataStream'].map((name) => ({ name, hidden: false }))
+        : undefined,
+      getFieldsFor: callbacksEnabled.fields
+        ? async () => [
+            { name: 'numberField', type: 'number' },
+            { name: 'stringField', type: 'string' },
+          ]
+        : undefined,
+      getMetaFields: callbacksEnabled.metaFields
+        ? async () => ['_version', '_id', '_index', '_source']
+        : undefined,
+      getPolicies: callbacksEnabled.policies
+        ? async () => [
+            {
+              name: 'my-policy',
+              sourceIndices: ['policyIndex'],
+              matchField: 'otherStringField',
+              enrichFields: ['otherNumberField'],
+            },
+          ]
+        : undefined,
+    }),
+    [callbacksEnabled]
+  );
+
+  useEffect(() => {
+    if (currentQuery === '') {
+      return;
+    }
+    validateQuery(
+      currentQuery,
+      getAstAndSyntaxErrors,
+      { ignoreOnMissingCallbacks: ignoreErrors },
+      callbacks
+    ).then(({ errors: validationErrors, warnings: validationWarnings }) => {
+      // syntax errors come with a slight different format than other validation errors
+      setErrors(validationErrors.map((e) => ('severity' in e ? e.message : e.text)));
+      setWarnings(validationWarnings.map((e) => e.text));
+    });
+  }, [currentQuery, ignoreErrors, callbacks]);
+
+  const checkboxes = [
+    {
+      id: 'sources',
+      label: 'getSources callback => index1, anotherIndex, dataStream',
+    },
+    {
+      id: 'fields',
+      label: 'getFieldsFor callback => numberField, stringField',
+    },
+    {
+      id: 'policies',
+      label: 'getPolicies callback => my-policy',
+    },
+    {
+      id: 'metaFields',
+      label: 'getMetaFields callback => _version, _id, _index, _source',
+    },
+  ];
+
+  return (
+    <EuiPage>
+      <EuiPageBody style={{ maxWidth: 800, margin: '0 auto' }}>
+        <EuiPageHeader paddingSize="s" bottomBorder={true} pageTitle="ES|QL validation example" />
+        <EuiPageSection paddingSize="s">
+          <p>This app shows how to use the ES|QL validation API with all its options</p>
+
+          <EuiSpacer />
+          <EuiForm component="form" fullWidth>
+            <EuiCheckboxGroup
+              options={checkboxes}
+              idToSelectedMap={callbacksEnabled}
+              onChange={(optionId) => {
+                setCallbacksEnabled({
+                  ...callbacksEnabled,
+                  [optionId]: !callbacksEnabled[optionId as keyof typeof callbacksEnabled],
+                });
+              }}
+            />
+
+            <EuiSpacer />
+
+            <EuiFormRow label="Validation options" hasChildLabel={false}>
+              <EuiSwitch
+                name="switch"
+                label="Ignore errors on missing callback"
+                checked={ignoreErrors}
+                onChange={() => setIgnoreErrors(!ignoreErrors)}
+              />
+            </EuiFormRow>
+            <EuiSpacer />
+            <EuiFormRow
+              label="Write the ES|QL query here"
+              helpText={currentErrors.length ? '' : 'No errors found'}
+              isInvalid={currentErrors.length > 0}
+              error={currentErrors}
+            >
+              <EuiFieldText
+                fullWidth
+                placeholder="from index1"
+                value={currentQuery}
+                onChange={(e) => setQuery(e.target.value)}
+                isInvalid={currentErrors.length > 0}
+              />
+            </EuiFormRow>
+            {currentWarnings.length ? (
+              <EuiCallOut title="Validation warnings" color="warning" iconType="warning">
+                <p>Here the list of warnings:</p>
+                <ul>
+                  {currentWarnings.map((message) => (
+                    <li key={message}>{message}</li>
+                  ))}
+                </ul>
+              </EuiCallOut>
+            ) : null}
+          </EuiForm>
+          <EuiSpacer />
+          <CodeSnippet
+            currentQuery={currentQuery}
+            callbacks={callbacksEnabled}
+            ignoreErrors={ignoreErrors}
+          />
+        </EuiPageSection>
+      </EuiPageBody>
+    </EuiPage>
+  );
+};

examples/esql_validation_example/public/code_snippet.tsx

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EuiCodeBlock } from '@elastic/eui';
+import React from 'react';
+
+interface CodeSnippetProps {
+  currentQuery: string;
+  callbacks: Record<'sources' | 'fields' | 'policies' | 'metaFields', boolean>;
+  ignoreErrors: boolean;
+}
+
+function getCallbacksCode(callbacks: CodeSnippetProps['callbacks']) {
+  return `({
+        ${
+          callbacks.sources
+            ? `getSources: async () =>
+            ['index1', 'anotherIndex', 'dataStream'].map((name) => ({ name, hidden: false })),`
+            : ''
+        }
+        ${
+          callbacks.fields
+            ? `
+        // the getFieldsFor callback gets an esql query to get the required fields
+        // note that the query is not optimized yet, so things like "| limit 0"
+        // might be appended to speed up the retrieval. 
+        getFieldsFor: async (esqlFieldsQuery: string) => [
+            { name: 'numberField', type: 'number' },
+            { name: 'stringField', type: 'string' },
+        ],`
+            : ''
+        }
+        ${
+          callbacks.metaFields
+            ? `getMetaFields: async () => ['_version', '_id', '_index', '_source'],`
+            : ''
+        }
+        ${
+          callbacks.policies
+            ? `getPolicies: async () => [
+            {
+                name: 'my-policy',
+                sourceIndices: ['policyIndex'],
+                matchField: 'otherStringField',
+                enrichFields: ['otherNumberField'],
+            },
+        ],`
+            : ''
+        }
+})`.replace(/^\s*\n/gm, '');
+}
+
+export function CodeSnippet({ currentQuery, callbacks, ignoreErrors }: CodeSnippetProps) {
+  return (
+    <EuiCodeBlock language="typescript" isCopyable>
+      {`
+import { ESQLCallbacks, validateQuery } from '@kbn/esql-validation-autocomplete';
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+
+const currentQuery = "${currentQuery}";
+
+const callbacks: ESQLCallbacks = () => ${getCallbacksCode(callbacks)};
+
+const {errors, warnings} = validateQuery(
+    currentQuery,
+    getAstAndSyntaxErrors,
+    { ignoreOnMissingCallbacks: ${Boolean(ignoreErrors)} },
+    callbacks
+);
+`}
+    </EuiCodeBlock>
+  );
+}

examples/esql_validation_example/public/index.tsx

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ESQLValidationExamplePlugin } from './plugin';
+
+export const plugin = () => new ESQLValidationExamplePlugin();

examples/esql_validation_example/public/mount.tsx

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as React from 'react';
+import { render, unmountComponentAtNode } from 'react-dom';
+
+import type { CoreSetup, AppMountParameters } from '@kbn/core/public';
+import type { StartDependencies } from './plugin';
+
+export const mount =
+  (coreSetup: CoreSetup<StartDependencies>) =>
+  async ({ element }: AppMountParameters) => {
+    const [core, plugins] = await coreSetup.getStartServices();
+    const { App } = await import('./app');
+
+    const i18nCore = core.i18n;
+
+    const reactElement = (
+      <i18nCore.Context>
+        <App core={core} plugins={plugins} />
+      </i18nCore.Context>
+    );
+
+    render(reactElement, element);
+    return () => {
+      unmountComponentAtNode(element);
+      plugins.data.search.session.clear();
+    };
+  };