[ES|QL] Recognize transformational commands with ast parsing (#184291)

## Summary

Closes #184139

It uses ast parsing to recognize if the query has a transformational
command. Although I added `metrics` on the list it wont work for it
until we support it on our ast system. I am just adding here
proactively. `Stats` and `keep` should work as expected though

<img width="1680" alt="image"
src="https://github.com/elastic/kibana/assets/17003240/75a15574-265a-407e-9df8-921eb5e20496">


### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

packages/kbn-esql-utils/index.ts

@@ -9,6 +9,7 @@
 export {
   getESQLAdHocDataview,
   getIndexPatternFromESQLQuery,
+  hasTransformationalCommand,
   getLimitFromESQLQuery,
   removeDropCommandsFromESQLQuery,
   getIndexForESQLQuery,

packages/kbn-esql-utils/src/index.ts

@@ -14,6 +14,7 @@ export {
   getIndexPatternFromESQLQuery,
   getLimitFromESQLQuery,
   removeDropCommandsFromESQLQuery,
+  hasTransformationalCommand,
 } from './utils/query_parsing_helpers';
 export { appendToESQLQuery, appendWhereClauseToESQLQuery } from './utils/append_to_query';
 export { getESQLQueryColumns, getESQLQueryColumnsRaw, getESQLResults } from './utils/run_query';

packages/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -10,6 +10,7 @@ import {
   getIndexPatternFromESQLQuery,
   getLimitFromESQLQuery,
   removeDropCommandsFromESQLQuery,
+  hasTransformationalCommand,
 } from './query_parsing_helpers';
 
 describe('esql query helpers', () => {
@@ -95,4 +96,25 @@ describe('esql query helpers', () => {
       ).toBe('from a | keep c ');
     });
   });
+
+  describe('hasTransformationalCommand', () => {
+    it('should return false for non transformational command', () => {
+      expect(hasTransformationalCommand('from a | eval b = 1')).toBeFalsy();
+    });
+
+    it('should return true for stats', () => {
+      expect(hasTransformationalCommand('from a | stats count() as total by a=b')).toBeTruthy();
+    });
+
+    it('should return true for keep', () => {
+      expect(hasTransformationalCommand('from a | keep field1, field2')).toBeTruthy();
+    });
+
+    it('should return false for commented out transformational command', () => {
+      expect(
+        hasTransformationalCommand(`from logstash-*
+      // | stats  var0 = avg(bytes) by geo.dest`)
+      ).toBeFalsy();
+    });
+  });
 });

packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -18,6 +18,13 @@ export function getIndexPatternFromESQLQuery(esql?: string) {
   return indices?.map((index) => index.text).join(',');
 }
 
+// For ES|QL we consider the following commands as transformational commands
+export function hasTransformationalCommand(esql?: string) {
+  const transformationalCommands = ['stats', 'keep', 'metrics'];
+  const { ast } = getAstAndSyntaxErrors(esql);
+  return transformationalCommands.some((command) => ast.find(({ name }) => name === command));
+}
+
 export function getLimitFromESQLQuery(esql: string): number {
   const limitCommands = esql.match(new RegExp(/LIMIT\s[0-9]+/, 'ig'));
   if (!limitCommands) {

src/plugins/discover/public/application/main/hooks/use_esql_mode.ts

@@ -8,6 +8,7 @@
 
 import { isEqual } from 'lodash';
 import { isOfAggregateQueryType, getAggregateQueryMode } from '@kbn/es-query';
+import { hasTransformationalCommand } from '@kbn/esql-utils';
 import { useCallback, useEffect, useRef } from 'react';
 import type { DataViewsContract } from '@kbn/data-views-plugin/public';
 import { switchMap } from 'rxjs';
@@ -17,9 +18,6 @@ import { getValidViewMode } from '../utils/get_valid_view_mode';
 import { FetchStatus } from '../../types';
 
 const MAX_NUM_OF_COLUMNS = 50;
-// For ES|QL we want in case of the following commands to display a table view, otherwise display a document view
-const TRANSFORMATIONAL_COMMANDS = ['stats', 'keep'];
-
 /**
  * Hook to take care of ES|QL state transformations when a new result is returned
  * If necessary this is setting displayed columns and selected data view
@@ -71,12 +69,9 @@ export function useEsqlMode({
           const hasResults = Boolean(next.result?.length);
           let queryHasTransformationalCommands = false;
           if ('esql' in query) {
-            TRANSFORMATIONAL_COMMANDS.forEach((command: string) => {
-              if (query.esql.toLowerCase().includes(command)) {
-                queryHasTransformationalCommands = true;
-                return;
-              }
-            });
+            if (hasTransformationalCommand(query.esql)) {
+              queryHasTransformationalCommands = true;
+            }
           }
 
           if (isEsqlQuery) {

src/plugins/unified_histogram/public/layout/helpers.ts

@@ -7,9 +7,8 @@
  */
 
 import { AggregateQuery } from '@kbn/es-query';
-
-const TRANSFORMATIONAL_COMMANDS = ['stats', 'keep'];
+import { hasTransformationalCommand } from '@kbn/esql-utils';
 
 export const shouldDisplayHistogram = (query: AggregateQuery) => {
-  return !TRANSFORMATIONAL_COMMANDS.some((command) => query.esql.toLowerCase().includes(command));
+  return !hasTransformationalCommand(query.esql);
 };