[ES|QL] gather inline function documentation from Elasticsearch (#184689

)

## Summary

Adds a script to gather inline function documentation sections from
Elasticsearch.

<img width="600" alt="Screenshot 2024-06-03 at 12 03 06 PM"
src="https://github.com/elastic/kibana/assets/315764/05e25486-8066-4e4c-b8c9-0be0bcd9543b">


To run the script:

`cd packages/kbn-text-based-editor && yarn make:docs
path/to/elasticsearch`

This script will be operationalized in a follow-up PR.

packages/kbn-text-based-editor/package.json

@@ -3,5 +3,13 @@
   "private": true,
   "version": "1.0.0",
   "license": "SSPL-1.0 OR Elastic License 2.0",
-  "sideEffects": ["*.scss"]
-}
+  "sideEffects": [
+    "*.scss"
+  ],
+  "scripts": {
+    "make:docs": "ts-node --transpileOnly scripts/generate_esql_docs.ts",
+    "postmake:docs": "yarn run lint:fix && yarn run i18n:fix",
+    "lint:fix": "cd ../.. && node ./scripts/eslint --fix ./packages/kbn-text-based-editor/src/esql_documentation_sections.tsx",
+    "i18n:fix": "cd ../.. && node ./scripts/i18n_check.js --fix"
+  }
+}

packages/kbn-text-based-editor/scripts/generate_esql_docs.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as recast from 'recast';
+const n = recast.types.namedTypes;
+import fs from 'fs';
+import path from 'path';
+import { functions } from '../src/esql_documentation_sections';
+
+(function () {
+  const pathToElasticsearch = process.argv[2];
+  const functionDocs = loadFunctionDocs(pathToElasticsearch);
+  writeFunctionDocs(functionDocs);
+})();
+
+function loadFunctionDocs(pathToElasticsearch: string) {
+  // Define the directory path
+  const dirPath = path.join(pathToElasticsearch, '/docs/reference/esql/functions/kibana/docs');
+
+  // Read the directory
+  const files = fs.readdirSync(dirPath);
+
+  // Initialize an empty map
+  const functionMap = new Map<string, string>();
+
+  // Iterate over each file in the directory
+  for (const file of files) {
+    // Ensure we only process .md files
+    if (path.extname(file) === '.md') {
+      // Read the file content
+      const content = fs.readFileSync(path.join(dirPath, file), 'utf-8');
+
+      // Get the function name from the file name by removing the .md extension
+      const functionName = path.basename(file, '.md');
+
+      // Add the function name and content to the map
+      functionMap.set(functionName, content);
+    }
+  }
+
+  return functionMap;
+}
+
+function writeFunctionDocs(functionDocs: Map<string, string>) {
+  const codeStrings = Array.from(functionDocs.entries()).map(([name, doc]) => {
+    const docWithoutLinks = removeAsciiDocInternalCrossReferences(
+      doc,
+      Array.from(functionDocs.keys())
+    );
+    return `
+  const foo = 
+  // Do not edit manually... automatically generated by scripts/generate_esql_docs.ts
+  {
+    label: i18n.translate(
+      'textBasedEditor.query.textBasedLanguagesEditor.documentationESQL.${name}',
+      {
+        defaultMessage: '${name.toUpperCase()}',
+      }
+    ),
+    description: (
+      <Markdown
+        markdownContent={i18n.translate(
+          'textBasedEditor.query.textBasedLanguagesEditor.documentationESQL.${name}.markdown',
+          {
+            defaultMessage: \`${docWithoutLinks.replaceAll('`', '\\`')}\`,
+            description:
+              'Text is in markdown. Do not translate function names, special characters, or field names like sum(bytes)',
+            ignoreTag: true,
+          }
+        )}
+      />
+    ),
+  };`;
+  });
+
+  const pathToDocsFile = path.join(__dirname, '../src/esql_documentation_sections.tsx');
+
+  const ast = recast.parse(fs.readFileSync(pathToDocsFile, 'utf-8'), {
+    parser: require('recast/parsers/babel'),
+  });
+
+  const functionsList = findFunctionsList(ast);
+
+  functionsList.elements = codeStrings.map(
+    (codeString) => recast.parse(codeString).program.body[0].declarations[0].init
+  );
+
+  const newFileContents = recast.print(ast);
+
+  fs.writeFileSync(pathToDocsFile, newFileContents.code);
+}
+
+/**
+ * Deals with asciidoc internal cross-references in the function descriptions
+ *
+ * Examples:
+ * <<esql-mv_max>> -> `MV_MAX`
+ * <<esql-st_intersects,ST_INTERSECTS>> -> `ST_INTERSECTS`
+ * <<esql-multivalued-fields, multivalued fields>> -> multivalued fields
+ */
+function removeAsciiDocInternalCrossReferences(asciidocString: string, functionNames: string[]) {
+  const internalCrossReferenceRegex = /<<(.+?)(,.+?)?>>/g;
+
+  const extractPossibleFunctionName = (id: string) => id.replace('esql-', '');
+
+  return asciidocString.replace(internalCrossReferenceRegex, (_match, anchorId, linkText) => {
+    const ret = linkText ? linkText.slice(1) : anchorId;
+
+    const matchingFunction = functionNames.find(
+      (name) =>
+        extractPossibleFunctionName(ret) === name.toLowerCase() ||
+        extractPossibleFunctionName(ret) === name.toUpperCase()
+    );
+    return matchingFunction ? `\`${matchingFunction.toUpperCase()}\`` : ret;
+  });
+}
+
+/**
+ * This function searches the AST for the functions list
+ */
+function findFunctionsList(ast: any): recast.types.namedTypes.ArrayExpression {
+  let foundArray: recast.types.namedTypes.ArrayExpression | null = null;
+
+  const functionsArrayIdentifier = Object.keys({ functions })[0];
+
+  recast.visit(ast, {
+    visitVariableDeclarator(astPath) {
+      if (
+        n.Identifier.check(astPath.node.id) &&
+        astPath.node.id.name === functionsArrayIdentifier
+      ) {
+        this.traverse(astPath);
+      }
+      return false;
+    },
+    visitObjectProperty(astPath) {
+      if (
+        n.Identifier.check(astPath.node.key) &&
+        astPath.node.key.name === 'items' &&
+        n.ArrayExpression.check(astPath.node.value)
+      ) {
+        foundArray = astPath.node.value;
+        this.abort();
+      }
+      return false;
+    },
+  });
+
+  if (!foundArray) {
+    throw new Error('Could not find the functions array in the AST');
+  }
+
+  return foundArray;
+}