[HTTP/OAS] Merge OpenAPI specs by using kbn-openapi-bundler (elasti…

…c#189262)

**Addresses:** elastic#186356
**Relates to:** elastic#184428

This PR adds a merging JS script based on the utility implemented in elastic#186356. Resulted OpenAPI bundle as committed in `oas_docs/output/kibana.serverless.bundled.yaml`.

elastic#188110 implements and exposes `merge` utility design to merge source OpenAPI specs without processing. It's has only a programmatic API. To merge OpenAPI specs it's required to add a JS script like below

```js
const { merge } = require('@kbn/openapi-bundler');

(async () => {
  await merge({
   sourceGlobs: [/* a list of source globs goes here */],
   outputFilePath: 'path/to/the/output/file.yaml',
  });
})();
```

The JS script added in this PR includes source OpenAPI specs presented in `oas_docs/makefile` plus Security Solution OpenAPI specs based on elastic#184428.

**To run** the script use the following command from Kibana root folder

```bash
node ./oas_docs/scripts/merge_serverless_oas.js
```

Running Spectral OpenAPI linter on the result bundle shows a number of errors caused by `no-$ref-siblings` rule. This caused by the current code generator implementation which requires `default` property to be set next to `$ref` though it's not correct for OpenAPI `3.0.3` while it's allowed in `3.1`. It seems that Bump.sh handles such cases properly though by properly showing a default value.

We need to analyze the problem and decide if/when we should fix it.

The rest of warnings look fixable and will be addressed in the next stage after setting up linter rules.

Since `@kbn/openapi-bundler` package is tailored specifically for Kibana it should replace Redocly currently used to merge OpenAPI specs. It also means `oas_docs/makefile` should be superseded by JS script(s) using `merge` utility form `@kbn/openapi-bundler` package.

`@kbn/openapi-bundler` SHOULD NOT replace OpenAPI linters since it doesn't perform thorough linting. It's good if we continue adopting `spectral-cli` for linting purposes.

.buildkite/scripts/steps/checks.sh

@@ -14,3 +14,4 @@ fi
 .buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
+.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Merge Kibana OpenAPI specs
+
+(cd oas_docs && make api-docs && make api-docs-lint)

oas_docs/kibana.info.yaml

@@ -7,15 +7,15 @@ info:
     Each request that you make happens in isolation from other calls and must include all of the necessary information for Kibana to fulfill the
     request.
     API requests return JSON output, which is a format that is machine-readable and works well for automation.
-    
+
     To interact with Kibana APIs, use the following operations:
-    
+
     - GET: Fetches the information.
     - PATCH: Applies partial modifications to the existing information.
     - POST: Adds new information.
     - PUT: Updates the existing information.
     - DELETE: Removes the information.
-    
+
     You can prepend any Kibana API endpoint with `kbn:` and run the request in **Dev Tools → Console**.
     For example:
 
@@ -24,26 +24,26 @@ info:
     ```
 
     For more information about the console, refer to [Run API requests](https://www.elastic.co/guide/en/kibana/current/console-kibana.html).
-  version: "1.0.2"
+  version: '1.0.2'
   license:
     name: Elastic License 2.0
     url: https://www.elastic.co/licensing/elastic-license
-  contact: 
+  contact:
     name: Kibana Team
-# servers:
-#   - url: https://{kibana_url}
-#     variables:
-#       kibana_url:
-#         default: localhost:5601
-# security:
-#   - apiKeyAuth: []
-# components:
-#   securitySchemes:
-#     apiKeyAuth:
-#       type: apiKey
-#       in: header
-#       name: Authorization
-#       description: >
-#         These APIs use key-based authentication.
-#         You must create an API key and use the encoded value in the request header.
-#         For example: `Authorization: ApiKey base64AccessApiKey`
+servers:
+  - url: https://{kibana_url}
+    variables:
+      kibana_url:
+        default: localhost:5601
+security:
+  - apiKeyAuth: []
+components:
+  securitySchemes:
+    apiKeyAuth:
+      type: apiKey
+      in: header
+      name: Authorization
+      description: >
+        These APIs use key-based authentication.
+        You must create an API key and use the encoded value in the request header.
+        For example: `Authorization: ApiKey base64AccessApiKey`

oas_docs/makefile

@@ -13,14 +13,16 @@
 # this material is strictly forbidden unless prior written
 # permission is obtained from Elasticsearch B.V.
 
-.PHONY: api-docs
-api-docs: ## Generate kibana.yaml
+api-docs: ## Generate ESS Kibana OpenAPI bundles with kbn-openapi-bundler
+	@node scripts/merge_ess_oas.js
+
+.PHONY: api-docs-redocly
+api-docs-redocly: ## Generate kibana.yaml
 	@npx @redocly/cli join "kibana.info.yaml" "../x-pack/plugins/alerting/docs/openapi/bundled.yaml" "../x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml" "../x-pack/plugins/cases/docs/openapi/bundled.yaml" "../x-pack/plugins/actions/docs/openapi/bundled.yaml" "../src/plugins/data_views/docs/openapi/bundled.yaml" "../x-pack/plugins/ml/common/openapi/ml_apis.yaml" "../packages/core/saved-objects/docs/openapi/bundled.yaml" "../x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml" "../x-pack/plugins/fleet/common/openapi/bundled.yaml" "bundle.json" -o "output/kibana.yaml" --prefix-components-with-info-prop title
 
 .PHONY: api-docs-stateful
 api-docs-stateful: ## Generate only kibana.yaml
-	@npx @redocly/cli join "kibana.info.yaml" "../x-pack/plugins/alerting/docs/openapi/bundled.yaml" "../x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml" "../x-pack/plugins/cases/docs/openapi/bundled.yaml" "../x-pack/plugins/actions/docs/openapi/bundled.yaml" "../src/plugins/data_views/docs/openapi/bundled.yaml" "../x-pack/plugins/ml/common/openapi/ml_apis.yaml" "../packages/core/saved-objects/docs/openapi/bundled.yaml" "../x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml" "../x-pack/plugins/fleet/common/openapi/bundled.yaml" "bundle.json" -o "output/kibana.yaml" --prefix-components-with-info-prop title
-# Temporarily omit "../x-pack/plugins/fleet/common/openapi/bundled.yaml" due to internals tag and tag sorting
+	@node scripts/merge_ess_oas.js
 
 .PHONY: api-docs-lint
 api-docs-lint: ## Run spectral API docs linter 

oas_docs/scripts/merge_ess_oas.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../../src/setup_node_env');
+const { merge } = require('@kbn/openapi-bundler');
+const { REPO_ROOT } = require('@kbn/repo-info');
+
+(async () => {
+  await merge({
+    sourceGlobs: [
+      `${REPO_ROOT}/oas_docs/bundle.json`,
+      `${REPO_ROOT}/x-pack/plugins/actions/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/src/plugins/data_views/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/ml/common/openapi/ml_apis.yaml`,
+      `${REPO_ROOT}/packages/core/saved-objects/docs/openapi/bundled.yaml`,
+
+      // Observability Solution
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml`,
+
+      // Security solution
+      `${REPO_ROOT}/x-pack/plugins/security_solution/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-lists-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-exceptions-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/packages/kbn-elastic-assistant-common/docs/openapi/ess/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/osquery/docs/openapi/ess/*.schema.yaml`,
+    ],
+    outputFilePath: `${REPO_ROOT}/oas_docs/output/kibana.yaml`,
+    options: {
+      prototypeDocument: `${REPO_ROOT}/oas_docs/kibana.info.yaml`,
+    },
+  });
+})();

oas_docs/scripts/merge_serverless_oas.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../../src/setup_node_env');
+const { merge } = require('@kbn/openapi-bundler');
+const { REPO_ROOT } = require('@kbn/repo-info');
+
+(async () => {
+  await merge({
+    sourceGlobs: [
+      `${REPO_ROOT}/oas_docs/bundle.serverless.json`,
+      `${REPO_ROOT}/x-pack/plugins/actions/docs/openapi/bundled_serverless.yaml`,
+      `${REPO_ROOT}/src/plugins/data_views/docs/openapi/bundled.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/ml/common/openapi/ml_apis_serverless.yaml`,
+      `${REPO_ROOT}/packages/core/saved-objects/docs/openapi/bundled_serverless.yaml`,
+
+      // Observability Solution
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/apm/docs/openapi/apm.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/observability_solution/slo/docs/openapi/slo/bundled.yaml`,
+
+      // Security solution
+      `${REPO_ROOT}/x-pack/plugins/security_solution/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-exceptions-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/packages/kbn-securitysolution-endpoint-exceptions-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/packages/kbn-elastic-assistant-common/docs/openapi/serverless/*.schema.yaml`,
+      `${REPO_ROOT}/x-pack/plugins/osquery/docs/openapi/serverless/*.schema.yaml`,
+    ],
+    outputFilePath: `${REPO_ROOT}/oas_docs/output/kibana.serverless.yaml`,
+    options: {
+      prototypeDocument: `${REPO_ROOT}/oas_docs/kibana.info.yaml`,
+    },
+  });
+})();

packages/kbn-openapi-bundler/src/bundler/merge_documents/enrich_with_version_mime_param.ts

@@ -12,9 +12,12 @@ import { isRefNode } from '../process_document';
 import { getOasDocumentVersion } from '../../utils/get_oas_document_version';
 import { KNOWN_HTTP_METHODS } from './http_methods';
 
+const DEFAULT_API_VERSION = '2023-10-31';
+const VERSION_REGEX = /\d{4}-\d{2}-\d{2}/;
+
 export function enrichWithVersionMimeParam(resolvedDocuments: ResolvedDocument[]): void {
   for (const resolvedDocument of resolvedDocuments) {
-    const version = getOasDocumentVersion(resolvedDocument);
+    const version = extractApiVersion(resolvedDocument);
     const paths = resolvedDocument.document.paths as OpenAPIV3.PathsObject;
 
     for (const path of Object.keys(paths ?? {})) {
@@ -80,3 +83,13 @@ function enrichContentWithVersion(
     delete content[mimeType];
   }
 }
+
+function extractApiVersion(resolvedDocument: ResolvedDocument): string {
+  const version = getOasDocumentVersion(resolvedDocument);
+
+  if (!VERSION_REGEX.test(version)) {
+    return DEFAULT_API_VERSION;
+  }
+
+  return version < DEFAULT_API_VERSION ? DEFAULT_API_VERSION : version;
+}