From 01b99e7cd8ecf54695ede50ddffa6e2d13724514 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Thu, 25 Jul 2024 19:33:17 +0200 Subject: [PATCH] [Security Solution] Implement shared components conflict resolution functionality (#188812) **Resolves:** https://github.com/elastic/kibana/issues/188817 This PR adds automatic shared components conflict resolution functionality for OpenAPI merger. It boils down to a similar result as `npx @redocly/cli join --prefix-components-with-info-prop title` produces by prefixing shared components with document's title in each source. OpenAPI bundler intentionally won't solve conflicts automatically since it's focused on bundling domain APIs where conflicts are usually indicators of upstream problems. While working with various OpenAPI specs it may happen that different specs use exactly the same name for some shared components but different definitions. It must be avoided inside one API domain but it's a usual situation when merging OpenAPI specs of different API domains. For example domains may define a shared `Id` or `404Response` schemas where `Id` is a string in one domain and a number in another. OpenAPI merger implemented in https://github.com/elastic/kibana/pull/188110 and OpenAPI bundler implemented in https://github.com/elastic/kibana/pull/171526 do not solve shared components related conflicts automatically. It works perfectly for a single API domain forcing engineers choosing shared schema names carefully. This PR adds automatic shared components conflict resolution for OpenAPI merger. It prefixes shared component names with a normalized document's title. OpenAPI bundler intentionally won't solve conflicts automatically since it's focused on bundling domain APIs where conflicts are usually indicators of upstream problems. Consider two following OpenAPI specs each defining local `MySchema` **spec1.schema.yaml** ```yaml openapi: 3.0.3 info: title: My endpoint version: '2023-10-31' paths: /api/some_api: get: operationId: MyEndpointGet responses: '200': content: application/json: schema: $ref: '#/components/schemas/MySchema' components: schemas: MySchema: type: string enum: - value1 ``` **spec2.schema.yaml** ```yaml openapi: 3.0.3 info: title: My another endpoint version: '2023-10-31' paths: /api/another_api: get: operationId: MyAnotherEndpointGet responses: '200': content: application/json: schema: $ref: '#/components/schemas/MySchema' components: schemas: MySchema: type: number ``` and a script to merge them ```js require('../../src/setup_node_env'); const { resolve } = require('path'); const { merge } = require('@kbn/openapi-bundler'); const { REPO_ROOT } = require('@kbn/repo-info'); (async () => { await merge({ sourceGlobs: [ `${REPO_ROOT}/oas_docs/spec1.schema.yaml`, `${REPO_ROOT}/oas_docs/spec2.schema.yaml`, ], outputFilePath: resolve(`${REPO_ROOT}/oas_docs/merged.yaml`), options: { mergedSpecInfo: { title: 'Merge result', version: 'my version', }, }, }); })(); ``` will be merged successfully to **merged.yaml** ```yaml openapi: 3.0.3 info: title: Merge result version: 'my version' paths: /api/another_api: get: operationId: MyAnotherEndpointGet responses: '200': content: application/json; Elastic-Api-Version=2023-10-31: schema: $ref: '#/components/schemas/My_another_endpoint_MySchema' /api/some_api: get: operationId: MyEndpointGet responses: '200': content: application/json; Elastic-Api-Version=2023-10-31: schema: $ref: '#/components/schemas/My_endpoint_MySchema' components: schemas: My_another_endpoint_MySchema: type: number My_endpoint_MySchema: enum: - value1 type: string ``` --- .../src/bundler/bundle_document.ts | 50 +- .../merge_documents/merge_documents.ts | 2 +- .../merge_shared_components.ts | 39 +- .../document_processors/bundle_refs.ts | 26 +- .../namespace_components.test.ts | 148 ++++ .../namespace_components.ts | 138 ++++ .../remove_unused_components.ts | 62 +- .../src/bundler/processor_sets.ts | 57 ++ .../src/bundler/ref_resolver/ref_resolver.ts | 8 +- .../src/openapi_bundler.ts | 18 +- .../kbn-openapi-bundler/src/openapi_merger.ts | 59 +- .../src/utils/extract_by_json_pointer.ts | 60 +- .../src/utils/insert_by_json_pointer.ts | 4 +- .../src/utils/read_document.ts | 45 ++ .../src/utils/read_json_document.ts | 16 - .../src/utils/read_yaml_document.ts | 17 - .../src/utils/write_yaml_document.ts | 19 +- .../tests/create_oas_document.ts | 14 +- .../merger/different_oas_versions.test.ts | 24 +- .../tests/merger/merge_specs.ts | 8 +- ..._specs_with_conflicting_components.test.ts | 711 ++++++++++++++++++ .../unresolvable_operation_conflicts.test.ts | 13 +- .../unresolvable_path_item_conflicts.test.ts | 6 +- 23 files changed, 1323 insertions(+), 221 deletions(-) create mode 100644 packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.test.ts create mode 100644 packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.ts create mode 100644 packages/kbn-openapi-bundler/src/bundler/processor_sets.ts create mode 100644 packages/kbn-openapi-bundler/src/utils/read_document.ts delete mode 100644 packages/kbn-openapi-bundler/src/utils/read_json_document.ts delete mode 100644 packages/kbn-openapi-bundler/src/utils/read_yaml_document.ts create mode 100644 packages/kbn-openapi-bundler/tests/merger/merging_specs_with_conflicting_components.test.ts diff --git a/packages/kbn-openapi-bundler/src/bundler/bundle_document.ts b/packages/kbn-openapi-bundler/src/bundler/bundle_document.ts index 3b827a15c90f0..3058f295de9de 100644 --- a/packages/kbn-openapi-bundler/src/bundler/bundle_document.ts +++ b/packages/kbn-openapi-bundler/src/bundler/bundle_document.ts @@ -9,24 +9,12 @@ import { isAbsolute } from 'path'; import { RefResolver } from './ref_resolver/ref_resolver'; import { processDocument } from './process_document/process_document'; -import { X_CODEGEN_ENABLED, X_INLINE, X_INTERNAL, X_LABELS, X_MODIFY } from './known_custom_props'; +import { X_INLINE } from './known_custom_props'; import { isPlainObjectType } from '../utils/is_plain_object_type'; import { ResolvedDocument } from './ref_resolver/resolved_document'; -import { ResolvedRef } from './ref_resolver/resolved_ref'; -import { createSkipNodeWithInternalPropProcessor } from './process_document/document_processors/skip_node_with_internal_prop'; -import { createSkipInternalPathProcessor } from './process_document/document_processors/skip_internal_path'; -import { createModifyPartialProcessor } from './process_document/document_processors/modify_partial'; -import { createModifyRequiredProcessor } from './process_document/document_processors/modify_required'; -import { createRemovePropsProcessor } from './process_document/document_processors/remove_props'; -import { - createFlattenFoldedAllOfItemsProcessor, - createMergeNonConflictingAllOfItemsProcessor, - createUnfoldSingleAllOfItemProcessor, -} from './process_document/document_processors/reduce_all_of_items'; -import { createIncludeLabelsProcessor } from './process_document/document_processors/include_labels'; import { BundleRefProcessor } from './process_document/document_processors/bundle_refs'; import { RemoveUnusedComponentsProcessor } from './process_document/document_processors/remove_unused_components'; -import { insertRefByPointer } from '../utils/insert_by_json_pointer'; +import { DocumentNodeProcessor } from './process_document/document_processors/types/document_node_processor'; export class SkipException extends Error { constructor(public documentPath: string, message: string) { @@ -34,10 +22,6 @@ export class SkipException extends Error { } } -interface BundleDocumentOptions { - includeLabels?: string[]; -} - /** * Bundles document into one file and performs appropriate document modifications. * @@ -54,7 +38,7 @@ interface BundleDocumentOptions { */ export async function bundleDocument( absoluteDocumentPath: string, - options?: BundleDocumentOptions + processors: Readonly = [] ): Promise { if (!isAbsolute(absoluteDocumentPath)) { throw new Error( @@ -75,26 +59,11 @@ export async function bundleDocument( throw new SkipException(resolvedDocument.absolutePath, 'Document has no paths defined'); } - const defaultProcessors = [ - createSkipNodeWithInternalPropProcessor(X_INTERNAL), - createSkipInternalPathProcessor('/internal'), - createModifyPartialProcessor(), - createModifyRequiredProcessor(), - createRemovePropsProcessor([X_INLINE, X_MODIFY, X_CODEGEN_ENABLED, X_LABELS]), - createFlattenFoldedAllOfItemsProcessor(), - createMergeNonConflictingAllOfItemsProcessor(), - createUnfoldSingleAllOfItemProcessor(), - ]; - - if (options?.includeLabels) { - defaultProcessors.push(createIncludeLabelsProcessor(options?.includeLabels)); - } - const bundleRefsProcessor = new BundleRefProcessor(X_INLINE); const removeUnusedComponentsProcessor = new RemoveUnusedComponentsProcessor(); await processDocument(resolvedDocument, refResolver, [ - ...defaultProcessors, + ...processors, bundleRefsProcessor, removeUnusedComponentsProcessor, ]); @@ -111,8 +80,6 @@ export async function bundleDocument( ); } - injectBundledRefs(resolvedDocument, bundleRefsProcessor.getBundledRefs()); - return resolvedDocument; } @@ -127,12 +94,3 @@ function hasPaths(document: MaybeObjectWithPaths): boolean { Object.keys(document.paths).length > 0 ); } - -function injectBundledRefs( - resolvedDocument: ResolvedDocument, - refs: IterableIterator -): void { - for (const ref of refs) { - insertRefByPointer(ref.pointer, ref.refNode, resolvedDocument.document); - } -} diff --git a/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_documents.ts b/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_documents.ts index 8e745c50ac679..24e507bd0e283 100644 --- a/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_documents.ts +++ b/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_documents.ts @@ -46,7 +46,7 @@ export async function mergeDocuments( // is the simplest way to take initial components into account. const documentsToMerge = [ { - absolutePath: 'MERGED OpenAPI SPEC', + absolutePath: 'MERGED RESULT', document: mergedDocument as unknown as ResolvedDocument['document'], }, ...documentsGroup, diff --git a/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_shared_components.ts b/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_shared_components.ts index f38341d3e0f94..2efddc09e1de0 100644 --- a/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_shared_components.ts +++ b/packages/kbn-openapi-bundler/src/bundler/merge_documents/merge_shared_components.ts @@ -10,7 +10,7 @@ import chalk from 'chalk'; import deepEqual from 'fast-deep-equal'; import { OpenAPIV3 } from 'openapi-types'; import { ResolvedDocument } from '../ref_resolver/resolved_document'; -import { extractByJsonPointer } from '../../utils/extract_by_json_pointer'; +import { extractObjectByJsonPointer } from '../../utils/extract_by_json_pointer'; import { logger } from '../../logger'; const MERGEABLE_COMPONENT_TYPES = [ @@ -34,7 +34,7 @@ export function mergeSharedComponents( const mergedTypedComponents = mergeObjects(bundledDocuments, `/components/${componentsType}`); if (Object.keys(mergedTypedComponents).length === 0) { - // Nothing was merged for this components type, go to the next component type + // Nothing was merged for that components type, go to the next component type continue; } @@ -63,26 +63,21 @@ function mergeObjects( const componentToAdd = object[name]; const existingComponent = merged[name]; - if (existingComponent) { + // Bundled documents may contain explicit references duplicates. For example + // shared schemas from `@kbn/openapi-common` has `NonEmptyString` which is + // widely used. After bundling references into a document (i.e. making them + // local references) we will have duplicates. This is why we need to check + // for exact match via `deepEqual()` to check whether components match. + if (existingComponent && !deepEqual(componentToAdd, existingComponent)) { const existingSchemaLocation = componentNameSourceLocationMap.get(name); - if (deepEqual(componentToAdd, existingComponent)) { - logger.warning( - `Found a duplicate component ${chalk.yellow( - `${sourcePointer}/${name}` - )} defined in ${chalk.blue(resolvedDocument.absolutePath)} and in ${chalk.magenta( - existingSchemaLocation - )}.` - ); - } else { - throw new Error( - `❌ Unable to merge documents due to conflicts in referenced ${mergedEntityName}. Component ${chalk.yellow( - `${sourcePointer}/${name}` - )} is defined in ${chalk.blue(resolvedDocument.absolutePath)} and in ${chalk.magenta( - existingSchemaLocation - )} but has not matching definitions.` - ); - } + throw new Error( + `❌ Unable to merge documents due to conflicts in referenced ${mergedEntityName}. Component ${chalk.yellow( + `${sourcePointer}/${name}` + )} is defined in ${chalk.blue(resolvedDocument.absolutePath)} and in ${chalk.magenta( + existingSchemaLocation + )} but definitions DO NOT match.` + ); } merged[name] = componentToAdd; @@ -98,9 +93,9 @@ function extractObjectToMerge( sourcePointer: string ): Record | undefined { try { - return extractByJsonPointer(resolvedDocument.document, sourcePointer); + return extractObjectByJsonPointer(resolvedDocument.document, sourcePointer); } catch (e) { - logger.debug( + logger.verbose( `JSON pointer "${sourcePointer}" is not resolvable in ${resolvedDocument.absolutePath}` ); return; diff --git a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/bundle_refs.ts b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/bundle_refs.ts index 76da77cc06b0e..0608ad945e01d 100644 --- a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/bundle_refs.ts +++ b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/bundle_refs.ts @@ -8,6 +8,7 @@ import deepEqual from 'fast-deep-equal'; import chalk from 'chalk'; +import { parseRef } from '../../../utils/parse_ref'; import { hasProp } from '../../../utils/has_prop'; import { isChildContext } from '../is_child_context'; import { insertRefByPointer } from '../../../utils/insert_by_json_pointer'; @@ -59,11 +60,6 @@ export class BundleRefProcessor implements DocumentNodeProcessor { inlineRef(node, resolvedRef); } else { const rootDocument = this.extractRootDocument(context); - - if (!rootDocument.components) { - rootDocument.components = {}; - } - const ref = this.refs.get(resolvedRef.pointer); if (ref && !deepEqual(ref.refNode, resolvedRef.refNode)) { @@ -77,16 +73,18 @@ export class BundleRefProcessor implements DocumentNodeProcessor { ref.pointer )} is defined in ${chalk.blue(ref.absolutePath)} and in ${chalk.magenta( resolvedRef.absolutePath - )} but has not matching definitions.` + )} but definitions DO NOT match.` ); } - node.$ref = this.saveComponent( - resolvedRef, - rootDocument.components as Record - ); + // Ref pointer might be modified by previous processors + // resolvedRef.pointer always has the original value + // while node.$ref might have updated + const currentRefPointer = parseRef(node.$ref).pointer; + + node.$ref = this.saveComponent(currentRefPointer, resolvedRef.refNode, rootDocument); - this.refs.set(resolvedRef.pointer, resolvedRef); + this.refs.set(currentRefPointer, resolvedRef); } } @@ -94,10 +92,10 @@ export class BundleRefProcessor implements DocumentNodeProcessor { return this.refs.values(); } - private saveComponent(ref: ResolvedRef, components: Record): string { - insertRefByPointer(ref.pointer, ref.refNode, components); + private saveComponent(pointer: string, refNode: DocumentNode, document: Document): string { + insertRefByPointer(pointer, refNode, document); - return `#${ref.pointer}`; + return `#${pointer}`; } private extractParentContext(context: TraverseDocumentContext): TraverseRootDocumentContext { diff --git a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.test.ts b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.test.ts new file mode 100644 index 0000000000000..88dbe37b0f67c --- /dev/null +++ b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createNamespaceComponentsProcessor } from './namespace_components'; + +describe('namespaceComponentsProcessor', () => { + it.each([ + { + sourceValue: 'Something', + originalRef: '#/components/schemas/SomeComponent', + expectedRef: '#/components/schemas/Something_SomeComponent', + }, + { + sourceValue: 'Some Domain API (Extra Information)', + originalRef: '#/components/schemas/SomeComponent', + expectedRef: '#/components/schemas/Some_Domain_API_SomeComponent', + }, + { + sourceValue: 'Hello, world!', + originalRef: '#/components/schemas/SomeComponent', + expectedRef: '#/components/schemas/Hello_world_SomeComponent', + }, + { + sourceValue: 'Something', + originalRef: '../path/to/some.schema.yaml#/components/schemas/SomeComponent', + expectedRef: '../path/to/some.schema.yaml#/components/schemas/Something_SomeComponent', + }, + { + sourceValue: 'Some Domain API (Extra Information)', + originalRef: '../path/to/some.schema.yaml#/components/schemas/SomeComponent', + expectedRef: '../path/to/some.schema.yaml#/components/schemas/Some_Domain_API_SomeComponent', + }, + { + sourceValue: 'Hello, world!', + originalRef: '../path/to/some.schema.yaml#/components/schemas/SomeComponent', + expectedRef: '../path/to/some.schema.yaml#/components/schemas/Hello_world_SomeComponent', + }, + ])( + 'prefixes reference "$originalRef" with normalized "$sourceValue"', + ({ sourceValue, originalRef, expectedRef }) => { + const processor = createNamespaceComponentsProcessor('/info/title'); + + const document = { + info: { + title: sourceValue, + }, + }; + + processor.onNodeEnter?.(document, { + resolvedDocument: { absolutePath: '', document }, + isRootNode: true, + parentNode: document, + parentKey: '', + }); + + const node = { $ref: originalRef }; + + processor.onRefNodeLeave?.( + node, + { pointer: '', refNode: {}, absolutePath: '', document: {} }, + { + resolvedDocument: { absolutePath: '', document }, + isRootNode: false, + parentNode: document, + parentKey: '', + } + ); + + expect(node).toMatchObject({ + $ref: expectedRef, + }); + } + ); + + it('prefixes security requirements', () => { + const processor = createNamespaceComponentsProcessor('/info/title'); + + const document = { + info: { + title: 'Something', + }, + }; + + processor.onNodeEnter?.(document, { + resolvedDocument: { absolutePath: '', document }, + isRootNode: true, + parentNode: document, + parentKey: '', + }); + + const node = { security: [{ SomeSecurityRequirement: [] }] }; + + processor.onNodeLeave?.(node, { + resolvedDocument: { absolutePath: '', document }, + isRootNode: false, + parentNode: document, + parentKey: '', + }); + + // eslint-disable-next-line @typescript-eslint/naming-convention + expect(node).toMatchObject({ security: [{ Something_SomeSecurityRequirement: [] }] }); + }); + + it('prefixes security requirement components', () => { + const processor = createNamespaceComponentsProcessor('/info/title'); + + const document = { + info: { + title: 'Something', + }, + components: { + securitySchemes: { + BasicAuth: { + scheme: 'basic', + type: 'http', + }, + }, + }, + }; + + processor.onNodeEnter?.(document, { + resolvedDocument: { absolutePath: '', document }, + isRootNode: true, + parentNode: document, + parentKey: '', + }); + + processor.onNodeLeave?.(document, { + resolvedDocument: { absolutePath: '', document }, + isRootNode: true, + parentNode: document, + parentKey: '', + }); + + expect(document.components.securitySchemes).toMatchObject({ + // eslint-disable-next-line @typescript-eslint/naming-convention + Something_BasicAuth: { + scheme: 'basic', + type: 'http', + }, + }); + }); +}); diff --git a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.ts b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.ts new file mode 100644 index 0000000000000..01ef116082fe1 --- /dev/null +++ b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/namespace_components.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { extractByJsonPointer } from '../../../utils/extract_by_json_pointer'; +import { isPlainObjectType } from '../../../utils/is_plain_object_type'; +import { parseRef } from '../../../utils/parse_ref'; +import { DocumentNodeProcessor } from './types/document_node_processor'; + +/** + * Creates a node processor to prefix possibly conflicting components and security requirements + * with a string considered as a `namespace`. Namespace value is extracted from the document by + * the provided JSON pointer. + */ +export function createNamespaceComponentsProcessor(pointer: string): DocumentNodeProcessor { + let namespace = ''; + + const prefixObjectKeys = (obj: Record): void => { + for (const name of Object.keys(obj)) { + if (name.startsWith(namespace)) { + continue; + } + + obj[`${namespace}_${name}`] = obj[name]; + delete obj[name]; + } + }; + + return { + onNodeEnter(node, context) { + // Skip non root nodes and referenced documents + if (!context.isRootNode || context.parentContext) { + return; + } + + const extractedNamespace = extractByJsonPointer(node, pointer); + + if (typeof extractedNamespace !== 'string') { + throw new Error(`"${pointer}" should resolve to a non empty string`); + } + + namespace = normalizeNamespace(extractedNamespace); + + if (extractedNamespace.trim() === '') { + throw new Error(`Namespace becomes an empty string after normalization`); + } + }, + onRefNodeLeave(node) { + // It's enough to decorate the base name and actual object manipulation + // will happen at bundling refs stage + node.$ref = decorateRefBaseName(node.$ref, namespace); + }, + // Items used in root level `security` values must match a scheme defined in the + // `components.securitySchemes`. It means items in `security` implicitly reference + // `components.securitySchemes` items which should be handled. + onNodeLeave(node, context) { + if ('security' in node && Array.isArray(node.security)) { + for (const securityRequirements of node.security) { + prefixObjectKeys(securityRequirements); + } + } + + if ( + context.isRootNode && + isPlainObjectType(node) && + isPlainObjectType(node.components) && + isPlainObjectType(node.components.securitySchemes) + ) { + prefixObjectKeys(node.components.securitySchemes); + } + }, + }; +} + +/** + * Adds provided `prefix` to the provided `ref`'s base name. Where `ref`'s + * base name is the last part of JSON Pointer representing a component name. + * + * @example + * + * Given + * + * `ref` = `../some/path/to/my.schema.yaml#/components/schema/MyComponent` + * `prefix` = `Some_Prefix` + * + * it will produce `../some/path/to/my.schema.yaml#/components/schema/Some_Prefix_MyComponent` + * + * Given + * + * `ref` = `#/components/responses/SomeResponse` + * `prefix` = `Prefix` + * + * it will produce `#/components/responses/Prefix_SomeResponse` + */ +function decorateRefBaseName(ref: string, prefix: string): string { + const { path, pointer } = parseRef(ref); + const pointerParts = pointer.split('/'); + const refName = pointerParts.pop()!; + + if (refName.startsWith(prefix)) { + return ref; + } + + return `${path}#${pointerParts.join('/')}/${prefix}_${refName}`; +} + +const PARENTHESES_INFO_REGEX = /\(.+\)+/g; +const ALPHANUMERIC_SYMBOLS_REGEX = /[^\w\n]+/g; +const SPACES_REGEX = /\s+/g; + +/** + * Normalizes provided `namespace` string by + * + * - getting rid of non alphanumeric symbols + * - getting rid of parentheses including text between them + * - collapsing and replacing spaces with underscores + * + * @example + * + * Given a namespace `Some Domain API (Extra Information)` + * it will produce `Security_Solution_Detections_API` + * + * Given a namespace `Hello, world!` + * it will produce `Hello_world` + * + */ +function normalizeNamespace(namespace: string): string { + // Using two replaceAll() to make sure there is no leading or trailing underscores + return namespace + .replaceAll(PARENTHESES_INFO_REGEX, ' ') + .replaceAll(ALPHANUMERIC_SYMBOLS_REGEX, ' ') + .trim() + .replaceAll(SPACES_REGEX, '_'); +} diff --git a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/remove_unused_components.ts b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/remove_unused_components.ts index f0bb72c34644c..5fbe794ad1dd2 100644 --- a/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/remove_unused_components.ts +++ b/packages/kbn-openapi-bundler/src/bundler/process_document/document_processors/remove_unused_components.ts @@ -6,10 +6,9 @@ * Side Public License, v 1. */ -import { hasProp } from '../../../utils/has_prop'; +import { parseRef } from '../../../utils/parse_ref'; import { isPlainObjectType } from '../../../utils/is_plain_object_type'; -import { ResolvedRef } from '../../ref_resolver/resolved_ref'; -import { PlainObjectNode } from '../types/node'; +import { DocumentNode, PlainObjectNode, RefNode } from '../types/node'; import { DocumentNodeProcessor } from './types/document_node_processor'; /** @@ -22,24 +21,61 @@ import { DocumentNodeProcessor } from './types/document_node_processor'; export class RemoveUnusedComponentsProcessor implements DocumentNodeProcessor { private refs = new Set(); - onRefNodeLeave(node: unknown, resolvedRef: ResolvedRef): void { - // If the reference has been inlined by one of the previous processors skip it - if (!hasProp(node, '$ref')) { + onRefNodeLeave(node: RefNode): void { + // Ref pointer might be modified by previous processors + // resolvedRef.pointer always has the original value + // while node.$ref might have updated + const currentRefPointer = parseRef(node.$ref).pointer; + + this.refs.add(currentRefPointer); + } + + // `security` entries implicitly refer security schemas + onNodeLeave(node: DocumentNode): void { + if (!hasSecurityRequirements(node)) { return; } - this.refs.add(resolvedRef.pointer); + for (const securityRequirementObj of node.security) { + if (!isPlainObjectType(securityRequirementObj)) { + continue; + } + + for (const securityRequirementName of Object.keys(securityRequirementObj)) { + this.refs.add(`/components/securitySchemes/${securityRequirementName}`); + } + } } removeUnusedComponents(components: PlainObjectNode): void { - if (!isPlainObjectType(components.schemas)) { - return; - } + for (const collectionName of COMPONENTS_TO_CLEAN) { + const objectsCollection = components?.[collectionName]; + + if (!isPlainObjectType(objectsCollection)) { + continue; + } - for (const schema of Object.keys(components.schemas)) { - if (!this.refs.has(`/components/schemas/${schema}`)) { - delete components.schemas[schema]; + for (const schema of Object.keys(objectsCollection)) { + if (!this.refs.has(`/components/${collectionName}/${schema}`)) { + delete objectsCollection[schema]; + } } } } } + +function hasSecurityRequirements(node: DocumentNode): node is { security: unknown[] } { + return 'security' in node && Array.isArray(node.security); +} + +const COMPONENTS_TO_CLEAN = [ + 'schemas', + 'responses', + 'parameters', + 'examples', + 'requestBodies', + 'headers', + 'securitySchemes', + 'links', + 'callbacks', +]; diff --git a/packages/kbn-openapi-bundler/src/bundler/processor_sets.ts b/packages/kbn-openapi-bundler/src/bundler/processor_sets.ts new file mode 100644 index 0000000000000..a87f473ee6031 --- /dev/null +++ b/packages/kbn-openapi-bundler/src/bundler/processor_sets.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { X_CODEGEN_ENABLED, X_INLINE, X_INTERNAL, X_LABELS, X_MODIFY } from './known_custom_props'; +import { createSkipNodeWithInternalPropProcessor } from './process_document/document_processors/skip_node_with_internal_prop'; +import { createSkipInternalPathProcessor } from './process_document/document_processors/skip_internal_path'; +import { createModifyPartialProcessor } from './process_document/document_processors/modify_partial'; +import { createModifyRequiredProcessor } from './process_document/document_processors/modify_required'; +import { createRemovePropsProcessor } from './process_document/document_processors/remove_props'; +import { + createFlattenFoldedAllOfItemsProcessor, + createMergeNonConflictingAllOfItemsProcessor, + createUnfoldSingleAllOfItemProcessor, +} from './process_document/document_processors/reduce_all_of_items'; +import { DocumentNodeProcessor } from './process_document/document_processors/types/document_node_processor'; +import { createIncludeLabelsProcessor } from './process_document/document_processors/include_labels'; +import { createNamespaceComponentsProcessor } from './process_document/document_processors/namespace_components'; + +/** + * Document modification includes the following + * - skips nodes with `x-internal: true` property + * - skips paths started with `/internal` + * - modifies nodes having `x-modify` + */ +export const DEFAULT_BUNDLING_PROCESSORS: Readonly = [ + createSkipNodeWithInternalPropProcessor(X_INTERNAL), + createSkipInternalPathProcessor('/internal'), + createModifyPartialProcessor(), + createModifyRequiredProcessor(), + createRemovePropsProcessor([X_INLINE, X_MODIFY, X_CODEGEN_ENABLED, X_LABELS]), + createFlattenFoldedAllOfItemsProcessor(), + createMergeNonConflictingAllOfItemsProcessor(), + createUnfoldSingleAllOfItemProcessor(), +]; + +/** + * Adds createIncludeLabelsProcessor processor, see createIncludeLabelsProcessor description + * for more details + */ +export function withIncludeLabelsProcessor( + processors: Readonly, + includeLabels: string[] +): Readonly { + return [...processors, createIncludeLabelsProcessor(includeLabels)]; +} + +export function withNamespaceComponentsProcessor( + processors: Readonly, + namespacePointer: string +): Readonly { + return [...processors, createNamespaceComponentsProcessor(namespacePointer)]; +} diff --git a/packages/kbn-openapi-bundler/src/bundler/ref_resolver/ref_resolver.ts b/packages/kbn-openapi-bundler/src/bundler/ref_resolver/ref_resolver.ts index 38ce43cf1d593..cc548b86eab93 100644 --- a/packages/kbn-openapi-bundler/src/bundler/ref_resolver/ref_resolver.ts +++ b/packages/kbn-openapi-bundler/src/bundler/ref_resolver/ref_resolver.ts @@ -7,8 +7,8 @@ */ import path from 'path'; -import { extractByJsonPointer } from '../../utils/extract_by_json_pointer'; -import { readYamlDocument } from '../../utils/read_yaml_document'; +import { extractObjectByJsonPointer } from '../../utils/extract_by_json_pointer'; +import { readDocument } from '../../utils/read_document'; import { ResolvedRef } from './resolved_ref'; import { ResolvedDocument } from './resolved_document'; @@ -22,7 +22,7 @@ export class RefResolver implements IRefResolver { async resolveRef(refDocumentAbsolutePath: string, pointer: string): Promise { const resolvedRefDocument = await this.resolveDocument(refDocumentAbsolutePath); - const refNode = extractByJsonPointer(resolvedRefDocument.document, pointer); + const refNode = extractObjectByJsonPointer(resolvedRefDocument.document, pointer); const resolvedRef = { absolutePath: refDocumentAbsolutePath, pointer, @@ -47,7 +47,7 @@ export class RefResolver implements IRefResolver { } try { - const document = await readYamlDocument(documentAbsolutePath); + const document = await readDocument(documentAbsolutePath); const resolvedRef = { absolutePath: documentAbsolutePath, document, diff --git a/packages/kbn-openapi-bundler/src/openapi_bundler.ts b/packages/kbn-openapi-bundler/src/openapi_bundler.ts index d4a19d2806863..90382150400dd 100644 --- a/packages/kbn-openapi-bundler/src/openapi_bundler.ts +++ b/packages/kbn-openapi-bundler/src/openapi_bundler.ts @@ -18,6 +18,7 @@ import { createBlankOpenApiDocument } from './bundler/merge_documents/create_bla import { writeDocuments } from './utils/write_documents'; import { ResolvedDocument } from './bundler/ref_resolver/resolved_document'; import { resolveGlobs } from './utils/resolve_globs'; +import { DEFAULT_BUNDLING_PROCESSORS, withIncludeLabelsProcessor } from './bundler/processor_sets'; export interface BundlerConfig { sourceGlob: string; @@ -51,9 +52,9 @@ export const bundle = async ({ logger.debug(`Processing schemas...`); - const resolvedDocuments = await resolveDocuments(schemaFilePaths, options); + const bundledDocuments = await bundleDocuments(schemaFilePaths, options); - logger.success(`Processed ${resolvedDocuments.length} schemas`); + logger.success(`Processed ${bundledDocuments.length} schemas`); const blankOasFactory = (oasVersion: string, apiVersion: string) => createBlankOpenApiDocument(oasVersion, { @@ -69,7 +70,7 @@ export const bundle = async ({ isUndefined ), }); - const resultDocumentsMap = await mergeDocuments(resolvedDocuments, blankOasFactory, { + const resultDocumentsMap = await mergeDocuments(bundledDocuments, blankOasFactory, { splitDocumentsByVersion: true, }); @@ -82,16 +83,19 @@ function logSchemas(schemaFilePaths: string[]): void { } } -async function resolveDocuments( +async function bundleDocuments( schemaFilePaths: string[], options?: BundleOptions ): Promise { const resolvedDocuments = await Promise.all( schemaFilePaths.map(async (schemaFilePath) => { try { - const resolvedDocument = await bundleDocument(schemaFilePath, { - includeLabels: options?.includeLabels, - }); + const resolvedDocument = await bundleDocument( + schemaFilePath, + options?.includeLabels + ? withIncludeLabelsProcessor(DEFAULT_BUNDLING_PROCESSORS, options.includeLabels) + : DEFAULT_BUNDLING_PROCESSORS + ); logger.debug(`Processed ${chalk.bold(basename(schemaFilePath))}`); diff --git a/packages/kbn-openapi-bundler/src/openapi_merger.ts b/packages/kbn-openapi-bundler/src/openapi_merger.ts index c5e67fe74f221..773edb816c472 100644 --- a/packages/kbn-openapi-bundler/src/openapi_merger.ts +++ b/packages/kbn-openapi-bundler/src/openapi_merger.ts @@ -8,33 +8,37 @@ import chalk from 'chalk'; import { OpenAPIV3 } from 'openapi-types'; -import { basename, extname } from 'path'; import { mergeDocuments } from './bundler/merge_documents'; import { logger } from './logger'; import { createBlankOpenApiDocument } from './bundler/merge_documents/create_blank_oas_document'; -import { readYamlDocument } from './utils/read_yaml_document'; -import { readJsonDocument } from './utils/read_json_document'; import { ResolvedDocument } from './bundler/ref_resolver/resolved_document'; import { writeDocuments } from './utils/write_documents'; import { resolveGlobs } from './utils/resolve_globs'; +import { bundleDocument } from './bundler/bundle_document'; +import { withNamespaceComponentsProcessor } from './bundler/processor_sets'; export interface MergerConfig { sourceGlobs: string[]; outputFilePath: string; - mergedSpecInfo?: Partial; + options?: { + mergedSpecInfo?: Partial; + conflictsResolution?: { + prependComponentsWith: 'title'; + }; + }; } export const merge = async ({ sourceGlobs, outputFilePath = 'merged.schema.yaml', - mergedSpecInfo, + options, }: MergerConfig) => { if (sourceGlobs.length < 1) { throw new Error('As minimum one source glob is expected'); } - logger.debug(chalk.bold(`Merging OpenAPI specs`)); - logger.debug( + logger.info(chalk.bold(`Merging OpenAPI specs`)); + logger.info( `👀 Searching for source files in ${sourceGlobs .map((glob) => chalk.underline(glob)) .join(', ')}` @@ -45,17 +49,18 @@ export const merge = async ({ logger.info(`🕵️‍♀️ Found ${schemaFilePaths.length} schemas`); logSchemas(schemaFilePaths); - logger.debug(`Merging schemas...`); + logger.info(`Merging schemas...`); - const resolvedDocuments = await resolveDocuments(schemaFilePaths); + const bundledDocuments = await bundleDocuments(schemaFilePaths); const blankOasDocumentFactory = (oasVersion: string) => createBlankOpenApiDocument(oasVersion, { title: 'Merged OpenAPI specs', version: 'not specified', - ...mergedSpecInfo, + ...(options?.mergedSpecInfo ?? {}), }); - const resultDocumentsMap = await mergeDocuments(resolvedDocuments, blankOasDocumentFactory, { + + const resultDocumentsMap = await mergeDocuments(bundledDocuments, blankOasDocumentFactory, { splitDocumentsByVersion: false, }); // Only one document is expected when `splitDocumentsByVersion` is set to `false` @@ -71,32 +76,10 @@ function logSchemas(schemaFilePaths: string[]): void { } } -async function resolveDocuments(schemaFilePaths: string[]): Promise { - const resolvedDocuments = await Promise.all( - schemaFilePaths.map(async (schemaFilePath) => { - const extension = extname(schemaFilePath); - - logger.debug(`Reading ${chalk.bold(basename(schemaFilePath))}`); - - switch (extension) { - case '.yaml': - case '.yml': - return { - absolutePath: schemaFilePath, - document: await readYamlDocument(schemaFilePath), - }; - - case '.json': - return { - absolutePath: schemaFilePath, - document: await readJsonDocument(schemaFilePath), - }; - - default: - throw new Error(`${extension} files are not supported`); - } - }) +async function bundleDocuments(schemaFilePaths: string[]): Promise { + return await Promise.all( + schemaFilePaths.map(async (schemaFilePath) => + bundleDocument(schemaFilePath, withNamespaceComponentsProcessor([], '/info/title')) + ) ); - - return resolvedDocuments; } diff --git a/packages/kbn-openapi-bundler/src/utils/extract_by_json_pointer.ts b/packages/kbn-openapi-bundler/src/utils/extract_by_json_pointer.ts index 937f708e0ce87..3826621591866 100644 --- a/packages/kbn-openapi-bundler/src/utils/extract_by_json_pointer.ts +++ b/packages/kbn-openapi-bundler/src/utils/extract_by_json_pointer.ts @@ -6,22 +6,16 @@ * Side Public License, v 1. */ +import chalk from 'chalk'; +import { dump } from 'js-yaml'; import { isPlainObjectType } from './is_plain_object_type'; /** - * Extract a node from a document using a provided [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901). + * Extract a value from a document using provided [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901). * - * JSON Pointer is the second part in [JSON Reference](https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03). - * For example an object `{ $ref: "./some-file.yaml#/components/schemas/MySchema"}` is a reference node. - * Where `/components/schemas/MySchema` is a JSON pointer. `./some-file.yaml` is a document reference. - * Yaml shares the same JSON reference standard and basically can be considered just as a different - * JS Object serialization format. See OpenAPI [Using $ref](https://swagger.io/docs/specification/using-ref/) for more information. - * - * @param document a document containing node to resolve by using the pointer - * @param pointer a JSON Pointer - * @returns resolved document node + * The final value type is not validated so it's responsibility of the outer code. */ -export function extractByJsonPointer(document: unknown, pointer: string): Record { +export function extractByJsonPointer(document: unknown, pointer: string): unknown { if (!pointer.startsWith('/')) { throw new Error('JSON pointer must start with a leading slash'); } @@ -30,19 +24,51 @@ export function extractByJsonPointer(document: unknown, pointer: string): Record throw new Error('document must be an object'); } - let target = document; + const path: string[] = ['']; + let target: unknown = document; for (const segment of pointer.slice(1).split('/')) { - const nextTarget = target[segment]; - - if (!isPlainObjectType(nextTarget)) { + if (!isPlainObjectType(target)) { throw new Error( - `JSON Pointer "${pointer}" is not resolvable in "${JSON.stringify(document)}"` + `JSON Pointer ${chalk.bold(pointer)} resolution failure. Expected ${chalk.magenta( + path.join('/') + )} to be a plain object but it has type "${typeof target}" in \n\n${dump(document)}` ); } - target = nextTarget; + path.push(segment); + target = target[segment]; } return target; } + +/** + * Extract a node from a document using provided [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901). + * + * JSON Pointer is the second part in [JSON Reference](https://datatracker.ietf.org/doc/html/draft-pbryan-zyp-json-ref-03). + * For example an object `{ $ref: "./some-file.yaml#/components/schemas/MySchema"}` is a reference node. + * Where `/components/schemas/MySchema` is a JSON pointer. `./some-file.yaml` is a document reference. + * Yaml shares the same JSON reference standard and basically can be considered just as a different + * JS Object serialization format. See OpenAPI [Using $ref](https://swagger.io/docs/specification/using-ref/) for more information. + * + * @param document a document containing node to resolve by using the pointer + * @param pointer a JSON Pointer + * @returns resolved document node + */ +export function extractObjectByJsonPointer( + document: unknown, + pointer: string +): Record { + const maybeObject = extractByJsonPointer(document, pointer); + + if (!isPlainObjectType(maybeObject)) { + throw new Error( + `JSON Pointer resolution failure. Expected ${chalk.magenta( + pointer + )} to be a plain object in \n\n${dump(document)}` + ); + } + + return maybeObject; +} diff --git a/packages/kbn-openapi-bundler/src/utils/insert_by_json_pointer.ts b/packages/kbn-openapi-bundler/src/utils/insert_by_json_pointer.ts index c7228f8d03de0..b5a7593a50dc0 100644 --- a/packages/kbn-openapi-bundler/src/utils/insert_by_json_pointer.ts +++ b/packages/kbn-openapi-bundler/src/utils/insert_by_json_pointer.ts @@ -18,13 +18,13 @@ export function insertRefByPointer( component: unknown, targetObject: Record ): void { - if (!pointer.startsWith('/components')) { + if (!pointer.startsWith('/')) { throw new Error( `insertRefByPointer expected a pointer starting with "/components" but got ${pointer}` ); } - // splitting '/components/some/path' by '/' gives ['', 'components'...] + // splitting '/components/some/path' by '/' gives ['', 'components',...] // where the first empty string should be skipped const segments = pointer.split('/').slice(1); let target = targetObject; diff --git a/packages/kbn-openapi-bundler/src/utils/read_document.ts b/packages/kbn-openapi-bundler/src/utils/read_document.ts new file mode 100644 index 0000000000000..019f5103cf621 --- /dev/null +++ b/packages/kbn-openapi-bundler/src/utils/read_document.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import fs from 'fs/promises'; +import { load } from 'js-yaml'; +import { basename, extname } from 'path'; +import chalk from 'chalk'; +import { logger } from '../logger'; + +export async function readDocument(documentPath: string): Promise> { + const extension = extname(documentPath); + + logger.debug(`Reading ${chalk.bold(basename(documentPath))}`); + + switch (extension) { + case '.yaml': + case '.yml': + return await readYamlDocument(documentPath); + + case '.json': + return await readJsonDocument(documentPath); + + default: + throw new Error(`${extension} files are not supported`); + } +} + +async function readYamlDocument(filePath: string): Promise> { + // Typing load's result to Record is optimistic as we can't be sure + // there is object inside a yaml file. We don't have this validation layer so far + // but using JSON Schemas here should mitigate this problem. + return load(await fs.readFile(filePath, { encoding: 'utf8' })); +} + +export async function readJsonDocument(filePath: string): Promise> { + // Typing load's result to Record is optimistic as we can't be sure + // there is object inside a yaml file. We don't have this validation layer so far + // but using JSON Schemas here should mitigate this problem. + return await JSON.parse(await fs.readFile(filePath, { encoding: 'utf8' })); +} diff --git a/packages/kbn-openapi-bundler/src/utils/read_json_document.ts b/packages/kbn-openapi-bundler/src/utils/read_json_document.ts deleted file mode 100644 index 61ce61c6df3d8..0000000000000 --- a/packages/kbn-openapi-bundler/src/utils/read_json_document.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import fs from 'fs/promises'; - -export async function readJsonDocument(filePath: string): Promise> { - // Typing load's result to Record is optimistic as we can't be sure - // there is object inside a yaml file. We don't have this validation layer so far - // but using JSON Schemas here should mitigate this problem. - return await JSON.parse(await fs.readFile(filePath, { encoding: 'utf8' })); -} diff --git a/packages/kbn-openapi-bundler/src/utils/read_yaml_document.ts b/packages/kbn-openapi-bundler/src/utils/read_yaml_document.ts deleted file mode 100644 index c8cbae710c1ba..0000000000000 --- a/packages/kbn-openapi-bundler/src/utils/read_yaml_document.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import fs from 'fs/promises'; -import { load } from 'js-yaml'; - -export async function readYamlDocument(filePath: string): Promise> { - // Typing load's result to Record is optimistic as we can't be sure - // there is object inside a yaml file. We don't have this validation layer so far - // but using JSON Schemas here should mitigate this problem. - return load(await fs.readFile(filePath, { encoding: 'utf8' })); -} diff --git a/packages/kbn-openapi-bundler/src/utils/write_yaml_document.ts b/packages/kbn-openapi-bundler/src/utils/write_yaml_document.ts index 99872292804be..980f898777aef 100644 --- a/packages/kbn-openapi-bundler/src/utils/write_yaml_document.ts +++ b/packages/kbn-openapi-bundler/src/utils/write_yaml_document.ts @@ -23,22 +23,21 @@ export async function writeYamlDocument(filePath: string, document: unknown): Pr function stringifyToYaml(document: unknown): string { try { + // We don't want to have `undefined` values serialized into YAML. + // `JSON.stringify()` simply skips `undefined` values while js-yaml v 3.14 DOES NOT. + // js-yaml >= v4 has it fixed so `dump()`'s behavior is consistent with `JSON.stringify()`. + // Until js-yaml is updated to v4 use the hack with JSON serialization/deserialization. + const clearedDocument = JSON.parse(JSON.stringify(document)); + // Disable YAML Anchors https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases // It makes YAML much more human readable - return dump(document, { + return dump(clearedDocument, { noRefs: true, sortKeys: sortYamlKeys, }); } catch (e) { - // RangeError might happened because of stack overflow - // due to circular references in the document - // since YAML Anchors are disabled - if (e instanceof RangeError) { - // Try to stringify with YAML Anchors enabled - return dump(document, { noRefs: false, sortKeys: sortYamlKeys }); - } - - throw e; + // Try to stringify with YAML Anchors enabled + return dump(document, { noRefs: false, sortKeys: sortYamlKeys }); } } diff --git a/packages/kbn-openapi-bundler/tests/create_oas_document.ts b/packages/kbn-openapi-bundler/tests/create_oas_document.ts index c91e535e0cf12..a16cef4119f03 100644 --- a/packages/kbn-openapi-bundler/tests/create_oas_document.ts +++ b/packages/kbn-openapi-bundler/tests/create_oas_document.ts @@ -13,8 +13,10 @@ export function createOASDocument(overrides: { info?: Partial; paths?: OpenAPIV3.PathsObject; components?: OpenAPIV3.ComponentsObject; + servers?: OpenAPIV3.ServerObject[]; + security?: OpenAPIV3.SecurityRequirementObject[]; }): OpenAPIV3.Document { - return { + const document: OpenAPIV3.Document = { openapi: overrides.openapi ?? '3.0.3', info: { title: 'Test endpoint', @@ -28,4 +30,14 @@ export function createOASDocument(overrides: { ...overrides.components, }, }; + + if (overrides.servers) { + document.servers = overrides.servers; + } + + if (overrides.security) { + document.security = overrides.security; + } + + return document; } diff --git a/packages/kbn-openapi-bundler/tests/merger/different_oas_versions.test.ts b/packages/kbn-openapi-bundler/tests/merger/different_oas_versions.test.ts index 83af0016236d7..3c3324a7254ed 100644 --- a/packages/kbn-openapi-bundler/tests/merger/different_oas_versions.test.ts +++ b/packages/kbn-openapi-bundler/tests/merger/different_oas_versions.test.ts @@ -13,11 +13,15 @@ describe('OpenAPI Merger - different OpenAPI versions', () => { it('merges specs having OpenAPI 3.0.x versions', async () => { const spec1 = createOASDocument({ openapi: '3.0.3', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); const spec2 = createOASDocument({ openapi: '3.0.0', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); const [mergedSpec] = Object.values( @@ -33,11 +37,15 @@ describe('OpenAPI Merger - different OpenAPI versions', () => { it('throws an error when different minor OAS versions encountered', async () => { const spec1 = createOASDocument({ openapi: '3.0.3', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); const spec2 = createOASDocument({ openapi: '3.1.0', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); expect( @@ -51,11 +59,15 @@ describe('OpenAPI Merger - different OpenAPI versions', () => { it('throws an error when different OAS 3.1.x patch versions encountered', async () => { const spec1 = createOASDocument({ openapi: '3.1.0', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); const spec2 = createOASDocument({ openapi: '3.1.1', - paths: {}, + paths: { + '/api/some/path': {}, + }, }); expect( diff --git a/packages/kbn-openapi-bundler/tests/merger/merge_specs.ts b/packages/kbn-openapi-bundler/tests/merger/merge_specs.ts index 743b40373053c..0455bb1088369 100644 --- a/packages/kbn-openapi-bundler/tests/merger/merge_specs.ts +++ b/packages/kbn-openapi-bundler/tests/merger/merge_specs.ts @@ -27,7 +27,7 @@ jest.mock('../../src/logger'); export async function mergeSpecs( oasSpecs: Record, - mergedSpecInfo?: MergerConfig['mergedSpecInfo'] + options?: MergerConfig['options'] ): Promise> { const randomStr = (Math.random() + 1).toString(36).substring(7); const folderToMergePath = join(ROOT_PATH, 'target', 'oas-test', randomStr); @@ -36,7 +36,7 @@ export async function mergeSpecs( dumpSpecs(folderToMergePath, oasSpecs); - await mergeFolder(folderToMergePath, mergedFilePathTemplate, mergedSpecInfo); + await mergeFolder(folderToMergePath, mergedFilePathTemplate, options); return readMergedSpecs(resultFolderPath); } @@ -75,11 +75,11 @@ export function readMergedSpecs(folderPath: string): Record { await merge({ sourceGlobs: [join(folderToMergePath, '*.schema.yaml')], outputFilePath: mergedFilePathTemplate, - mergedSpecInfo, + options, }); } diff --git a/packages/kbn-openapi-bundler/tests/merger/merging_specs_with_conflicting_components.test.ts b/packages/kbn-openapi-bundler/tests/merger/merging_specs_with_conflicting_components.test.ts new file mode 100644 index 0000000000000..c77d2a08a644a --- /dev/null +++ b/packages/kbn-openapi-bundler/tests/merger/merging_specs_with_conflicting_components.test.ts @@ -0,0 +1,711 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createOASDocument } from '../create_oas_document'; +import { mergeSpecs } from './merge_specs'; + +// Disable naming convention check due to tests on spec title prefixes +// like Spec1_Something which violates that rule +/* eslint-disable @typescript-eslint/naming-convention */ + +describe('OpenAPI Merger - merging specs with conflicting components', () => { + it('prefixes schemas component names for each source spec ', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + get: { + responses: { + '200': { + description: 'Successful response', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SomeSchema', + }, + }, + }, + }, + }, + }, + }, + }, + components: { + schemas: { + SomeSchema: { + type: 'string', + }, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/some_api': { + post: { + responses: { + '200': { + description: 'Successful response', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SomeSchema', + }, + }, + }, + }, + }, + }, + }, + }, + components: { + schemas: { + SomeSchema: { + type: 'string', + }, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.get?.responses['200']).toMatchObject({ + content: { + 'application/json; Elastic-Api-Version=2023-10-31': { + schema: { + $ref: '#/components/schemas/Spec1_SomeSchema', + }, + }, + }, + }); + expect(mergedSpec.paths['/api/some_api']?.post?.responses['200']).toMatchObject({ + content: { + 'application/json; Elastic-Api-Version=2023-10-31': { + schema: { + $ref: '#/components/schemas/Spec2_SomeSchema', + }, + }, + }, + }); + expect(mergedSpec.components?.schemas).toMatchObject({ + Spec1_SomeSchema: expect.anything(), + Spec2_SomeSchema: expect.anything(), + }); + }); + + it('prefixes responses component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + get: { + responses: { + '200': { + $ref: '#/components/responses/GetResponse', + }, + }, + }, + }, + }, + components: { + responses: { + GetResponse: { + description: 'Successful response', + content: { + 'application/json': { + schema: { + type: 'string', + }, + }, + }, + }, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/some_api': { + post: { + responses: { + '200': { + $ref: '#/components/responses/PostResponse', + }, + }, + }, + }, + }, + components: { + responses: { + PostResponse: { + description: 'Successful response', + content: { + 'application/json': { + schema: { + type: 'string', + }, + }, + }, + }, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.get?.responses['200']).toMatchObject({ + $ref: '#/components/responses/Spec1_GetResponse', + }); + expect(mergedSpec.paths['/api/some_api']?.post?.responses['200']).toMatchObject({ + $ref: '#/components/responses/Spec2_PostResponse', + }); + expect(mergedSpec.components?.responses).toMatchObject({ + Spec1_GetResponse: expect.anything(), + Spec2_PostResponse: expect.anything(), + }); + }); + + it('prefixes parameters component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api/{id}': { + parameters: [ + { + $ref: '#/components/parameters/SomeApiIdParam', + }, + ], + get: { + responses: {}, + }, + }, + }, + components: { + parameters: { + SomeApiIdParam: { + name: 'id', + in: 'path', + }, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api/{id}': { + get: { + parameters: [ + { + $ref: '#/components/parameters/AnotherApiIdParam', + }, + ], + responses: {}, + }, + }, + }, + components: { + parameters: { + AnotherApiIdParam: { + name: 'id', + in: 'path', + }, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api/{id}']?.parameters).toEqual([ + { + $ref: '#/components/parameters/Spec1_SomeApiIdParam', + }, + ]); + expect(mergedSpec.paths['/api/another_api/{id}']?.get?.parameters).toEqual([ + { + $ref: '#/components/parameters/Spec2_AnotherApiIdParam', + }, + ]); + expect(mergedSpec.components?.parameters).toMatchObject({ + Spec1_SomeApiIdParam: expect.anything(), + Spec2_AnotherApiIdParam: expect.anything(), + }); + }); + + it('prefixes request bodies component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + post: { + requestBody: { + $ref: '#/components/requestBodies/SomeApiRequestBody', + }, + responses: {}, + }, + }, + }, + components: { + requestBodies: { + SomeApiRequestBody: { + content: {}, + }, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + post: { + requestBody: { + $ref: '#/components/requestBodies/AnotherApiRequestBody', + }, + responses: {}, + }, + }, + }, + components: { + requestBodies: { + AnotherApiRequestBody: { + content: {}, + }, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.post?.requestBody).toMatchObject({ + $ref: '#/components/requestBodies/Spec1_SomeApiRequestBody', + }); + expect(mergedSpec.paths['/api/another_api']?.post?.requestBody).toMatchObject({ + $ref: '#/components/requestBodies/Spec2_AnotherApiRequestBody', + }); + expect(mergedSpec.components?.requestBodies).toMatchObject({ + Spec1_SomeApiRequestBody: expect.anything(), + Spec2_AnotherApiRequestBody: expect.anything(), + }); + }); + + it('prefixes examples component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + get: { + responses: { + example: { $ref: '#/components/examples/SomeApiGetResponseExample' }, + }, + }, + }, + }, + components: { + examples: { + SomeApiGetResponseExample: {}, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + get: { + responses: { + example: { $ref: '#/components/examples/AnotherApiGetResponseExample' }, + }, + }, + }, + }, + components: { + examples: { + AnotherApiGetResponseExample: {}, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.get?.responses.example).toMatchObject({ + $ref: '#/components/examples/Spec1_SomeApiGetResponseExample', + }); + expect(mergedSpec.paths['/api/another_api']?.get?.responses.example).toMatchObject({ + $ref: '#/components/examples/Spec2_AnotherApiGetResponseExample', + }); + expect(mergedSpec.components?.examples).toMatchObject({ + Spec1_SomeApiGetResponseExample: expect.anything(), + Spec2_AnotherApiGetResponseExample: expect.anything(), + }); + }); + + it('prefixes headers component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + post: { + requestBody: { + content: { + 'application/json': { + encoding: { + something: { + headers: { + 'x-request-header': { + $ref: '#/components/headers/SomeApiRequestHeader', + }, + }, + }, + }, + }, + }, + }, + responses: {}, + }, + }, + }, + components: { + headers: { + SomeApiRequestHeader: {}, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + get: { + responses: { + '200': { + description: 'Successful response', + headers: { + 'x-response-header': { + $ref: '#/components/headers/AnotherApiResponseHeader', + }, + }, + }, + }, + }, + }, + }, + components: { + headers: { + AnotherApiResponseHeader: {}, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.post?.requestBody).toMatchObject({ + content: { + 'application/json; Elastic-Api-Version=2023-10-31': { + encoding: { + something: { + headers: { + 'x-request-header': { + $ref: '#/components/headers/Spec1_SomeApiRequestHeader', + }, + }, + }, + }, + }, + }, + }); + expect(mergedSpec.paths['/api/another_api']?.get?.responses['200']).toMatchObject({ + headers: { + 'x-response-header': { + $ref: '#/components/headers/Spec2_AnotherApiResponseHeader', + }, + }, + }); + expect(mergedSpec.components?.headers).toMatchObject({ + Spec1_SomeApiRequestHeader: expect.anything(), + Spec2_AnotherApiResponseHeader: expect.anything(), + }); + }); + + it('prefixes security schemes component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + security: [ + { + SomeApiAuth: [], + }, + ], + paths: { + '/api/some_api': { + get: { + responses: {}, + }, + }, + }, + components: { + securitySchemes: { + SomeApiAuth: { + type: 'http', + scheme: 'Basic', + }, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + get: { + security: [ + { + AnotherApiAuth: [], + }, + ], + responses: {}, + }, + }, + }, + components: { + securitySchemes: { + AnotherApiAuth: { + type: 'http', + scheme: 'Basic', + }, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.security).toEqual( + expect.arrayContaining([ + { + Spec1_SomeApiAuth: [], + }, + ]) + ); + expect(mergedSpec.paths['/api/another_api']?.get?.security).toEqual([ + { + Spec2_AnotherApiAuth: [], + }, + ]); + expect(mergedSpec.components?.securitySchemes).toMatchObject({ + Spec1_SomeApiAuth: expect.anything(), + Spec2_AnotherApiAuth: expect.anything(), + }); + }); + + it('prefixes links component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + get: { + responses: { + '200': { + description: 'Successful response', + links: { + SomeLink: { + $ref: '#/components/links/SomeLink', + }, + }, + }, + }, + }, + }, + }, + components: { + links: { + SomeLink: {}, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + get: { + responses: { + '200': { + description: 'Successful response', + links: { + SomeLink: { + $ref: '#/components/links/SomeLink', + }, + }, + }, + }, + }, + }, + }, + components: { + links: { + SomeLink: {}, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.get?.responses['200']).toMatchObject({ + links: { + SomeLink: { + $ref: '#/components/links/Spec1_SomeLink', + }, + }, + }); + expect(mergedSpec.paths['/api/another_api']?.get?.responses['200']).toMatchObject({ + links: { + SomeLink: { + $ref: '#/components/links/Spec2_SomeLink', + }, + }, + }); + expect(mergedSpec.components?.links).toMatchObject({ + Spec1_SomeLink: expect.anything(), + Spec2_SomeLink: expect.anything(), + }); + }); + + it('prefixes callbacks component names for each source spec', async () => { + const spec1 = createOASDocument({ + info: { + title: 'Spec1', + }, + paths: { + '/api/some_api': { + get: { + responses: {}, + callbacks: { + SomeCallback: { + $ref: '#/components/callbacks/SomeCallback', + }, + }, + }, + }, + }, + components: { + callbacks: { + SomeCallback: {}, + }, + }, + }); + const spec2 = createOASDocument({ + info: { + title: 'Spec2', + }, + paths: { + '/api/another_api': { + get: { + responses: {}, + callbacks: { + SomeCallback: { + $ref: '#/components/callbacks/SomeCallback', + }, + }, + }, + }, + }, + components: { + callbacks: { + SomeCallback: {}, + }, + }, + }); + + const [mergedSpec] = Object.values( + await mergeSpecs({ + 1: spec1, + 2: spec2, + }) + ); + + expect(mergedSpec.paths['/api/some_api']?.get?.callbacks).toMatchObject({ + SomeCallback: { + $ref: '#/components/callbacks/Spec1_SomeCallback', + }, + }); + expect(mergedSpec.paths['/api/another_api']?.get?.callbacks).toMatchObject({ + SomeCallback: { + $ref: '#/components/callbacks/Spec2_SomeCallback', + }, + }); + expect(mergedSpec.components?.callbacks).toMatchObject({ + Spec1_SomeCallback: expect.anything(), + Spec2_SomeCallback: expect.anything(), + }); + }); +}); diff --git a/packages/kbn-openapi-bundler/tests/merger/unresolvable_operation_conflicts.test.ts b/packages/kbn-openapi-bundler/tests/merger/unresolvable_operation_conflicts.test.ts index 0305b31772287..b1fdb50498e7b 100644 --- a/packages/kbn-openapi-bundler/tests/merger/unresolvable_operation_conflicts.test.ts +++ b/packages/kbn-openapi-bundler/tests/merger/unresolvable_operation_conflicts.test.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { OpenAPIV3 } from 'openapi-types'; import { createOASDocument } from '../create_oas_document'; import { mergeSpecs } from './merge_specs'; @@ -266,13 +267,17 @@ describe('OpenAPI Merger - unresolvable operation object conflicts', () => { '/api/my/endpoint': { get: { requestBody: { - // SomeRequestBody definition is omitted for brivity since it's not validated by the merger $ref: '#/components/requestBodies/SomeRequestBody', }, responses: {}, }, }, }, + components: { + requestBodies: { + SomeRequestBody: {} as OpenAPIV3.RequestBodyObject, + }, + }, }); expect( @@ -299,13 +304,17 @@ describe('OpenAPI Merger - unresolvable operation object conflicts', () => { get: { responses: { 200: { - // SomeResponse definition is omitted for brivity since it's not validated by the merger $ref: '#/components/responses/SomeResponse', }, }, }, }, }, + components: { + responses: { + SomeResponse: {} as OpenAPIV3.ResponseObject, + }, + }, }); expect( diff --git a/packages/kbn-openapi-bundler/tests/merger/unresolvable_path_item_conflicts.test.ts b/packages/kbn-openapi-bundler/tests/merger/unresolvable_path_item_conflicts.test.ts index 3488c55fccb87..6f71727d2b4cd 100644 --- a/packages/kbn-openapi-bundler/tests/merger/unresolvable_path_item_conflicts.test.ts +++ b/packages/kbn-openapi-bundler/tests/merger/unresolvable_path_item_conflicts.test.ts @@ -120,10 +120,14 @@ describe('OpenAPI Merger - unresolvable path item object conflicts', () => { const spec2 = createOASDocument({ paths: { '/api/my/endpoint': { - // PathItemDefinition definition is omitted for brivity since it's not validated by the merger $ref: '#/components/schemas/PathItemDefinition', }, }, + components: { + schemas: { + PathItemDefinition: {}, + }, + }, }); expect(