diff --git a/CHANGELOG.md b/CHANGELOG.md index f99a07a70..b8fd3cefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) - Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Changed @@ -116,6 +121,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Security diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 951f00cdd..43c30d7f3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -146,6 +146,10 @@ This repository includes several OpenAPI Specification Extensions to fill in any - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them. - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml). - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema. +- `x-distributions-included`: Contains a list of distributions known to include the API. +- `x-distributions-excluded`: Contains a list of distributions known to exclude the API. + +Use `opensearch.org` for the official distribution in `x-distributions-*`, `amazon-managed` for Amazon Managed OpenSearch, and `amazon-serverless` for Amazon OpenSearch Serverless. ## Writing Spec Tests diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1328cc863..d35c96aa5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -11,6 +11,7 @@ - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) - [Waiting for Tasks](#waiting-for-tasks) - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) @@ -209,7 +210,59 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne status: 200 ``` -The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. +The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. + +### Managing Distributions + +OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions-included` or `x-distributions-excluded` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not. + +```yaml +/: + get: + operationId: info.0 + x-distributions-included: + - opensearch.org + - amazon-managed + x-distributions-excluded: + - amazon-serverless + description: Returns basic information about the cluster. +``` + +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. + +```yaml +description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org +chapters: + - synopsis: Get server info. + path: / + method: GET + response: + status: 200 +``` + +To test a particular distribution pass `--opensearch-distribution` to the test tool. For example, the following runs tests against an Amazon Managed OpenSearch instance. + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export AWS_SESSION_TOKEN=... +export AWS_REGION=us-west-2 + +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec -- --opensearch-distribution=amazon-managed +``` + +The output will visible skip APIs that are not available in the `amazon-managed` distribution. + +``` +PASSED _core/bulk.yaml (.../_core/bulk.yaml) +PASSED _core/info.yaml (.../_core/info.yaml) +SKIPPED indices/forcemerge.yaml (Skipped because distribution amazon-managed is not opensearch.org.) +``` ### Waiting for Tasks diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6d1916509..928ba4bad 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -21,6 +21,8 @@ properties: $ref: '#/definitions/Chapter' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' required: [chapters,description] additionalProperties: false @@ -85,6 +87,8 @@ definitions: $ref: '#/definitions/Output' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' retry: $ref: '#/definitions/Retry' required: [method, path] @@ -106,6 +110,13 @@ definitions: The semver range to execute the story or chapter against. type: string + Distributions: + description: | + The list of distributions that support this API. + type: array + items: + type: string + Retry: description: | Number of times to retry on error. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9aa5dbeaa..bc90411bb 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -9,6 +9,8 @@ paths: operationId: info.0 x-operation-group: info x-version-added: '1.0' + x-distributions-excluded: + - amazon-serverless description: Returns basic information about the cluster. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index ff0efdc21..cab8696c2 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -562,6 +562,14 @@ components: write_operations: description: The total number of write operations for the device completed since starting OpenSearch. type: number + read_time: + type: number + write_time: + type: number + queue_size: + type: number + io_time_in_millis: + $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' Jvm: type: object properties: @@ -1068,8 +1076,10 @@ components: type: boolean enforced: type: boolean + total_rejections_breakup: + $ref: '#/components/schemas/TotalRejectionsBreakup' total_rejections_breakup_shadow_mode: - $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode' + $ref: '#/components/schemas/TotalRejectionsBreakup' ShardSearchBackpressureStats: type: object properties: @@ -1140,7 +1150,11 @@ components: type: number cancellation_limit_reached_count: type: number - TotalRejectionsBreakupShadowMode: + cancelled_task_percentage: + type: number + current_cancellation_eligible_tasks_count: + type: number + TotalRejectionsBreakup: type: object properties: node_limits: diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7086c17ca..6fbcefdc2 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -102,14 +102,10 @@ components: search_pipelines: $ref: '#/components/schemas/NodeInfoSearchPipelines' required: - - attributes - build_hash - build_type - - host - - ip - name - roles - - transport_address - version NodeInfoHttp: type: object @@ -156,7 +152,7 @@ components: bundled_jdk: type: boolean using_bundled_jdk: - type: boolean + type: [boolean, 'null'] using_compressed_ordinary_object_pointers: oneOf: - type: boolean @@ -167,16 +163,9 @@ components: type: string required: - bundled_jdk - - gc_collectors - - input_arguments - mem - - memory_pools - pid - start_time_in_millis - - version - - vm_name - - vm_vendor - - vm_version NodeInfoJvmMemory: type: object properties: @@ -256,12 +245,8 @@ components: swap: $ref: '#/components/schemas/NodeInfoMemory' required: - - arch - available_processors - - name - - pretty_name - refresh_interval_in_millis - - version NodeInfoOSCPU: type: object properties: diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 67e245235..9cc644a6b 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -2,6 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org chapters: - synopsis: Get server info. path: / diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 8e7a04351..32f9b3c2d 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -124,4 +124,3 @@ chapters: title: Drive year: 2011 order: 2 - diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 38865e9e5..196cff4b3 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -53,7 +53,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat with master response (format=json). method: GET @@ -66,7 +65,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_master: 'true' - synopsis: Cat with cluster_manager response (format=json). @@ -80,7 +78,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_cluster_manager: 'true' - synopsis: Cat in different formats (format=yaml). @@ -93,9 +90,10 @@ chapters: content_type: application/yaml payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -105,9 +103,10 @@ chapters: content_type: application/cbor payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -117,5 +116,4 @@ chapters: content_type: application/smile payload: - node.total: '1' - status: yellow node.data: '1' diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index c20df6fe8..6b6130823 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -71,6 +71,8 @@ chapters: status: 200 content_type: application/yaml - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: @@ -79,6 +81,8 @@ chapters: status: 200 content_type: application/cbor - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index cb9dc328c..01dba1cae 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. + distributions: + - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index db4db9dd5..2e056a526 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 94087d494..0b1b0c784 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. +distributions: + - opensearch.org chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index d537e3a7c..cc1608cb6 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -2,6 +2,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index da87fec7b..0445ebd64 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -1,7 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. - +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 5bfada4b9..80906a7e3 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -11,6 +11,8 @@ epilogues: status: [200, 404] chapters: - synopsis: Get global settings. + distributions: + - opensearch.org path: /_settings method: GET parameters: @@ -23,6 +25,8 @@ chapters: response: status: 200 - synopsis: Get global settings (cluster_manager_timeout). + distributions: + - opensearch.org path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index a08efa0ce..8c036ae6e 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index 4bdac64e1..f92ee82d0 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml index f4f23e817..340a659d5 100644 --- a/tests/default/security/api/certificates.yaml +++ b/tests/default/security/api/certificates.yaml @@ -4,20 +4,18 @@ description: Test certificates endpoints. version: '> 2.14' # ADMIN-CERT only. These tests require explicit rest api admin privileges. -chapters: - - synopsis: Get node. - id: get_node - path: /_cat/nodes +prologues: + - path: /_cat/nodes + id: node method: GET parameters: - h: - - id full_id: true - response: - status: 200 - content_type: text/plain + size: 1 + format: json + h: id output: - node_id: payload + id: payload[0].id +chapters: - synopsis: Get all certificates. path: /_plugins/_security/api/certificates method: GET @@ -29,7 +27,7 @@ chapters: path: /_plugins/_security/api/certificates/{node_id} method: GET parameters: - node_id: ${get_node.node_id} + node_id: ${node.id} cert_type: all response: - status: 403 + status: 403 \ No newline at end of file diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index afbb7d5fa..314a73d45 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -73,6 +73,7 @@ export interface OpenSearchHttpClientOptions { export type OpenSearchHttpClientCliOptions = { opensearchUrl?: string + opensearchDistribution?: string, opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts index 52de7e899..836c802ea 100644 --- a/tools/src/_utils/semver.ts +++ b/tools/src/_utils/semver.ts @@ -9,7 +9,8 @@ import * as semver from 'semver' -export function coerce(version: string) : string { +export function coerce(version?: string): undefined | string { + if (version === undefined) return undefined return semver.coerce(version)?.toString() ?? version } diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index 06672285b..c463e49d9 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -16,7 +16,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemasValidator { diff --git a/tools/src/linter/components/OperationGroup.ts b/tools/src/linter/components/OperationGroup.ts index dc45765bc..f980e56b5 100644 --- a/tools/src/linter/components/OperationGroup.ts +++ b/tools/src/linter/components/OperationGroup.ts @@ -13,7 +13,7 @@ import ValidatorBase from './base/ValidatorBase' export default class OperationGroup extends ValidatorBase { static readonly OP_PRIORITY = ['operationId', 'x-operation-group', 'x-ignorable', 'deprecated', - 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', + 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', 'x-distributions-included', 'x-distributions-excluded', 'description', 'externalDocs', 'parameters', 'requestBody', 'responses'] name: string diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 3f2fb396b..a2ccbffa4 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -17,12 +17,14 @@ import * as semver from '../_utils/semver' export default class OpenApiVersionExtractor { private _spec?: Record private _source_spec: OpenAPIV3.Document - private _target_version: string + private _target_version?: string + private _target_distribution?: string private _logger: Logger - constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { + constructor(source_spec: OpenAPIV3.Document, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this._source_spec = source_spec this._target_version = semver.coerce(target_version) + this._target_distribution = target_distribution this._logger = logger this._spec = undefined } @@ -45,16 +47,37 @@ export default class OpenApiVersionExtractor { #extract() : void { this._logger.info(`Extracting version ${this._target_version} ...`) this.#remove_keys_not_matching_semver() + this.#remove_keys_not_matching_distribution() this.#remove_unused() } #exclude_per_semver(obj: any): boolean { + if (this._target_version == undefined) return false + const x_version_added = semver.coerce(obj['x-version-added'] as string) const x_version_removed = semver.coerce(obj['x-version-removed'] as string) - if (x_version_added && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { + if (x_version_added !== undefined && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { return true - } else if (x_version_removed && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + } else if (x_version_removed !== undefined && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + return true + } + + return false + } + + #exclude_per_distribution(obj: any): boolean { + if (this._target_distribution == undefined) return false + + const x_distributions_included = obj['x-distributions-included'] as string[] + + if (x_distributions_included?.length > 0 && !x_distributions_included.includes(this._target_distribution)) { + return true + } + + const x_distributions_excluded = obj['x-distributions-excluded'] as string[] + + if (x_distributions_excluded?.length > 0 && x_distributions_excluded.includes(this._target_distribution)) { return true } @@ -63,9 +86,16 @@ export default class OpenApiVersionExtractor { // Remove any elements that are x-version-added/removed incompatible with the target server version. #remove_keys_not_matching_semver(): void { + if (this._target_version == undefined) return delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this)) } + // Remove any elements that are x-distributions-included incompatible with the target distribution. + #remove_keys_not_matching_distribution(): void { + if (this._target_distribution === undefined) return + delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this)) + } + #remove_unused(): void { if (this._spec === undefined) return diff --git a/tools/src/tester/MergedOpenApiSpec.ts b/tools/src/tester/MergedOpenApiSpec.ts index ee21f55ac..d4a3e2604 100644 --- a/tools/src/tester/MergedOpenApiSpec.ts +++ b/tools/src/tester/MergedOpenApiSpec.ts @@ -20,21 +20,23 @@ export default class MergedOpenApiSpec { logger: Logger file_path: string target_version?: string + target_distribution?: string protected _spec: OpenAPIV3.Document | undefined - constructor (spec_path: string, target_version?: string, logger: Logger = new Logger()) { + constructor (spec_path: string, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this.logger = logger this.file_path = spec_path this.target_version = target_version + this.target_distribution = target_distribution } spec (): OpenAPIV3.Document { if (this._spec) return this._spec const merger = new OpenApiMerger(this.file_path, this.logger) var spec = merger.spec() - if (this.target_version !== undefined) { - const version_extractor = new OpenApiVersionExtractor(spec, this.target_version) + if (this.target_version !== undefined || this.target_distribution !== undefined) { + const version_extractor = new OpenApiVersionExtractor(spec, this.target_version, this.target_distribution) spec = version_extractor.extract() } const ctx = new SpecificationContext(this.file_path) diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 7f71e3870..01b55b5ad 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -19,7 +19,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemaValidator { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 9020968fa..3f396a3c6 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,8 +26,8 @@ export default class StoryEvaluator { this._supplemental_chapter_evaluator = supplemental_chapter_evaluator } - async evaluate({ story, display_path, full_path }: StoryFile, version?: string, dry_run: boolean = false): Promise { - if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { + if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { return { result: Result.SKIPPED, display_path, @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } + if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error } const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) return { display_path, @@ -76,7 +86,7 @@ export default class StoryEvaluator { } } - async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string): Promise { + async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { if (dry_run) { @@ -85,6 +95,9 @@ export default class StoryEvaluator { } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0b197d648..dc880f946 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -34,19 +34,25 @@ export default class TestRunner { this._result_logger = result_logger } - async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { + async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { - const info = await this._http_client.wait_until_available() - console.log(`OpenSearch ${ansi.green(info.version.number)}\n`) - version = info.version.number + if (distribution === 'amazon-serverless') { + // TODO: Fetch OpenSearch version when Amazon Serverless OpenSearch supports multiple. + version = '2.1' + } else { + const info = await this._http_client.wait_until_available() + version = info.version.number + } + + console.log(`OpenSearch ${ansi.green(version)}\n`) } for (const story_file of story_files) { - const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run) + const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 134888d7f..da025a34b 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -47,6 +47,9 @@ const command = new Command() .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(new Option('--opensearch-version ', 'target OpenSearch schema version').default(undefined)) + .addOption(new Option('--opensearch-distribution ', 'OpenSearch distribution') + .default('opensearch.org') + .env('OPENSEARCH_DISTRIBUTION')) .addOption(OPENSEARCH_URL_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) @@ -63,7 +66,7 @@ const command = new Command() const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) -const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) +const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, opts.opensearchDistribution, new Logger(LogLevel.error)) const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) @@ -73,7 +76,7 @@ const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapt const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) -runner.run(opts.testsPath, spec.api_version(), opts.dryRun) +runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( ({ results, failed }) => { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e418ff762..83d307b44 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -50,6 +50,14 @@ export type Payload = {} | any[] | string | number | boolean; * via the `definition` "Version". */ export type Version = string; +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export type Distributions = string[]; /** * Number of times to retry on error. * @@ -94,6 +102,7 @@ export interface Story { epilogues?: SupplementalChapter[]; chapters: Chapter[]; version?: Version; + distributions?: Distributions; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -112,6 +121,7 @@ export interface ChapterRequest { request?: Request; output?: Output; version?: Version; + distributions?: Distributions; retry?: Retry; } /** @@ -131,7 +141,7 @@ export interface Request { * The values are paths to the values in the response. * The values should be in the form: * - `payload.` for the payload - * - `headers.` for the headers + * - `headers.` for the headers. * * * This interface was referenced by `Story`'s JSON-Schema @@ -157,6 +167,9 @@ export interface ExpectedResponse { * via the `definition` "Warnings". */ export interface Warnings { + /** + * Enable/disable warnings about multiple paths being tested in the same story. + */ 'multiple-paths-detected'?: boolean; } /** diff --git a/tools/src/types.ts b/tools/src/types.ts index a9497dbcb..b6871d022 100644 --- a/tools/src/types.ts +++ b/tools/src/types.ts @@ -16,6 +16,8 @@ export interface OperationSpec extends OpenAPIV3.OperationObject { 'x-version-deprecated'?: string 'x-deprecation-message'?: string 'x-ignorable'?: boolean + 'x-distributions-included'?: string[] + 'x-distributions-excluded'?: string[] parameters?: OpenAPIV3.ReferenceObject[] requestBody?: OpenAPIV3.ReferenceObject diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index 61bc42622..ca3e14eb0 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -17,7 +17,7 @@ describe('extract() from a merged API spec', () => { const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') describe('1.3', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3') + const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') describe('write_to', () => { var temp: tmp.DirResult @@ -43,18 +43,18 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0', 'ignore') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless' ]) }) @@ -81,12 +81,12 @@ describe('extract() from a merged API spec', () => { }) describe('2.1', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'oracle-managed') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml index d0fa222f7..a4497ccbb 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml @@ -35,6 +35,10 @@ paths: x-version-removed: '2.0' added-1.3-removed-2.0: $ref: '#/components/responses/info@added-1.3-removed-2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -106,6 +110,8 @@ components: description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@removed-2.0: description: Removed in 2.0 via attribute next to ref. nodes.info@200: diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml index cfb4279d2..c684b7af9 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml @@ -57,6 +57,10 @@ paths: added-2.0: $ref: '#/components/responses/info@added-2.0' x-version-added: '2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -144,6 +148,8 @@ components: type: object info@added-2.0: description: Added in 2.0 via attribute next to ref. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. nodes.info@200: description: All nodes. content: diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 7e6c64641..db852a030 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -13,7 +13,7 @@ import MergedOpenApiSpec from "tester/MergedOpenApiSpec" describe('merged API spec', () => { describe('defaults', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, undefined, new Logger()) test('has an api version', () => { expect(spec.api_version()).toEqual('1.2.3') @@ -30,7 +30,8 @@ describe('merged API spec', () => { test('has all responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1' + '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) @@ -65,31 +66,67 @@ describe('merged API spec', () => { }) describe('1.3', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'oracle-managed', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', undefined, new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('2.0 amazon-serverless', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'amazon-serverless', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all' + ]) + }) + }) + + describe('2.0 oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'oracle-managed', new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.1', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml new file mode 100644 index 000000000..cc4ef9a19 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml + +result: SKIPPED +description: This story should be skipped because of distributions. +message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index 68337dc67..0c0a09234 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -28,6 +28,20 @@ paths: $ref: '#/components/responses/info@500' '503': $ref: '#/components/responses/info@503' + distributed-included-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions-included: + - amazon-managed + - amazon-serverless + - opensearch.org + distributed-included-amazon-managed: + $ref: '#/components/responses/info@distributed-amazon-managed' + x-distributions-included: + - amazon-managed + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless components: responses: info@200: @@ -76,6 +90,10 @@ components: info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' + info@distributed-amazon-managed: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@500: content: application/json: diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml new file mode 100644 index 000000000..8efebbb2f --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml @@ -0,0 +1,9 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions. +distributions: + - another + - some +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index ce11b6ad8..d6d7e5639 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index b0333504f..0259c29ac 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -16,7 +16,7 @@ test('stories folder', async () => { const info = await opensearch_http_client.wait_until_available() expect(info.version).toBeDefined() - const run = await test_runner.run('tools/tests/tester/fixtures/stories') + const run = await test_runner.run('tools/tests/tester/fixtures/stories', undefined, 'opensearch.org') expect(run.failed).toBeTruthy() @@ -29,14 +29,15 @@ test('stories folder', async () => { } const passed = load_expected_evaluation('passed', true) - const skipped = load_expected_evaluation('skipped/semver', true) + const skipped_semver = load_expected_evaluation('skipped/semver', true) + const skipped_distributions = load_expected_evaluation('skipped/distributions', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const output_error = load_expected_evaluation('error/output_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped] + const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] expect(actual_evaluations).toEqual(expected_evaluations) })