From 7f3899d9cb92e432a8f8e0ebf801f9cd6e7fa411 Mon Sep 17 00:00:00 2001 From: Michael Dowling Date: Thu, 13 Apr 2023 22:28:51 -0700 Subject: [PATCH] Provide a public lexer and use it in the parser This commit exposes a lexer that we can use to provide better IDE and formatter support. The lexer based parser also typically provides more accurate source locations, and more informative error messages. --- ...dlModelParser.java => IdlModelLoader.java} | 1285 +++++++---------- .../smithy/model/loader/IdlNodeParser.java | 299 ++-- .../model/loader/IdlReferenceResolver.java | 51 + .../smithy/model/loader/IdlShapeIdParser.java | 111 ++ ...IdlTextParser.java => IdlStringLexer.java} | 173 ++- .../amazon/smithy/model/loader/IdlToken.java | 73 + .../smithy/model/loader/IdlTokenizer.java | 825 +++++++++++ .../smithy/model/loader/IdlTraitParser.java | 279 ++-- .../smithy/model/loader/LoadOperation.java | 13 +- .../model/loader/LoadOperationProcessor.java | 15 +- .../smithy/model/loader/ModelAssembler.java | 8 +- .../smithy/model/loader/ModelLoader.java | 13 +- .../smithy/model/loader/StringTable.java | 97 ++ .../smithy/model/selector/SelectorParser.java | 14 +- .../linters/EmitEachSelectorValidator.java | 18 +- .../model/loader/AstModelLoaderTest.java | 2 +- .../model/loader/IdlModelLoaderTest.java | 32 +- .../model/loader/IdlTextParserTest.java | 84 -- .../smithy/model/loader/StringTableTest.java | 65 + .../smithy/model/loader/TokenizerTest.java | 524 +++++++ .../loader/dupe-list-member-names.errors | 2 +- .../loader/dupe-map-member-names.errors | 2 +- .../loader/dupe-resource-binding.errors | 2 +- .../loader/dupe-service-binding.errors | 2 +- .../loader/dupe-set-member-names.errors | 2 +- .../loader/dupe-structure-member-names.errors | 2 +- .../loader/dupe-trait-member-names.errors | 2 +- .../loader/dupe-trait-object-keys.errors | 2 +- .../annotation-invalid-object-key1.smithy | 4 - .../annotation-invalid-object-key2.smithy | 4 - .../annotation-invalid-object-key3.smithy | 4 - .../invalid/annotation-unclosed-list1.smithy | 4 - .../invalid/annotation-unclosed-list2.smithy | 4 - .../invalid/annotation-unclosed-list3.smithy | 4 - .../invalid/annotation-unclosed-list4.smithy | 3 - .../annotation-unclosed-object1.smithy | 4 - .../annotation-unclosed-object2.smithy | 4 - .../annotation-unclosed-object3.smithy | 4 - .../annotation-unclosed-object4.smithy | 4 - .../annotation-unclosed-object5.smithy | 4 - .../annotation-unclosed-object6.smithy | 4 - .../apply/apply-block-missing-closing.smithy | 2 +- .../apply/apply-missing-newline.smithy | 2 +- .../invalid/apply/apply-missing-space.smithy | 2 +- .../apply/apply-missing-trait-value.smithy | 2 +- .../control/control-colon-newline.smithy | 2 +- .../control/control-missing-value.smithy | 2 +- .../invalid/control/control-newline.smithy | 2 +- .../control-statement-before-others.smithy | 2 +- .../control-version-defined-twice.smithy | 2 +- .../control/control-with-invalid-key.smithy | 2 +- .../control/control-with-invalid-key2.smithy | 2 +- .../control/control-with-no-colon.smithy | 2 +- .../control/no-newline-after-control.smithy | 2 +- .../invalid/defaults/default-in-v1.smithy | 2 +- .../defaults/default-incomplete-node.smithy | 2 +- .../default-missing-structure-close.smithy | 2 +- .../defaults/default-missing-value.smithy | 2 +- .../default-with-newline-before-value.smithy | 2 +- .../missing-newline-after-assignment.smithy | 2 +- .../unions-do-not-support-defaults.smithy | 2 +- .../elided-union-member-from-resource.smithy | 2 +- .../{ => elision}/for-after-with.smithy | 2 +- .../for-referencing-non-resource.smithy | 0 .../missing-newline-after-assignment.smithy | 7 - .../enum-with-bad-values.smithy | 0 .../enum-with-newline-before-value.smithy | 2 +- .../{ => enums}/enum-without-member.smithy | 0 .../{ => enums}/idl-enums-in-v1.smithy | 0 .../int-enum-without-member.smithy | 0 .../intEnum-with-bad-value.smithy | 0 .../intEnum-with-float.smithy | 0 .../intEnum-with-long.smithy | 0 .../missing-newline-after-assignment.smithy | 7 + .../expected-shape-id-after-type.smithy | 4 - .../expected-shape-name-but-eof.smithy | 4 - .../invalid/inline-io/inline-error.smithy | 2 +- .../inline-io/inline-name-override.smithy | 2 +- .../inline-io/inline-single-error.smithy | 2 +- .../inline-io/inline-structure-member.smithy | 2 +- .../inline-io/walrus-operator-in-node.smithy | 2 +- .../invalid-object-shape-id-key.smithy | 4 - .../invalid-object-text-block-key.smithy | 5 - .../invalid/invalid-unquoted-shape-id.smithy | 5 - .../loader/invalid/list-invalid-member.smithy | 6 - .../{ => lists}/list-empty-members.smithy | 0 .../invalid/lists/list-invalid-member.smithy | 6 + .../invalid/{ => lists}/no-sets-in-v2.smithy | 0 .../{ => lists}/set-empty-members.smithy | 0 .../invalid/lists/set-invalid-member.smithy | 6 + .../invalid/map-unclosed-parameters1.smithy | 3 - .../invalid/map-unclosed-parameters2.smithy | 3 - .../invalid/map-unclosed-parameters3.smithy | 3 - .../invalid/map-unclosed-parameters4.smithy | 3 - .../{ => maps}/map-empty-members.smithy | 0 .../{ => maps}/map-invalid-member.smithy | 2 +- .../maps/map-unclosed-parameters1.smithy | 3 + .../maps/map-unclosed-parameters2.smithy | 3 + .../maps/map-unclosed-parameters3.smithy | 3 + .../maps/map-unclosed-parameters4.smithy | 3 + .../invalid-object-shape-id-key.smithy | 4 + .../invalid-object-text-block-key.smithy | 5 + .../metadata/invalid-unquoted-shape-id.smithy | 5 + .../metadata/metadata-multiple-lines.smithy | 4 +- ...metadata-must-come-before-namespace.smithy | 2 +- .../metadata/metadata-with-invalid-key.smithy | 2 +- .../metadata/no-newline-after-metadata.smithy | 2 +- .../not-newline-after-metadata.smithy | 2 +- .../space-after-metadata-quoted.smithy | 2 +- .../space-after-metadata-unquoted.smithy | 2 +- .../invalid/mixins/dangling-with.smithy | 2 +- ...alid-statement-after-structure-name.smithy | 2 +- .../mixins/missing-rest-of-with-word.smithy | 2 +- .../mixins-usage-in-explicit-1-0.smithy | 2 +- .../mixins-usage-in-implicit-1-0.smithy | 2 +- .../invalid/mixins/with-but-no-ids.smithy | 2 +- .../with-on-next-line-bigDecimal.smithy | 3 +- .../with-on-next-line-bigInteger.smithy | 3 +- .../mixins/with-on-next-line-blob.smithy | 3 +- .../mixins/with-on-next-line-boolean.smithy | 3 +- .../mixins/with-on-next-line-byte.smithy | 3 +- .../mixins/with-on-next-line-document.smithy | 3 +- .../mixins/with-on-next-line-double.smithy | 3 +- .../mixins/with-on-next-line-float.smithy | 3 +- .../mixins/with-on-next-line-integer.smithy | 3 +- .../mixins/with-on-next-line-list.smithy | 2 +- .../mixins/with-on-next-line-long.smithy | 3 +- .../mixins/with-on-next-line-map.smithy | 2 +- .../mixins/with-on-next-line-resource.smithy | 2 +- .../mixins/with-on-next-line-service.smithy | 2 +- .../mixins/with-on-next-line-set.smithy | 13 - .../mixins/with-on-next-line-short.smithy | 3 +- .../mixins/with-on-next-line-string.smithy | 2 +- .../mixins/with-on-next-line-structure.smithy | 2 +- .../mixins/with-on-next-line-timestamp.smithy | 3 +- .../mixins/with-on-next-line-union.smithy | 2 +- .../namespace/namespace-before-shapes.smithy | 2 +- .../namespace/namespace-before-traits.smithy | 2 +- .../namespace/namespace-defined-twice.smithy | 2 +- .../namespace/namespace-multiple-lines.smithy | 2 +- .../namespace/namespace-no-newline.smithy | 2 +- .../namespace/namespace-syntax-error.smithy | 2 +- .../invalid/namespace/unclosed-string1.smithy | 2 +- ...closed-string2-invalid-single-quote.smithy | 2 +- .../invalid/namespace/unclosed-string3.smithy | 2 +- .../decimal-with-no-digit-after.smithy | 2 +- .../numbers/exponent-with-no-digit.smithy | 2 +- .../exponent-with-sign-then-no-digit.smithy | 2 +- .../invalid/numbers/leading-decimal.smithy | 2 +- .../numbers/negative-with-no-digit.smithy | 2 +- .../operations/input-defined-twice.smithy | 2 +- .../operations/missing-ws-after-input.smithy | 2 +- .../parse-errors/absolute-shape-name.smithy | 4 + .../expected-shape-id-after-type.smithy | 4 + .../expected-shape-name-but-eof.smithy | 4 + .../invalid-newline-after-shape-type.smithy | 2 +- .../missing-space-after-string.smithy | 2 +- .../missing-space-after-structure.smithy | 2 +- .../parse-errors/shape-name-syntax.smithy | 4 + .../parse-errors/shape-name-syntax2.smithy | 4 + .../parse-errors/shape-name-syntax3.smithy | 4 + .../statement-unknown.smithy | 2 +- .../parse-errors/syntax-error-token.smithy | 2 + .../loader/invalid/set-invalid-member.smithy | 6 - .../loader/invalid/shape-name-syntax.smithy | 4 - .../loader/invalid/shape-name-syntax2.smithy | 4 - .../loader/invalid/shape-name-syntax3.smithy | 4 - .../strings/invalid-escape-apostrophe.smithy | 2 +- .../strings/invalid-escape-space.smithy | 2 +- .../strings/invalid-unicode-escape.smithy | 2 +- .../strings/invalid-unicode-escape2.smithy | 2 +- .../strings/invalid-unicode-escape3.smithy | 2 +- .../strings/invalid-unicode-escape4.smithy | 2 +- .../strings/invalid-unicode-escape5.smithy | 2 +- .../invalid/strings/text-block-empty.smithy | 2 +- .../strings/text-block-invalid-unicode.smithy | 2 +- .../strings/text-block-missing-newline.smithy | 2 +- .../invalid/strings/unclosed-string.smithy | 2 +- .../invalid/strings/unclosed-string2.smithy | 2 +- .../invalid/strings/unclosed-string3.smithy | 2 +- .../strings/unclosed-text-block.smithy | 2 +- .../loader/invalid/syntax-error-token.smithy | 2 - .../loader/invalid/trait-name-syntax.smithy | 5 - .../invalid/trait-targets/trait-on-eof.smithy | 4 - .../trait-targets/trait-trailing.smithy | 8 - .../trait-targets/traits-on-apply.smithy | 5 - .../trait-targets/traits-on-metadata.smithy | 3 - .../trait-targets/traits-on-namespace.smithy | 3 - .../annotation-invalid-object-key1.smithy | 4 + .../annotation-invalid-object-key2.smithy | 4 + .../annotation-invalid-object-key3.smithy | 4 + .../traits/annotation-unclosed-list1.smithy | 4 + .../traits/annotation-unclosed-list2.smithy | 4 + .../traits/annotation-unclosed-list3.smithy | 4 + .../traits/annotation-unclosed-list4.smithy | 3 + .../traits/annotation-unclosed-object1.smithy | 4 + .../traits/annotation-unclosed-object2.smithy | 4 + .../traits/annotation-unclosed-object3.smithy | 4 + .../traits/annotation-unclosed-object4.smithy | 4 + .../traits/annotation-unclosed-object5.smithy | 4 + .../traits/annotation-unclosed-object6.smithy | 4 + .../invalid-trait-shape-id-key.smithy | 2 +- .../invalid-trait-textblock-key.smithy | 2 +- .../invalid/traits/trait-name-syntax.smithy | 5 + .../loader/invalid/traits/trait-on-eof.smithy | 4 + .../invalid/traits/trait-trailing.smithy | 8 + .../invalid/traits/traits-on-apply.smithy | 5 + .../invalid/traits/traits-on-metadata.smithy | 3 + .../invalid/traits/traits-on-namespace.smithy | 3 + .../loader/invalid/use/use-after-shape.smithy | 2 +- .../invalid/use/use-after-traitdef.smithy | 2 +- .../invalid/use/use-empty-namespace.smithy | 2 +- .../invalid/use/use-invalid-namespace.smithy | 2 +- .../use/use-missing-after-namespace.smithy | 2 +- .../invalid/use/use-missing-namespace.smithy | 2 +- .../invalid/use/use-missing-newline.smithy | 2 +- .../invalid/use/use-missing-space.smithy | 2 +- .../use/use-multiple-lines-comment.smithy | 2 +- .../invalid/use/use-multiple-lines.smithy | 2 +- .../use/use-must-come-after-namespace.smithy | 2 +- .../use/use-statement-invalid-id.smithy | 2 +- .../loader/invalid/use/use-with-member.smithy | 5 + .../{ => version}/idl-int-enums-in-v1.smithy | 0 .../{ => version}/invalid-version-type.smithy | 0 .../{ => version}/properties-v2-only.json | 0 .../{ => version}/properties-v2-only.smithy | 0 .../version/unsupported-version.smithy | 2 +- .../version/version-set-more-than-once.smithy | 2 +- .../doc-comments-ignored-after-shape.json | 1 + .../doc-comments-ignored-after-shape.smithy | 1 + .../valid/doc-comments/doc-comments.json | 5 +- .../valid/doc-comments/doc-comments.smithy | 2 +- .../model/loader/valid/enums/enum-docs.json | 3 + .../model/loader/valid/enums/enum-docs.smithy | 1 + .../amazon/smithy/utils/CodeFormatter.java | 14 +- .../amazon/smithy/utils/MediaType.java | 2 +- .../amazon/smithy/utils/SimpleParser.java | 80 +- .../amazon/smithy/utils/SimpleParserTest.java | 8 +- 238 files changed, 3167 insertions(+), 1497 deletions(-) rename smithy-model/src/main/java/software/amazon/smithy/model/loader/{IdlModelParser.java => IdlModelLoader.java} (54%) create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java rename smithy-model/src/main/java/software/amazon/smithy/model/loader/{IdlTextParser.java => IdlStringLexer.java} (63%) create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java delete mode 100644 smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java create mode 100644 smithy-model/src/test/java/software/amazon/smithy/model/loader/StringTableTest.java create mode 100644 smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key1.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key2.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key3.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list1.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list2.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list3.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list4.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object1.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object2.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object3.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object4.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object5.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object6.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => elision}/for-after-with.smithy (71%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => elision}/for-referencing-non-resource.smithy (100%) delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/missing-newline-after-assignment.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{enum-values => enums}/enum-with-bad-values.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{enum-values => enums}/enum-with-newline-before-value.smithy (66%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => enums}/enum-without-member.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => enums}/idl-enums-in-v1.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => enums}/int-enum-without-member.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{enum-values => enums}/intEnum-with-bad-value.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{enum-values => enums}/intEnum-with-float.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{enum-values => enums}/intEnum-with-long.smithy (100%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/missing-newline-after-assignment.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-id-after-type.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-name-but-eof.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-shape-id-key.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-text-block-key.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-unquoted-shape-id.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-invalid-member.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => lists}/list-empty-members.smithy (100%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-invalid-member.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => lists}/no-sets-in-v2.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => lists}/set-empty-members.smithy (100%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-invalid-member.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters1.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters2.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters3.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters4.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => maps}/map-empty-members.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => maps}/map-invalid-member.smithy (54%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters1.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters2.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters3.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters4.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-shape-id-key.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-text-block-key.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-unquoted-shape-id.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-set.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/absolute-shape-name.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-id-after-type.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax2.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax3.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => parse-errors}/statement-unknown.smithy (74%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/syntax-error-token.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-invalid-member.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax2.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax3.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/syntax-error-token.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-name-syntax.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-on-eof.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-trailing.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-apply.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-metadata.smithy delete mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-namespace.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key1.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key2.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key3.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list1.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list2.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list3.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list4.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object1.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object2.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object3.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object4.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object5.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object6.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => traits}/invalid-trait-shape-id-key.smithy (51%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => traits}/invalid-trait-textblock-key.smithy (51%) create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-name-syntax.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-on-eof.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-trailing.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-apply.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-metadata.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-namespace.smithy create mode 100644 smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-with-member.smithy rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => version}/idl-int-enums-in-v1.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => version}/invalid-version-type.smithy (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => version}/properties-v2-only.json (100%) rename smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/{ => version}/properties-v2-only.smithy (100%) diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java similarity index 54% rename from smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java rename to smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java index 5ded16f137c..187edded716 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -17,8 +17,6 @@ import static java.lang.String.format; -import java.math.BigDecimal; -import java.math.BigInteger; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -26,31 +24,19 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.StringJoiner; -import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import software.amazon.smithy.model.SourceLocation; import software.amazon.smithy.model.node.ArrayNode; import software.amazon.smithy.model.node.Node; -import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; import software.amazon.smithy.model.shapes.AbstractShapeBuilder; -import software.amazon.smithy.model.shapes.BigDecimalShape; -import software.amazon.smithy.model.shapes.BigIntegerShape; -import software.amazon.smithy.model.shapes.BlobShape; -import software.amazon.smithy.model.shapes.BooleanShape; -import software.amazon.smithy.model.shapes.ByteShape; import software.amazon.smithy.model.shapes.CollectionShape; -import software.amazon.smithy.model.shapes.DocumentShape; -import software.amazon.smithy.model.shapes.DoubleShape; import software.amazon.smithy.model.shapes.EnumShape; -import software.amazon.smithy.model.shapes.FloatShape; import software.amazon.smithy.model.shapes.IntEnumShape; -import software.amazon.smithy.model.shapes.IntegerShape; import software.amazon.smithy.model.shapes.ListShape; -import software.amazon.smithy.model.shapes.LongShape; import software.amazon.smithy.model.shapes.MapShape; import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; @@ -59,10 +45,7 @@ import software.amazon.smithy.model.shapes.SetShape; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.shapes.ShapeType; -import software.amazon.smithy.model.shapes.ShortShape; -import software.amazon.smithy.model.shapes.StringShape; import software.amazon.smithy.model.shapes.StructureShape; -import software.amazon.smithy.model.shapes.TimestampShape; import software.amazon.smithy.model.shapes.UnionShape; import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.DocumentationTrait; @@ -75,13 +58,8 @@ import software.amazon.smithy.model.validation.ValidationEvent; import software.amazon.smithy.model.validation.Validator; import software.amazon.smithy.utils.ListUtils; -import software.amazon.smithy.utils.SimpleParser; -import software.amazon.smithy.utils.StringUtils; -final class IdlModelParser extends SimpleParser { - - /** Only allow nesting up to 250 arrays/objects in node values. */ - private static final int MAX_NESTING_LEVEL = 250; +final class IdlModelLoader { private static final String PUT_KEY = "put"; private static final String CREATE_KEY = "create"; @@ -115,76 +93,83 @@ final class IdlModelParser extends SimpleParser { } private final String filename; + private final IdlTokenizer tokenizer; private final Map useShapes = new HashMap<>(); + private final IdlReferenceResolver resolver; private Consumer operations; private Version modelVersion = Version.VERSION_1_0; private String namespace; - private TraitEntry pendingDocumentationComment; private boolean emittedVersion = false; private String operationInputSuffix = "Input"; private String operationOutputSuffix = "Output"; - // A pending trait that also doesn't yet have a resolved trait shape ID. - static final class TraitEntry { - final String traitName; - final Node value; - final boolean isAnnotation; - - TraitEntry(String traitName, Node value, boolean isAnnotation) { - this.traitName = traitName; - this.value = value; - this.isAnnotation = isAnnotation; - } - } - - IdlModelParser(String filename, String model) { - super(model, MAX_NESTING_LEVEL); + IdlModelLoader(String filename, CharSequence model, Function stringTable) { this.filename = filename; + this.tokenizer = IdlTokenizer.builder() + .filename(filename) + .model(model) + .stringTable(stringTable) + .build(); + this.resolver = this::addForwardReference; } void parse(Consumer operationConsumer) { operations = operationConsumer; - ws(); + tokenizer.skipWsAndDocs(); parseControlSection(); // Emit a version from the current location if the assumed 1.0 is used. if (!emittedVersion) { - operations.accept(new LoadOperation.ModelVersion(modelVersion, currentLocation())); + addOperation(new LoadOperation.ModelVersion(modelVersion, tokenizer.getCurrentTokenLocation())); } + tokenizer.skipWsAndDocs(); parseMetadataSection(); parseShapeSection(); } - LoadOperation.DefineShape createShape(AbstractShapeBuilder builder) { - return new LoadOperation.DefineShape(modelVersion, builder); + void emit(ValidationEvent event) { + addOperation(new LoadOperation.Event(event)); } void addOperation(LoadOperation operation) { operations.accept(operation); } - void emit(ValidationEvent event) { - addOperation(new LoadOperation.Event(event)); + public ModelSyntaxException syntax(String message) { + return syntax(null, message); + } + + ModelSyntaxException syntax(ShapeId shapeId, String message) { + return ModelSyntaxException.builder() + .message(format("Syntax error at line %d, column %d: %s", + tokenizer.getCurrentTokenLine(), tokenizer.getCurrentTokenColumn(), message)) + .sourceLocation(tokenizer.getCurrentTokenLocation()) + .shapeId(shapeId) + .build(); } - void addForwardReference(String id, BiConsumer> consumer) { + void addForwardReference(String id, BiFunction receiver) { int memberPosition = id.indexOf('$'); // Check for members by removing the member and checking for the root shape. if (memberPosition > 0 && memberPosition < id.length() - 1) { - addForwardReference(id.substring(0, memberPosition), (resolved, type) -> { - consumer.accept(resolved.withMember(id.substring(memberPosition + 1)), type); - }); + addForwardReference( + id.substring(0, memberPosition), + (resolved, type) -> receiver.apply(resolved.withMember(id.substring(memberPosition + 1)), type) + ); } else { String resolved = useShapes.containsKey(id) ? useShapes.get(id).toString() : id; - addOperation(new LoadOperation.ForwardReference(namespace, resolved, consumer)); + addOperation(new LoadOperation.ForwardReference(namespace, resolved, receiver)); } } void addForwardReference(String id, Consumer consumer) { - addForwardReference(id, (resolved, found) -> consumer.accept(resolved)); + addForwardReference(id, (resolved, found) -> { + consumer.accept(resolved); + return null; + }); } String expectNamespace() { @@ -195,100 +180,73 @@ String expectNamespace() { } /** - * Overrides whitespace parsing to handle comments. + * Adds a trait to a shape after resolving all shape IDs. + * + *

Resolving the trait against a trait definition is deferred until + * the entire model is loaded. A namespace is required to have been set + * if the trait name is not absolute. + * + * @param target Shape to add the trait to. + * @param traitName Trait name to add. + * @param traitValue Trait value as a Node object. + * @param isAnnotation Set to true to indicate that the value for the trait was omitted. */ - @Override - public void ws() { - while (!eof()) { - switch (peek()) { - case '/': - if (peekDocComment()) { - parseDocComment(); - } else { - parseComment(); - } - break; - case ' ': - case '\t': - case '\r': - case '\n': - case ',': - skip(); - break; - default: - return; - } - } - } - - // required space - private void rsp() { - int cc = column(); - sp(); - if (column() == cc) { - throw syntax("Expected one or more spaces"); - } - } - - // Required whitespace. - private void rws() { - int line = line(); - int column = column(); - ws(); - if (line() == line && column == column()) { - throw syntax("Expected one or more whitespace characters"); - } + private void onDeferredTrait(ShapeId target, String traitName, Node traitValue, boolean isAnnotation) { + addForwardReference(traitName, (traitId, type) -> { + Node coerced = coerceTraitValue(traitValue, isAnnotation, type); + addOperation(new LoadOperation.ApplyTrait( + modelVersion, + traitValue.getSourceLocation(), + expectNamespace(), + target, + traitId, + coerced + )); + return null; + }); } - @Override - public void sp() { - while (isSpaceOrComma(peek())) { - skip(); + private Node coerceTraitValue(Node value, boolean isAnnotation, ShapeType targetType) { + if (isAnnotation && value.isNullNode()) { + if (targetType == null || targetType == ShapeType.STRUCTURE || targetType == ShapeType.MAP) { + // The targetType == null condition helps mitigate a confusing + // failure mode where a trait isn't defined in the model, but a + // TraitService is found as a service provider for the trait. + // If the TraitService creates an annotation trait, then using null + // instead of object results in a failure about passing null for an + // annotation trait, and that's confusing because the actual error + // message should be about the missing trait definition. Because the + // vast majority of annotation traits are modeled as objects, this + // makes the assumption that the value is an object (which addresses + // the above failure case). + return new ObjectNode(Collections.emptyMap(), value.getSourceLocation()); + } else if (targetType == ShapeType.LIST || targetType == ShapeType.SET) { + return new ArrayNode(Collections.emptyList(), value.getSourceLocation()); + } } - } - private boolean isSpaceOrComma(char c) { - return c == ' ' || c == '\t' || c == ','; + return value; } - @Override - public void br() { - int line = line(); - ws(); - if (line == line() && !eof()) { - throw syntax("Expected a line break"); - } - } + private void parseControlSection() { + Set definedKeys = new HashSet<>(); - @Override - public ModelSyntaxException syntax(String message) { - return syntax(null, message); - } + while (tokenizer.getCurrentToken() == IdlToken.DOLLAR) { + tokenizer.next(); + tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); + String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); - ModelSyntaxException syntax(ShapeId shapeId, String message) { - return ModelSyntaxException.builder() - .message(format("Parse error at line %d, column %d near `%s`: %s", - line(), column(), peekDebugMessage(), message)) - .sourceLocation(filename, line(), column()) - .shapeId(shapeId) - .build(); - } + tokenizer.next(); + tokenizer.skipSpaces(); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + tokenizer.skipSpaces(); - private void parseControlSection() { - Set definedKeys = new HashSet<>(); - while (peek() == '$') { - expect('$'); - String key = IdlNodeParser.parseNodeObjectKey(this); - sp(); - expect(':'); - sp(); - - if (definedKeys.contains(key)) { + if (!definedKeys.add(key)) { throw syntax(format("Duplicate control statement `%s`", key)); } - definedKeys.add(key); - Node value = IdlNodeParser.parseNode(this); + Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); switch (key) { case "version": @@ -311,7 +269,7 @@ private void parseControlSection() { break; } - br(); + tokenizer.expectAndSkipBr(); } } @@ -330,378 +288,435 @@ private void onVersion(Node value) { emittedVersion = true; modelVersion = resolvedVersion; - operations.accept(new LoadOperation.ModelVersion(modelVersion, value.getSourceLocation())); + addOperation(new LoadOperation.ModelVersion(modelVersion, value.getSourceLocation())); } private void parseMetadataSection() { - while (peek() == 'm') { - expect('m'); - expect('e'); - expect('t'); - expect('a'); - expect('d'); - expect('a'); - expect('t'); - expect('a'); - rsp(); - String key = IdlNodeParser.parseNodeObjectKey(this); - sp(); - expect('='); - sp(); - operations.accept(new LoadOperation.PutMetadata(modelVersion, key, IdlNodeParser.parseNode(this))); - br(); + while (tokenizer.doesCurrentIdentifierStartWith('m')) { + tokenizer.expectCurrentLexeme("metadata"); + tokenizer.next(); // skip "metadata" + tokenizer.expectAndSkipSpaces(); + tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); + String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + tokenizer.next(); + tokenizer.skipSpaces(); + tokenizer.expect(IdlToken.EQUAL); + tokenizer.next(); + tokenizer.skipSpaces(); + Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + operations.accept(new LoadOperation.PutMetadata(modelVersion, key, value)); + tokenizer.expectAndSkipBr(); } } private void parseShapeSection() { - if (peek() == 'n') { - expect('n'); - expect('a'); - expect('m'); - expect('e'); - expect('s'); - expect('p'); - expect('a'); - expect('c'); - expect('e'); - rsp(); + if (tokenizer.doesCurrentIdentifierStartWith('n')) { + tokenizer.expectCurrentLexeme("namespace"); + tokenizer.next(); // skip "namespace" + tokenizer.expectAndSkipSpaces(); // Parse the namespace. - int start = position(); - ParserUtils.consumeNamespace(this); - namespace = sliceFrom(start); - // Clear out any erroneous documentation comments. - clearPendingDocs(); - br(); - + namespace = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeIdNamespace(tokenizer)); + tokenizer.expectAndSkipBr(); + + // An unfortunate side effect of allowing insignificant documentation comments: + // Keep track of a potential documentation comment location because we need to skip doc comments to parse + // a potential `use statement`. If a `use` statement is present, captured documentation comments are + // correctly cleared from the lexer. If not found, the captured comments are applied to the first shape. + SourceLocation possibleDocCommentLocation = tokenizer.getCurrentTokenLocation(); + tokenizer.skipWsAndDocs(); parseUseSection(); - parseShapeStatements(); - } else if (!eof()) { - if (!ParserUtils.isIdentifierStart(peek())) { - throw syntax("Expected a namespace definition, but found unexpected syntax"); - } else { - throw syntax("A namespace must be defined before a use statement or shapes"); - } + parseFirstShapeStatement(possibleDocCommentLocation); + parseSubsequentShapeStatements(); + } else if (tokenizer.hasNext()) { + throw syntax("Expected a namespace definition but found " + + tokenizer.getCurrentToken().getDebug(tokenizer.getCurrentTokenLexeme())); } } private void parseUseSection() { - while (peek() == 'u' && peek(1) == 's') { - expect('u'); - expect('s'); - expect('e'); - rsp(); - - int start = position(); - SourceLocation location = currentLocation(); - ParserUtils.consumeNamespace(this); - expect('#'); - ParserUtils.consumeIdentifier(this); - String lexeme = sliceFrom(start); - // Clear out any erroneous documentation comments. - clearPendingDocs(); - br(); - - ShapeId target = ShapeId.from(lexeme); + while (tokenizer.getCurrentToken() == IdlToken.IDENTIFIER) { + // Don't over-parse here for unions. + String keyword = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + if (!keyword.equals("use")) { + break; + } + + tokenizer.next(); // skip "use" + tokenizer.expectAndSkipSpaces(); + + SourceLocation idLocation = tokenizer.getCurrentTokenLocation(); + String idString = tokenizer.internString(IdlShapeIdParser.expectAndSkipAbsoluteShapeId(tokenizer)); + ShapeId id = ShapeId.from(idString); + + if (id.hasMember()) { + throw new ModelSyntaxException("Use statements cannot use members", idLocation); + } + + if (useShapes.containsKey(id.getName())) { + ShapeId previous = useShapes.get(id.getName()); + String message = String.format("Cannot use name `%s` because it conflicts with `%s`", id, previous); + throw new ModelSyntaxException(message, idLocation); + } else { + useShapes.put(id.getName(), id); + } // Validate use statements when the model is fully loaded. - addForwardReference(lexeme, (resolved, typeProvider) -> { - if (typeProvider.apply(resolved) == null) { - ValidationEvent event = ValidationEvent.builder() + addForwardReference(idString, (resolved, type) -> { + if (type != null) { + return null; + } else { + return ValidationEvent.builder() .id(Validator.MODEL_ERROR) .severity(Severity.WARNING) - .sourceLocation(location) - .message("Use statement refers to undefined shape: " + lexeme) + .sourceLocation(idLocation) + .message("Use statement refers to undefined shape: " + id) .build(); - emit(event); } }); - useShape(target, location); + tokenizer.expectAndSkipBr(); + tokenizer.skipWsAndDocs(); } } - void useShape(ShapeId id, SourceLocation location) { - if (useShapes.containsKey(id.getName())) { - ShapeId previous = useShapes.get(id.getName()); - String message = String.format("Cannot use name `%s` because it conflicts with `%s`", id, previous); - throw new ModelSyntaxException(message, location); + private void parseApplyStatement() { + tokenizer.expectCurrentLexeme("apply"); + tokenizer.next(); + tokenizer.expectAndSkipWhitespace(); + String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + tokenizer.expectAndSkipWhitespace(); + List traitsToApply; + + if (IdlToken.AT == tokenizer.expect(IdlToken.AT, IdlToken.LBRACE)) { + // Parse a single trait. + traitsToApply = Collections.singletonList(IdlTraitParser.expectAndSkipTrait(tokenizer, resolver)); + } else { + // Parse a trait block. + tokenizer.next(); // skip opening LBRACE. + tokenizer.skipWsAndDocs(); + traitsToApply = IdlTraitParser.expectAndSkipTraits(tokenizer, resolver); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); } - useShapes.put(id.getName(), id); + // First, resolve the targeted shape. + addForwardReference(target, id -> { + for (IdlTraitParser.Result trait : traitsToApply) { + String traitNameString = tokenizer.internString(trait.getTraitName()); + onDeferredTrait(id, traitNameString, trait.getValue(), + trait.getTraitType() == IdlTraitParser.TraitType.ANNOTATION); + } + }); + + tokenizer.expectAndSkipBr(); } - private void parseShapeStatements() { - while (!eof()) { - if (peek() == 'a') { + private void parseFirstShapeStatement(SourceLocation possibleDocCommentLocation) { + if (tokenizer.getCurrentToken() != IdlToken.EOF) { + if (tokenizer.doesCurrentIdentifierStartWith('a')) { parseApplyStatement(); } else { - boolean docsOnly = pendingDocumentationComment != null; - List traits = parseDocsAndTraits(); - if (parseShapeDefinition(traits, docsOnly)) { + List traits; + boolean hasDocComment = tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT; + + if (possibleDocCommentLocation == null) { + traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + } else { + // In this case, this is the first shape encountered for a model file that doesn't have any + // use statements. We need to take the previously skipped documentation comments to parse + // potential use statements and apply them to this first shape. + String docLines = tokenizer.removePendingDocCommentLines(); + traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + // Note that possibleDocCommentLocation is just a mark of where docs _could be_. + if (docLines != null) { + hasDocComment = true; + traits.add(new IdlTraitParser.Result(DocumentationTrait.ID.toString(), + new StringNode(docLines, possibleDocCommentLocation), + IdlTraitParser.TraitType.DOC_COMMENT)); + } + } + + if (parseShapeDefinition(traits, hasDocComment)) { parseShape(traits); } } } } - private void clearPendingDocs() { - pendingDocumentationComment = null; - } - - private boolean parseShapeDefinition(List traits, boolean docsOnly) { - if (eof()) { - return !traits.isEmpty() && !docsOnly; - } else { - return true; + private void parseSubsequentShapeStatements() { + while (tokenizer.getCurrentToken() != IdlToken.EOF) { + if (tokenizer.doesCurrentIdentifierStartWith('a')) { + parseApplyStatement(); + } else { + List traits; + boolean hasDocComment = tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT; + traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + if (parseShapeDefinition(traits, hasDocComment)) { + parseShape(traits); + } + } } } - private List parseDocsAndTraits() { - // Grab the pending docs, if present, and clear its state. - TraitEntry docComment = pendingDocumentationComment; - clearPendingDocs(); - - // Parse traits, if any. - ws(); - List traits = IdlTraitParser.parseTraits(this); - if (docComment != null) { - traits.add(docComment); + private boolean parseShapeDefinition(List traits, boolean hasDocComment) { + if (tokenizer.getCurrentToken() != IdlToken.EOF) { + // Continue to parse if not at the end of the file. + return true; + } else if (hasDocComment) { + // Ignore a standalone documentation comment and treat it like a normal comment. + return traits.size() > 1; + } else { + // Fail because there are traits, it's not just a documentation comment, and there's no more IDL to parse. + return !traits.isEmpty(); } - ws(); - - return traits; } - private void parseShape(List traits) { - SourceLocation location = currentLocation(); - - // Do a check here to give better parsing error messages. - String shapeType = ParserUtils.parseIdentifier(this); - if (!SHAPE_TYPES.contains(shapeType)) { - switch (shapeType) { - case "use": - throw syntax("A use statement must come before any shape definition"); - case "namespace": - throw syntax("Only a single namespace can be declared per/file"); - case "metadata": - throw syntax("Metadata statements must appear before a namespace statement"); - default: - throw syntax("Unexpected shape type: " + shapeType); - } - } - - rsp(); + private void parseShape(List traits) { + SourceLocation location = tokenizer.getCurrentTokenLocation(); + tokenizer.expect(IdlToken.IDENTIFIER); + String shapeType = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + ShapeType type = ShapeType.fromString(shapeType) + .orElseThrow(() -> syntax("Unknown shape type: " + shapeType)); + tokenizer.next(); + tokenizer.expectAndSkipSpaces(); ShapeId id = parseShapeName(); - switch (shapeType) { - case "service": - parseServiceStatement(id, location); - break; - case "resource": - parseResourceStatement(id, location); - break; - case "operation": - parseOperationStatement(id, location); - break; - case "structure": - parseStructuredShape(id, location, StructureShape.builder(), MemberParsing.PARSING_STRUCTURE_MEMBER); - break; - case "union": - parseStructuredShape(id, location, UnionShape.builder(), MemberParsing.PARSING_MEMBER); + switch (type) { + case STRING: + case BLOB: + case BOOLEAN: + case DOCUMENT: + case BYTE: + case SHORT: + case INTEGER: + case LONG: + case FLOAT: + case DOUBLE: + case BIG_DECIMAL: + case BIG_INTEGER: + case TIMESTAMP: + parseSimpleShape(id, location, type.createBuilderForType()); break; - case "list": + case LIST: parseCollection(id, location, ListShape.builder()); break; - case "set": + case SET: parseCollection(id, location, SetShape.builder()); break; - case "map": + case MAP: parseMapStatement(id, location); break; - case "boolean": - parseSimpleShape(id, location, BooleanShape.builder()); - break; - case "string": - parseSimpleShape(id, location, StringShape.builder()); - break; - case "enum": + case ENUM: parseEnumShape(id, location, EnumShape.builder()); break; - case "blob": - parseSimpleShape(id, location, BlobShape.builder()); - break; - case "byte": - parseSimpleShape(id, location, ByteShape.builder()); - break; - case "short": - parseSimpleShape(id, location, ShortShape.builder()); - break; - case "integer": - parseSimpleShape(id, location, IntegerShape.builder()); - break; - case "intEnum": + case INT_ENUM: parseEnumShape(id, location, IntEnumShape.builder()); break; - case "long": - parseSimpleShape(id, location, LongShape.builder()); - break; - case "float": - parseSimpleShape(id, location, FloatShape.builder()); - break; - case "document": - parseSimpleShape(id, location, DocumentShape.builder()); + case STRUCTURE: + parseStructuredShape(id, location, StructureShape.builder(), MemberParsing.PARSING_STRUCTURE_MEMBER); break; - case "double": - parseSimpleShape(id, location, DoubleShape.builder()); + case UNION: + parseStructuredShape(id, location, UnionShape.builder(), MemberParsing.PARSING_MEMBER); break; - case "bigInteger": - parseSimpleShape(id, location, BigIntegerShape.builder()); + case SERVICE: + parseServiceStatement(id, location); break; - case "bigDecimal": - parseSimpleShape(id, location, BigDecimalShape.builder()); + case RESOURCE: + parseResourceStatement(id, location); break; - case "timestamp": - parseSimpleShape(id, location, TimestampShape.builder()); + case OPERATION: + parseOperationStatement(id, location); break; default: - // Unreachable. - throw syntax(id, "Unexpected shape type: " + shapeType); + throw syntax("Shape type unknown: " + shapeType); } addTraits(id, traits); - clearPendingDocs(); - br(); + tokenizer.expectAndSkipBr(); + } + + private void addTraits(ShapeId id, List traits) { + for (IdlTraitParser.Result result : traits) { + String traitName = tokenizer.internString(result.getTraitName()); + onDeferredTrait(id, traitName, result.getValue(), + result.getTraitType() == IdlTraitParser.TraitType.ANNOTATION); + } } private ShapeId parseShapeName() { - SourceLocation currentLocation = currentLocation(); - String name = ParserUtils.parseIdentifier(this); + int line = tokenizer.getCurrentTokenLine(); + int column = tokenizer.getCurrentTokenColumn(); + tokenizer.expect(IdlToken.IDENTIFIER); + String name = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); ShapeId id = ShapeId.fromRelative(expectNamespace(), name); - if (useShapes.containsKey(name)) { ShapeId previous = useShapes.get(name); String message = String.format("Shape name `%s` conflicts with imported shape `%s`", name, previous); - throw new ModelSyntaxException(message, currentLocation); + throw new ModelSyntaxException(message, filename, line, column); } - + tokenizer.next(); return id; } private void parseSimpleShape(ShapeId id, SourceLocation location, AbstractShapeBuilder builder) { LoadOperation.DefineShape operation = createShape(builder.source(location).id(id)); parseMixins(operation); - operations.accept(operation); + addOperation(operation); } - private void parseEnumShape(ShapeId id, SourceLocation location, AbstractShapeBuilder builder) { - LoadOperation.DefineShape operation = createShape(builder.id(id).source(location)); - parseMixins(operation); + LoadOperation.DefineShape createShape(AbstractShapeBuilder builder) { + return new LoadOperation.DefineShape(modelVersion, builder); + } - ws(); - expect('{'); - clearPendingDocs(); - ws(); + private void parseMixins(LoadOperation.DefineShape operation) { + tokenizer.skipSpaces(); - while (!eof() && peek() != '}') { - List memberTraits = parseDocsAndTraits(); - SourceLocation memberLocation = currentLocation(); - String memberName = ParserUtils.parseIdentifier(this); - MemberShape.Builder memberBuilder = MemberShape.builder() - .id(id.withMember(memberName)) - .source(memberLocation) - .target(UnitTypeTrait.UNIT); - operation.addMember(memberBuilder); - addTraits(memberBuilder.getId(), memberTraits); + if (!tokenizer.doesCurrentIdentifierStartWith('w')) { + return; + } - // Check for optional value assignment. - sp(); - if (peek() == '=') { - expect('='); - sp(); - Node value = IdlNodeParser.parseNode(this); - memberBuilder.addTrait(new EnumValueTrait.Provider().createTrait(memberBuilder.getId(), value)); - clearPendingDocs(); - br(); - } else { - ws(); - } + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.expectCurrentLexeme("with"); + + if (!modelVersion.supportsMixins()) { + throw syntax(operation.toShapeId(), "Mixins can only be used with Smithy version 2 or later. " + + "Attempted to use mixins with version `" + modelVersion + "`."); } - expect('}'); - clearPendingDocs(); - operations.accept(operation); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + + do { + String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(target, resolved -> { + operation.addDependency(resolved); + operation.addModifier(new ApplyMixin(resolved)); + }); + tokenizer.skipWsAndDocs();; + } while (tokenizer.getCurrentToken() != IdlToken.RBRACKET); + + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); } - // See parseMap for information on why members are parsed before the - // list/set is registered with the ModelFile. + // See parseMap for information on why members are parsed before the list/set is registered with the ModelFile. private void parseCollection(ShapeId id, SourceLocation location, CollectionShape.Builder builder) { LoadOperation.DefineShape operation = createShape(builder.id(id).source(location)); parseMixins(operation); - ws(); - expect('{'); - clearPendingDocs(); - ws(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWs(); parsePossiblyElidedMember(operation, "member"); - ws(); - expect('}'); - - clearPendingDocs(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); operations.accept(operation); } // Parsed list, set, and map members. private void parsePossiblyElidedMember(LoadOperation.DefineShape operation, String memberName) { boolean isElided = false; - List memberTraits = parseDocsAndTraits(); + List memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + SourceLocation location = tokenizer.getCurrentTokenLocation(); - if (peek() == '$') { + if (tokenizer.getCurrentToken() == IdlToken.DOLLAR) { isElided = true; if (!modelVersion.supportsTargetElision()) { throw syntax(operation.toShapeId().withMember(memberName), "Members can only elide targets in IDL version 2 or later"); } - expect('$'); - } else if (peek() != memberName.charAt(0)) { - if (!memberTraits.isEmpty()) { - throw syntax("Expected member definition to follow traits"); + tokenizer.next(); + tokenizer.expect(IdlToken.IDENTIFIER); + } else { + if (!tokenizer.doesCurrentIdentifierStartWith(memberName.charAt(0))) { + if (!memberTraits.isEmpty()) { + throw syntax("Expected member definition to follow traits"); + } + return; } - return; } MemberShape.Builder memberBuilder = MemberShape.builder() .id(operation.toShapeId().withMember(memberName)) - .source(currentLocation()); + .source(location); - for (int i = 0; i < memberName.length(); i++) { - expect(memberName.charAt(i)); - } + tokenizer.expectCurrentLexeme(memberName); + tokenizer.next(); if (!isElided) { - sp(); - expect(':'); - sp(); - addForwardReference(ParserUtils.parseShapeId(this), memberBuilder::target); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + String id = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(id, memberBuilder::target); } operation.addMember(memberBuilder); addTraits(memberBuilder.getId(), memberTraits); - clearPendingDocs(); + } + + private void parseEnumShape(ShapeId id, SourceLocation location, AbstractShapeBuilder builder) { + LoadOperation.DefineShape operation = createShape(builder.id(id).source(location)); + parseMixins(operation); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWs(); + + while (tokenizer.getCurrentToken() != IdlToken.EOF && tokenizer.getCurrentToken() != IdlToken.RBRACE) { + List memberTraits = IdlTraitParser + .parseDocsAndTraitsBeforeShape(tokenizer, resolver); + SourceLocation memberLocation = tokenizer.getCurrentTokenLocation(); + tokenizer.expect(IdlToken.IDENTIFIER); + String memberName = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + + MemberShape.Builder memberBuilder = MemberShape.builder() + .id(id.withMember(memberName)) + .source(memberLocation) + .target(UnitTypeTrait.UNIT); + operation.addMember(memberBuilder); + addTraits(memberBuilder.getId(), memberTraits); + + // Check for optional value assignment. + tokenizer.next(); + tokenizer.skipSpaces(); + + if (tokenizer.getCurrentToken() == IdlToken.EQUAL) { + tokenizer.next(); + tokenizer.skipSpaces(); + Node value = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + memberBuilder.addTrait(new EnumValueTrait.Provider().createTrait(memberBuilder.getId(), value)); + tokenizer.expectAndSkipBr(); + } else { + tokenizer.skipWs(); + } + } + + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + operations.accept(operation); } private void parseMapStatement(ShapeId id, SourceLocation location) { LoadOperation.DefineShape operation = createShape(MapShape.builder().id(id).source(location)); parseMixins(operation); - ws(); - expect('{'); - clearPendingDocs(); - ws(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWs(); parsePossiblyElidedMember(operation, "key"); - ws(); + tokenizer.skipWs(); parsePossiblyElidedMember(operation, "value"); - ws(); - expect('}'); - clearPendingDocs(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); operations.accept(operation); } @@ -722,43 +737,9 @@ private void parseStructuredShape( // Parse optional "with" statements to add mixins, but only if it's supported by the version. parseMixins(operation); parseMembers(operation, memberParsing); - clearPendingDocs(); operations.accept(operation); } - private void parseMixins(LoadOperation.DefineShape operation) { - sp(); - if (peek() != 'w') { - return; - } - - expect('w'); - expect('i'); - expect('t'); - expect('h'); - - if (!modelVersion.supportsMixins()) { - throw syntax(operation.toShapeId(), "Mixins can only be used with Smithy version 2 or later. " - + "Attempted to use mixins with version `" + modelVersion + "`."); - } - - ws(); - expect('['); - ws(); - - do { - String target = ParserUtils.parseShapeId(this); - addForwardReference(target, resolved -> { - operation.addDependency(resolved); - operation.addModifier(new ApplyMixin(resolved)); - }); - ws(); - } while (peek() != ']'); - - expect(']'); - clearPendingDocs(); - } - private enum MemberParsing { PARSING_STRUCTURE_MEMBER { @Override @@ -791,41 +772,47 @@ Trait createAssignmentTrait(ShapeId id, Node value) { private void parseMembers(LoadOperation.DefineShape op, MemberParsing memberParsing) { Set definedMembers = new HashSet<>(); - ws(); - expect('{'); - ws(); - - while (!eof()) { - if (peek() == '}') { - break; - } + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWs(); + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { parseMember(op, definedMembers, memberParsing); - - ws(); + tokenizer.skipWs(); } - clearPendingDocs(); - expect('}'); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); } private void parseMember(LoadOperation.DefineShape operation, Set defined, MemberParsing memberParsing) { ShapeId parent = operation.toShapeId(); // Parse optional member traits. - List memberTraits = parseDocsAndTraits(); - SourceLocation memberLocation = currentLocation(); + List memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + SourceLocation memberLocation = tokenizer.getCurrentTokenLocation(); + + // Handle the case of a dangling documentation comment followed by "}". + if (tokenizer.getCurrentToken() == IdlToken.RBRACE) { + if (memberTraits.size() == 1 + && memberTraits.get(0).getTraitType() == IdlTraitParser.TraitType.DOC_COMMENT) { + return; + } + } - boolean isTargetElided = peek() == '$'; + boolean isTargetElided = tokenizer.getCurrentToken() == IdlToken.DOLLAR; if (isTargetElided) { - expect('$'); + tokenizer.expect(IdlToken.DOLLAR); + tokenizer.next(); } - String memberName = ParserUtils.parseIdentifier(this); + tokenizer.expect(IdlToken.IDENTIFIER); + String memberName = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); if (defined.contains(memberName)) { // This is a duplicate member name. - throw syntax(parent, "Duplicate member of " + parent + ": '" + memberName + '\''); + throw syntax(parent, "Duplicate member of `" + parent + "`: '" + memberName + '\''); } defined.add(memberName); @@ -838,32 +825,31 @@ private void parseMember(LoadOperation.DefineShape operation, Set define MemberShape.Builder memberBuilder = MemberShape.builder().id(memberId).source(memberLocation); - // Members whose targets are elided will have those targets resolved later, - // for example by SetResourceBasedTargets + tokenizer.next(); // skip past the identifier. + + // Members whose targets are elided will have those targets resolved later e.g. by SetResourceBasedTargets if (!isTargetElided) { - sp(); - expect(':'); - sp(); - addForwardReference(ParserUtils.parseShapeId(this), memberBuilder::target); + tokenizer.skipSpaces(); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + tokenizer.skipSpaces(); + String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(target, memberBuilder::target); } // Skip spaces to check if there is default trait sugar. - sp(); + tokenizer.skipSpaces(); - if (memberParsing.supportsAssignment() && peek() == '=') { + if (memberParsing.supportsAssignment() && tokenizer.getCurrentToken() == IdlToken.EQUAL) { if (!modelVersion.isDefaultSupported()) { throw syntax("@default assignment is only supported in IDL version 2 or later"); } - expect('='); - sp(); - memberBuilder.addTrait(memberParsing.createAssignmentTrait(memberId, IdlNodeParser.parseNode(this))); - br(); - } else { - // Clears out any previously captured documentation - // comments that may have been found when parsing the member. - // Default value parsing may safely load comments for the next - // member, so leave those intact. - clearPendingDocs(); + tokenizer.expect(IdlToken.EQUAL); + tokenizer.next(); + tokenizer.skipSpaces(); + Node node = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + memberBuilder.addTrait(memberParsing.createAssignmentTrait(memberId, node)); + tokenizer.expectAndSkipBr(); } // Only add the member once fully parsed. @@ -871,102 +857,14 @@ private void parseMember(LoadOperation.DefineShape operation, Set define addTraits(memberBuilder.getId(), memberTraits); } - private void parseOperationStatement(ShapeId id, SourceLocation location) { - OperationShape.Builder builder = OperationShape.builder().id(id).source(location); - LoadOperation.DefineShape operation = createShape(builder); - parseMixins(operation); - ws(); - expect('{'); - ws(); - - parseProperties(id, propertyName -> { - switch (propertyName) { - case "input": - TraitEntry inputTrait = new TraitEntry(InputTrait.ID.toString(), Node.objectNode(), true); - parseInlineableOperationMember(id, operationInputSuffix, builder::input, inputTrait); - break; - case "output": - TraitEntry outputTrait = new TraitEntry(OutputTrait.ID.toString(), Node.objectNode(), true); - parseInlineableOperationMember(id, operationOutputSuffix, builder::output, outputTrait); - break; - case "errors": - parseIdList(builder::addError); - break; - default: - throw syntax(id, String.format("Unknown property %s for %s", propertyName, id)); - } - rws(); - }); - - expect('}'); - clearPendingDocs(); - operations.accept(operation); - } - - private void parseProperties(ShapeId id, Consumer valueParser) { - Set defined = new HashSet<>(); - while (!eof() && peek() != '}') { - String key = ParserUtils.parseIdentifier(this); - if (defined.contains(key)) { - throw syntax(id, String.format("Duplicate operation %s property for %s", key, id)); - } - defined.add(key); - - ws(); - expect(':'); - valueParser.accept(key); - ws(); - } - } - - private void parseInlineableOperationMember( - ShapeId id, - String suffix, - Consumer consumer, - TraitEntry defaultTrait - ) { - if (peek() == '=') { - if (!modelVersion.supportsInlineOperationIO()) { - throw syntax(id, "Inlined operation inputs and outputs can only be used with Smithy version 2 or " - + "later. Attempted to use inlined IO with version `" + modelVersion + "`."); - } - expect('='); - clearPendingDocs(); - ws(); - consumer.accept(parseInlineStructure(id.getName() + suffix, defaultTrait)); - } else { - ws(); - addForwardReference(ParserUtils.parseShapeId(this), consumer); - } - } - - private ShapeId parseInlineStructure(String name, TraitEntry defaultTrait) { - List traits = parseDocsAndTraits(); - if (defaultTrait != null) { - traits.add(defaultTrait); - } - ShapeId id = ShapeId.fromRelative(expectNamespace(), name); - SourceLocation location = currentLocation(); - StructureShape.Builder builder = StructureShape.builder().id(id).source(location); - LoadOperation.DefineShape operation = createShape(builder); - parseMixins(operation); - parseForResource(operation); - parseMembers(operation, MemberParsing.PARSING_STRUCTURE_MEMBER); - addTraits(id, traits); - clearPendingDocs(); - operations.accept(operation); - return id; - } - private void parseForResource(LoadOperation.DefineShape operation) { - sp(); - if (peek() != 'f') { + tokenizer.skipSpaces(); + + if (!tokenizer.doesCurrentIdentifierStartWith('f')) { return; } - expect('f'); - expect('o'); - expect('r'); + tokenizer.expectCurrentLexeme("for"); if (!modelVersion.supportsTargetElision()) { throw syntax(operation.toShapeId(), "Structures can only be bound to resources with Smithy version 2 or " @@ -974,42 +872,29 @@ private void parseForResource(LoadOperation.DefineShape operation) { + modelVersion + "`."); } - rsp(); + tokenizer.next(); + tokenizer.expectAndSkipSpaces(); - addForwardReference(ParserUtils.parseShapeId(this), shapeId -> { + String forTarget = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(forTarget, shapeId -> { operation.addDependency(shapeId); operation.addModifier(new ApplyResourceBasedTargets(shapeId)); }); } - private void parseIdList(Consumer consumer) { - increaseNestingLevel(); - ws(); - expect('['); - ws(); - - while (!eof() && peek() != ']') { - addForwardReference(ParserUtils.parseShapeId(this), consumer); - ws(); - } - - expect(']'); - decreaseNestingLevel(); - } - private void parseServiceStatement(ShapeId id, SourceLocation location) { ServiceShape.Builder builder = new ServiceShape.Builder().id(id).source(location); LoadOperation.DefineShape operation = createShape(builder); parseMixins(operation); - ws(); - ObjectNode shapeNode = IdlNodeParser.parseObjectNode(this, id.toString()); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(tokenizer, resolver).expectObjectNode(); LoaderUtils.checkForAdditionalProperties(shapeNode, id, SERVICE_PROPERTY_NAMES).ifPresent(this::emit); shapeNode.getStringMember(VERSION_KEY).map(StringNode::getValue).ifPresent(builder::version); optionalIdList(shapeNode, OPERATIONS_KEY, builder::addOperation); optionalIdList(shapeNode, RESOURCES_KEY, builder::addResource); optionalIdList(shapeNode, ERRORS_KEY, builder::addError); AstModelLoader.loadServiceRenameIntoBuilder(builder, shapeNode); - clearPendingDocs(); operations.accept(operation); } @@ -1033,9 +918,10 @@ private void parseResourceStatement(ShapeId id, SourceLocation location) { LoadOperation.DefineShape operation = createShape(builder); parseMixins(operation); - ws(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + ObjectNode shapeNode = IdlNodeParser.expectAndSkipNode(tokenizer, resolver).expectObjectNode(); - ObjectNode shapeNode = IdlNodeParser.parseObjectNode(this, id.toString()); LoaderUtils.checkForAdditionalProperties(shapeNode, id, RESOURCE_PROPERTY_NAMES).ifPresent(this::emit); optionalId(shapeNode, PUT_KEY, builder::put); optionalId(shapeNode, CREATE_KEY, builder::create); @@ -1067,194 +953,119 @@ private void parseResourceStatement(ShapeId id, SourceLocation location) { addForwardReference(target.getValue(), targetId -> builder.addProperty(name, targetId)); } }); - clearPendingDocs(); operations.accept(operation); } - // "//" *(not_newline) - private void parseComment() { - expect('/'); - consumeRemainingCharactersOnLine(); - } - - private void parseDocComment() { - SourceLocation location = currentLocation(); - StringJoiner joiner = new StringJoiner("\n"); - do { - joiner.add(parseDocCommentLine()); - } while (peekDocComment()); - pendingDocumentationComment = new TraitEntry( - DocumentationTrait.ID.toString(), new StringNode(joiner.toString(), location), false); - } - - private boolean peekDocComment() { - return peek() == '/' && peek(1) == '/' && peek(2) == '/'; - } - - // documentation_comment = "///" *(not_newline) - private String parseDocCommentLine() { - expect('/'); - expect('/'); - expect('/'); - // Skip a leading space, if present. - if (peek() == ' ') { - skip(); - } - int start = position(); - consumeRemainingCharactersOnLine(); - nl(); - sp(); - return StringUtils.stripEnd(sliceFrom(start), " \t\r\n"); - } - - private void nl() { - switch (peek()) { - case '\n': - skip(); - break; - case '\r': - skip(); - if (peek() == '\n') { - expect('\n'); - } - break; - default: - throw syntax("Expected a newline"); - } - } - - private void parseApplyStatement() { - expect('a'); - expect('p'); - expect('p'); - expect('l'); - expect('y'); - rsp(); - - String name = ParserUtils.parseShapeId(this); - rws(); - - // Account for singular or block apply statements. - List traitsToApply; - if (peek() == '{') { - expect('{'); - ws(); - traitsToApply = IdlTraitParser.parseTraits(this); - expect('}'); - } else { - traitsToApply = Collections.singletonList(IdlTraitParser.parseTraitValue(this)); - } + private void parseOperationStatement(ShapeId id, SourceLocation location) { + OperationShape.Builder builder = OperationShape.builder().id(id).source(location); + LoadOperation.DefineShape operation = createShape(builder); + parseMixins(operation); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWsAndDocs(); - // First, resolve the targeted shape. - addForwardReference(name, target -> { - for (TraitEntry traitEntry : traitsToApply) { - onDeferredTrait(target, traitEntry.traitName, traitEntry.value, traitEntry.isAnnotation); + Set defined = new HashSet<>(); + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACE) { + tokenizer.expect(IdlToken.IDENTIFIER); + String key = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + if (!defined.add(key)) { + throw syntax(id, String.format("Duplicate operation %s property for `%s`", key, id)); } - }); - // Clear out any errantly captured pending docs. - clearPendingDocs(); - br(); - } - - private void addTraits(ShapeId id, List traits) { - for (TraitEntry traitEntry : traits) { - onDeferredTrait(id, traitEntry.traitName, traitEntry.value, traitEntry.isAnnotation); - } - } + tokenizer.next(); + tokenizer.skipWsAndDocs(); - /** - * Adds a trait to a shape after resolving all shape IDs. - * - *

Resolving the trait against a trait definition is deferred until - * the entire model is loaded. A namespace is required to have been set - * if the trait name is not absolute. - * - * @param target Shape to add the trait to. - * @param traitName Trait name to add. - * @param traitValue Trait value as a Node object. - * @param isAnnotation Set to true to indicate that the value for the trait was omitted. - */ - private void onDeferredTrait(ShapeId target, String traitName, Node traitValue, boolean isAnnotation) { - addForwardReference(traitName, (traitId, typeProvider) -> { - Node coerced = coerceTraitValue(traitValue, isAnnotation, typeProvider.apply(traitId)); - operations.accept(new LoadOperation.ApplyTrait( - modelVersion, traitValue.getSourceLocation(), expectNamespace(), target, traitId, coerced)); - }); - } - - private Node coerceTraitValue(Node value, boolean isAnnotation, ShapeType targetType) { - if (isAnnotation && value.isNullNode()) { - if (targetType == null || targetType == ShapeType.STRUCTURE || targetType == ShapeType.MAP) { - // The targetType == null condition helps mitigate a confusing - // failure mode where a trait isn't defined in the model, but a - // TraitService is found as a service provider for the trait. - // If the TraitService creates an annotation trait, then using null - // instead of object results in a failure about passing null for an - // annotation trait, and that's confusing because the actual error - // message should be about the missing trait definition. Because the - // vast majority of annotation traits are modeled as objects, this - // makes the assumption that the value is an object (which addresses - // the above failure case). - return new ObjectNode(Collections.emptyMap(), value.getSourceLocation()); - } else if (targetType == ShapeType.LIST || targetType == ShapeType.SET) { - return new ArrayNode(Collections.emptyList(), value.getSourceLocation()); + switch (key) { + case "input": + IdlToken nextInput = tokenizer.expect(IdlToken.COLON, IdlToken.WALRUS); + tokenizer.next(); + IdlTraitParser.Result inputTrait = new IdlTraitParser.Result( + InputTrait.ID.toString(), Node.objectNode(), IdlTraitParser.TraitType.ANNOTATION); + parseInlineableOperationMember(id, nextInput, operationInputSuffix, builder::input, inputTrait); + break; + case "output": + IdlToken nextOutput = tokenizer.expect(IdlToken.COLON, IdlToken.WALRUS); + tokenizer.next(); + IdlTraitParser.Result outputTrait = new IdlTraitParser.Result( + OutputTrait.ID.toString(), Node.objectNode(), IdlTraitParser.TraitType.ANNOTATION); + parseInlineableOperationMember(id, nextOutput, operationOutputSuffix, builder::output, outputTrait); + break; + case "errors": + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + parseIdList(builder::addError); + break; + default: + throw syntax(id, String.format("Unknown property %s for %s", key, id)); } + tokenizer.expectAndSkipWhitespace(); + tokenizer.skipWsAndDocs(); } - return value; - } - - SourceLocation currentLocation() { - return new SourceLocation(filename, line(), column()); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + operations.accept(operation); } - NumberNode parseNumberNode(SourceLocation location) { - String lexeme = ParserUtils.parseNumber(this); - - if (lexeme.contains("e") || lexeme.contains("E") || lexeme.contains(".")) { - double value = Double.parseDouble(lexeme); - if (Double.isFinite(value)) { - return new NumberNode(value, location); + private void parseInlineableOperationMember( + ShapeId id, + IdlToken token, + String suffix, + Consumer consumer, + IdlTraitParser.Result defaultTrait + ) { + if (token == IdlToken.WALRUS) { + if (!modelVersion.supportsInlineOperationIO()) { + throw syntax(id, "Inlined operation inputs and outputs can only be used with Smithy version 2 or " + + "later. Attempted to use inlined IO with version `" + modelVersion + "`."); } - return new NumberNode(new BigDecimal(lexeme), location); + // Remove any pending, invalid docs that may have come before the inline shape. + tokenizer.removePendingDocCommentLines(); + tokenizer.next(); + // don't skip docs here in case there are docs on the inlined structure. + tokenizer.skipWs(); + consumer.accept(parseInlineStructure(id.getName() + suffix, defaultTrait)); } else { - try { - return new NumberNode(Long.parseLong(lexeme), location); - } catch (NumberFormatException e) { - return new NumberNode(new BigInteger(lexeme), location); - } + tokenizer.skipWsAndDocs(); + String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(target, consumer); } } - private String peekDebugMessage() { - StringBuilder result = new StringBuilder(expression().length()); - - char c = peek(); + private ShapeId parseInlineStructure(String name, IdlTraitParser.Result defaultTrait) { + List traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(tokenizer, resolver); + if (defaultTrait != null) { + traits.add(defaultTrait); + } + ShapeId id = ShapeId.fromRelative(expectNamespace(), name); + SourceLocation location = tokenizer.getCurrentTokenLocation(); + StructureShape.Builder builder = StructureShape.builder().id(id).source(location); + LoadOperation.DefineShape operation = createShape(builder); + parseMixins(operation); + parseForResource(operation); + parseMembers(operation, MemberParsing.PARSING_STRUCTURE_MEMBER); + addTraits(id, traits); + operations.accept(operation); + return id; + } - // Try to read an entire identifier for context (16 char max) if that's what's being peeked. - if (c == ' ' || ParserUtils.isIdentifierStart(c) || ParserUtils.isDigit(c)) { - if (c == ' ') { - result.append(' '); - } - for (int i = c == ' ' ? 1 : 0; i < 16; i++) { - c = peek(i); - if (ParserUtils.isIdentifierStart(c) || ParserUtils.isDigit(c)) { - result.append(c); - } else { - break; - } - } - } else { - // Take two characters for context. - for (int i = 0; i < 2; i++) { - char peek = peek(i); - if (peek != Character.MIN_VALUE) { - result.append(peek); - } - } + private void parseIdList(Consumer consumer) { + tokenizer.increaseNestingLevel(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + + while (tokenizer.hasNext() && tokenizer.getCurrentToken() != IdlToken.RBRACKET) { + tokenizer.expect(IdlToken.IDENTIFIER); + String target = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + addForwardReference(target, consumer); + tokenizer.skipWsAndDocs(); } - return result.length() == 0 ? "[EOF]" : result.toString(); + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); + tokenizer.decreaseNestingLevel(); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java index f5d43ab7656..e98e3154979 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import software.amazon.smithy.model.node.BooleanNode; import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.node.NullNode; +import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; import software.amazon.smithy.model.validation.Severity; @@ -28,161 +29,209 @@ import software.amazon.smithy.utils.Pair; /** - * Parses IDL nodes. + * Parses Node values from a {@link IdlTokenizer}. */ final class IdlNodeParser { private static final String SYNTACTIC_SHAPE_ID_TARGET = "SyntacticShapeIdTarget"; - private IdlNodeParser() {} - - static Node parseNode(IdlModelParser parser) { - return parseNode(parser, parser.currentLocation()); + private IdlNodeParser() { } + + /** + * Expects that the current token is a valid Node, and parses it into a {@link Node} value. + * + *

The tokenizer is advanced to the next token after parsing the Node value.

+ * + * @param tokenizer Tokenizer to consume and advance. + * @param resolver Forward reference resolver. + * @return Returns the parsed node value. + * @throws ModelSyntaxException if the Node is not well-formed. + */ + static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + return expectAndSkipNode(tokenizer, resolver, tokenizer.getCurrentTokenLocation()); } - static Node parseNode(IdlModelParser parser, SourceLocation location) { - char c = parser.peek(); - switch (c) { - case '{': - return parseObjectNode(parser, "object node", location); - case '[': - return parseArrayNode(parser, location); - case '"': { - if (peekTextBlock(parser)) { - return parseTextBlock(parser, location); - } else { - return new StringNode(IdlTextParser.parseQuotedString(parser), location); - } - } - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - return parser.parseNumberNode(location); - default: { - return parseNodeTextWithKeywords(parser, location, ParserUtils.parseShapeId(parser)); - } + /** + * Expects that the current token is a valid Node, parses it into a {@link Node} value, and assigns it a custom + * {@link SourceLocation}. + * + *

The tokenizer is advanced to the next token after parsing the Node value.

+ * + * @param tokenizer Tokenizer to consume and advance. + * @param resolver Forward reference resolver. + * @param location Source location to assign to the node. + * @return Returns the parsed node value. + * @throws ModelSyntaxException if the Node is not well-formed. + */ + static Node expectAndSkipNode(IdlTokenizer tokenizer, IdlReferenceResolver resolver, SourceLocation location) { + IdlToken token = tokenizer.expect(IdlToken.STRING, IdlToken.TEXT_BLOCK, IdlToken.NUMBER, IdlToken.IDENTIFIER, + IdlToken.LBRACE, IdlToken.LBRACKET); + + switch (token) { + case STRING: + case TEXT_BLOCK: + Node result = new StringNode(tokenizer.getCurrentTokenStringSlice().toString(), location); + tokenizer.next(); + return result; + case IDENTIFIER: + String shapeId = tokenizer.internString(IdlShapeIdParser.expectAndSkipShapeId(tokenizer)); + return parseIdentifier(resolver, shapeId, location); + case NUMBER: + Number number = tokenizer.getCurrentTokenNumberValue(); + tokenizer.next(); + return new NumberNode(number, location); + case LBRACE: + return parseObjectNode(tokenizer, resolver, location); + case LBRACKET: + default: + return parseArrayNode(tokenizer, resolver, location); } } - static Node parseNodeTextWithKeywords(IdlModelParser parser, SourceLocation location, String text) { - switch (text) { - case "true": + /** + * Parse a Node identifier String, taking into account keywords and forward references. + * + * @param resolver Forward reference resolver. + * @param identifier Identifier to parse. + * @param location Source location to assign to the identifier. + * @return Returns the parsed identifier. + */ + static Node parseIdentifier( + IdlReferenceResolver resolver, + String identifier, + SourceLocation location + ) { + Keyword keyword = Keyword.from(identifier); + return keyword == null + ? parseSyntacticShapeId(resolver, identifier, location) + : keyword.createNode(location); + } + + private enum Keyword { + TRUE { + @Override + protected Node createNode(SourceLocation location) { return new BooleanNode(true, location); - case "false": + } + }, + FALSE { + @Override + protected Node createNode(SourceLocation location) { return new BooleanNode(false, location); - case "null": + } + }, + NULL { + @Override + protected Node createNode(SourceLocation location) { return new NullNode(location); - default: - // Unquoted node values syntactically are assumed to be references - // to shapes. A lazy string node is used because the shape ID may - // not be able to be resolved until after the entire model is loaded. - Pair> pair = StringNode.createLazyString(text, location); - Consumer consumer = pair.right; - parser.addForwardReference(text, (id, typeProvider) -> { - if (typeProvider.apply(id) == null) { - parser.emit(ValidationEvent.builder() - .id(SYNTACTIC_SHAPE_ID_TARGET) - .severity(Severity.DANGER) - .message(String.format("Syntactic shape ID `%s` does not resolve to a valid shape ID: " - + "`%s`. Did you mean to quote this string? Are you missing a " - + "model file?", text, id)) - .sourceLocation(location) - .build()); - } - consumer.accept(id.toString()); - }); - return pair.left; + } + }; + + protected abstract Node createNode(SourceLocation location); + + static Keyword from(String keyword) { + switch (keyword) { + case "true": + return Keyword.TRUE; + case "false": + return Keyword.FALSE; + case "null": + return Keyword.NULL; + default: + return null; + } } } - static boolean peekTextBlock(IdlModelParser parser) { - return parser.peek() == '"' - && parser.peek(1) == '"' - && parser.peek(2) == '"'; - } - - static Node parseTextBlock(IdlModelParser parser, SourceLocation location) { - parser.expect('"'); - parser.expect('"'); - parser.expect('"'); - return new StringNode(IdlTextParser.parseQuotedTextAndTextBlock(parser, true), location); + private static Node parseSyntacticShapeId( + IdlReferenceResolver resolver, + String identifier, + SourceLocation location + ) { + // Unquoted node values syntactically are assumed to be references to shapes. A lazy string node is + // used because the shape ID may not be able to be resolved until after the entire model is loaded. + Pair> pair = StringNode.createLazyString(identifier, location); + Consumer consumer = pair.right; + resolver.resolve(identifier, (id, type) -> { + consumer.accept(id.toString()); + if (type != null) { + return null; + } else { + return ValidationEvent.builder() + .id(SYNTACTIC_SHAPE_ID_TARGET) + .severity(Severity.DANGER) + .message(String.format("Syntactic shape ID `%s` does not resolve to a valid shape ID: " + + "`%s`. Did you mean to quote this string? Are you missing a " + + "model file?", identifier, id)) + .sourceLocation(location) + .build(); + } + }); + return pair.left; } - static ObjectNode parseObjectNode(IdlModelParser parser, String parent) { - return parseObjectNode(parser, parent, parser.currentLocation()); - } + private static ArrayNode parseArrayNode( + IdlTokenizer tokenizer, + IdlReferenceResolver resolver, + SourceLocation location + ) { + tokenizer.increaseNestingLevel(); + ArrayNode.Builder builder = ArrayNode.builder().sourceLocation(location); - static ObjectNode parseObjectNode(IdlModelParser parser, String parent, SourceLocation location) { - parser.increaseNestingLevel(); - ObjectNode.Builder builder = ObjectNode.builder() - .sourceLocation(location); - parser.expect('{'); - parser.ws(); + tokenizer.expect(IdlToken.LBRACKET); + tokenizer.next(); + tokenizer.skipWsAndDocs(); - while (!parser.eof()) { - char c = parser.peek(); - if (c == '}') { + do { + if (tokenizer.getCurrentToken() == IdlToken.RBRACKET) { break; } else { - SourceLocation keyLocation = parser.currentLocation(); - String key = parseNodeObjectKey(parser); - parser.ws(); - parser.expect(':'); - if (parser.peek() == '=') { - throw parser.syntax("The `:=` syntax may only be used when defining inline operation input and " - + "output shapes."); - } - parser.ws(); - Node value = parseNode(parser); - StringNode keyNode = new StringNode(key, keyLocation); - if (builder.hasMember(key)) { - throw parser.syntax("Duplicate member of " + parent + ": '" + keyNode.getValue() + '\''); - } - builder.withMember(keyNode, value); - parser.ws(); + builder.withValue(expectAndSkipNode(tokenizer, resolver)); + tokenizer.skipWsAndDocs(); } - } + } while (true); - parser.expect('}'); - parser.decreaseNestingLevel(); + tokenizer.expect(IdlToken.RBRACKET); + tokenizer.next(); + tokenizer.decreaseNestingLevel(); return builder.build(); } - static String parseNodeObjectKey(IdlModelParser parser) { - if (parser.peek() == '"') { - return IdlTextParser.parseQuotedString(parser); - } else { - return ParserUtils.parseIdentifier(parser); - } - } - - private static ArrayNode parseArrayNode(IdlModelParser parser, SourceLocation location) { - parser.increaseNestingLevel(); - ArrayNode.Builder builder = ArrayNode.builder() - .sourceLocation(location); - parser.expect('['); - parser.ws(); - - while (!parser.eof()) { - char c = parser.peek(); - if (c == ']') { + private static ObjectNode parseObjectNode( + IdlTokenizer tokenizer, + IdlReferenceResolver resolver, + SourceLocation location + ) { + tokenizer.expect(IdlToken.LBRACE); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + tokenizer.increaseNestingLevel(); + ObjectNode.Builder builder = ObjectNode.builder().sourceLocation(location); + + while (tokenizer.hasNext()) { + if (tokenizer.expect(IdlToken.RBRACE, IdlToken.STRING, IdlToken.IDENTIFIER) == IdlToken.RBRACE) { break; - } else { - builder.withValue(parseNode(parser)); - parser.ws(); } + + String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + SourceLocation keyLocation = tokenizer.getCurrentTokenLocation(); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + + Node value = expectAndSkipNode(tokenizer, resolver); + if (builder.hasMember(key)) { + throw new ModelSyntaxException("Duplicate member: '" + key + '\'', keyLocation); + } + builder.withMember(key, value); + tokenizer.skipWsAndDocs(); } - parser.expect(']'); - parser.decreaseNestingLevel(); + tokenizer.expect(IdlToken.RBRACE); + tokenizer.next(); + tokenizer.decreaseNestingLevel(); return builder.build(); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java new file mode 100644 index 00000000000..a9e9ade4576 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlReferenceResolver.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.ShapeType; +import software.amazon.smithy.model.validation.ValidationEvent; + +/** + * Resolves forward references for parsers, providing them resolved shape IDs and the shape type if found. + */ +@FunctionalInterface +interface IdlReferenceResolver { + /** + * Defer the resolution of a shape ID within a namespace. + * + * @param name Name of the shape to resolve. + * @param receiver Receiver that receives the resolved shape ID and type, or null if the shape was not found. + * The receiver can return a {@link ValidationEvent} if the shape could not be resolved, or it + * can return null if the shape was resolved. + */ + void resolve(String name, BiFunction receiver); + + /** + * Defer the resolution of a shape ID within a namespace. + * + * @param name Name of the shape to resolve. + * @param receiver Receiver that receives the resolved shape ID and type, or null if the shape was not found. + */ + default void resolve(String name, BiConsumer receiver) { + resolve(name, (id, type) -> { + receiver.accept(id, type); + return null; + }); + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java new file mode 100644 index 00000000000..8be8375cc96 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +/** + * Parses Shape ID lexemes from a {@link IdlTokenizer}. + */ +final class IdlShapeIdParser { + + private IdlShapeIdParser() { } + + /** + * Expects that the current token and subsequent tokens make up a Smithy namespace production (e.g., "foo.bar"). + * + *

After consuming the namespace, the tokenizer is moved to the next token. + * + * @param tokenizer Tokenizer to consume and advance. + * @return Returns the sequence of characters that make up a namespace. + * @throws ModelSyntaxException if the tokenizer is unable to parse a valid namespace. + */ + static CharSequence expectAndSkipShapeIdNamespace(IdlTokenizer tokenizer) { + int startPosition = tokenizer.getCurrentTokenStart(); + int endOffset = skipShapeIdNamespace(tokenizer); + return sliceFrom(tokenizer, startPosition, endOffset); + } + + /** + * Expects that the tokenizer is on an absolute shape ID, skips over it, and returns a borrowed slice of the + * shape ID. + * + * @param tokenizer Tokenizer to consume and advance. + * @return Returns the borrowed shape ID. + */ + static CharSequence expectAndSkipAbsoluteShapeId(IdlTokenizer tokenizer) { + int startPosition = tokenizer.getCurrentTokenStart(); + int endOffset = skipAbsoluteShapeId(tokenizer); + return sliceFrom(tokenizer, startPosition, endOffset); + } + + /** + * Expects that the tokenizer is on a relative or absolute shape ID, skips over it, and returns a borrowed slice + * of the shape ID. + * + * @param tokenizer Tokenizer to consume and advance. + * @return Returns the borrowed shape ID. + */ + static CharSequence expectAndSkipShapeId(IdlTokenizer tokenizer) { + int startPosition = tokenizer.getCurrentTokenStart(); + int offset = skipShapeIdNamespace(tokenizer); + // Keep parsing if we find a $ or a #. + if (tokenizer.getCurrentToken() != IdlToken.DOLLAR && tokenizer.getCurrentToken() != IdlToken.POUND) { + return sliceFrom(tokenizer, startPosition, offset); + } + tokenizer.next(); + offset = skipRelativeRootShapeId(tokenizer); + return sliceFrom(tokenizer, startPosition, offset); + } + + private static CharSequence sliceFrom(IdlTokenizer tokenizer, int startPosition, int endOffset) { + return tokenizer.getInput(startPosition, tokenizer.getPosition() - endOffset); + } + + private static int skipShapeIdNamespace(IdlTokenizer tokenizer) { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); + // Keep track of how many characters from the end to omit (don't include "#" or whatever is next in the slice). + int endOffset = tokenizer.getCurrentTokenSpan(); + while (tokenizer.getCurrentToken() == IdlToken.DOT) { + tokenizer.next(); + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); + endOffset = tokenizer.getCurrentTokenSpan(); + } + return endOffset; + } + + private static int skipAbsoluteShapeId(IdlTokenizer tokenizer) { + skipShapeIdNamespace(tokenizer); + tokenizer.expect(IdlToken.POUND); + tokenizer.next(); + return skipRelativeRootShapeId(tokenizer); + } + + private static int skipRelativeRootShapeId(IdlTokenizer tokenizer) { + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); + // Don't include whatever character comes next in the slice. + int endOffset = tokenizer.getCurrentTokenSpan(); + if (tokenizer.getCurrentToken() == IdlToken.DOLLAR) { + tokenizer.next(); + tokenizer.expect(IdlToken.IDENTIFIER); + tokenizer.next(); + // It had a member, so update the offset to not include. + endOffset = tokenizer.getCurrentTokenSpan(); + } + return endOffset; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlStringLexer.java similarity index 63% rename from smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java rename to smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlStringLexer.java index f667bccb100..d6128d0451b 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlStringLexer.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -15,63 +15,57 @@ package software.amazon.smithy.model.loader; -/** - * Parses IDL text and text blocks. - */ -final class IdlTextParser { - - private IdlTextParser() {} - - // quoted_text = DQUOTE *quoted_char DQUOTE - static String parseQuotedString(IdlModelParser parser) { - parser.expect('"'); - if (parser.peek() == '"') { // open and closed string. - parser.skip(); - return ""; // "" - } else { // " - return parseQuotedTextAndTextBlock(parser, false); - } - } +import java.util.ArrayList; +import java.util.List; - // Parses both quoted_text and text_block - static String parseQuotedTextAndTextBlock(IdlModelParser parser, boolean triple) { - int start = parser.position(); +final class IdlStringLexer { - while (!parser.eof()) { - char next = parser.peek(); - if (next == '"' && (!triple || (parser.peek(1) == '"' && parser.peek(2) == '"'))) { - // Found closing quotes of quoted_text and/or text_block - break; - } - parser.skip(); - if (next == '\\') { - parser.skip(); - } + private IdlStringLexer() { } + + private enum State { NORMAL, AFTER_ESCAPE, UNICODE } + + // Use the original lexeme of a string when possible, but creates a new string when escapes are used. + private static final class StringBuilderProxy { + private final CharSequence lexeme; + private StringBuilder builder; + private int position = 0; + + StringBuilderProxy(CharSequence lexeme) { + this.lexeme = lexeme; } - // Strip the ending '"'. - String result = parser.sliceFrom(start); - parser.expect('"'); + // Called when a string uses escapes, and begins buffering to a newly created string. + void capture() { + if (builder == null) { + builder = new StringBuilder(lexeme.length()); + builder.append(lexeme, 0, position); + } + } - if (triple) { - parser.expect('"'); - parser.expect('"'); + void append(char c) { + if (builder != null) { + builder.append(c); + } else { + position++; + } } - return parseStringContents(parser, result, triple); + CharSequence getResult() { + return builder == null ? lexeme : builder; + } } - enum State { NORMAL, AFTER_ESCAPE, UNICODE } - - private static String parseStringContents(IdlModelParser parser, String lexeme, boolean triple) { + static CharSequence scanStringContents(CharSequence lexeme, boolean scanningTextBlock) { lexeme = normalizeLineEndings(lexeme); // Format the text block and remove incidental whitespace. - if (triple) { - lexeme = formatTextBlock(parser, lexeme); + if (scanningTextBlock) { + lexeme = formatTextBlock(lexeme); } - StringBuilder result = new StringBuilder(lexeme.length()); + //StringBuilder result = new StringBuilder(lexeme.length()); + StringBuilderProxy result = new StringBuilderProxy(lexeme); + State state = State.NORMAL; int hexCount = 0; int unicode = 0; @@ -83,10 +77,11 @@ private static String parseStringContents(IdlModelParser parser, String lexeme, case NORMAL: if (c == '\\') { state = State.AFTER_ESCAPE; - } else if (isValidNormalCharacter(c, triple)) { + result.capture(); + } else if (isValidNormalCharacter(c, scanningTextBlock)) { result.append(c); } else { - throw parser.syntax("Invalid character: `" + c + "`"); + throw new RuntimeException("Invalid string character: `" + c + "`"); } break; case AFTER_ESCAPE: @@ -123,7 +118,7 @@ private static String parseStringContents(IdlModelParser parser, String lexeme, // Skip writing the escaped new line. break; default: - throw parser.syntax("Invalid escape found in string: `\\" + c + "`"); + throw new RuntimeException("Invalid escape found in string: `\\" + c + "`"); } break; case UNICODE: @@ -134,7 +129,7 @@ private static String parseStringContents(IdlModelParser parser, String lexeme, } else if (c >= 'A' && c <= 'F') { unicode = (unicode << 4) | (10 + c - 'A'); } else { - throw parser.syntax("Invalid unicode escape character: `" + c + "`"); + throw new RuntimeException("Invalid unicode escape character: `" + c + "`"); } if (++hexCount == 4) { @@ -149,17 +144,17 @@ private static String parseStringContents(IdlModelParser parser, String lexeme, } if (state == State.UNICODE) { - throw parser.syntax("Invalid unclosed unicode escape found in string"); + throw new RuntimeException("Invalid unclosed unicode escape found in string"); } - return result.toString(); + return result.getResult(); } // New lines in strings are normalized from CR (u000D) and CRLF (u000Du000A) to // LF (u000A). This ensures that strings defined in a Smithy model are equivalent // across platforms. If a literal \r is desired, it can be added a string value - // using the Unicode escape \\u000d. - private static String normalizeLineEndings(String lexeme) { + // using the Unicode escape (\)u000d. + private static CharSequence normalizeLineEndings(CharSequence lexeme) { if (!containsCarriageReturn(lexeme)) { return lexeme; } @@ -178,12 +173,10 @@ private static String normalizeLineEndings(String lexeme) { } } - return builder.toString(); + return builder; } - // A slightly more efficient way than String#contains to check if a string - // contains a carriage return. - private static boolean containsCarriageReturn(String lexeme) { + private static boolean containsCarriageReturn(CharSequence lexeme) { for (int i = 0; i < lexeme.length(); i++) { if (lexeme.charAt(i) == '\r') { return true; @@ -192,49 +185,53 @@ private static boolean containsCarriageReturn(String lexeme) { return false; } - /** - * Formats a text block by removing leading incidental whitespace. - * - *

Text block formatting occurs before expanding escapes. - * - * @param lexeme Lexeme to format. - * @return Returns the formatted textblock. - * @throws IllegalArgumentException if the block does not start with \n or is empty. - */ - private static String formatTextBlock(IdlModelParser parser, String lexeme) { - if (lexeme.isEmpty()) { - throw parser.syntax("Invalid text block: text block is empty"); + private static CharSequence formatTextBlock(CharSequence lexeme) { + if (lexeme.length() == 0) { + throw new RuntimeException("Text block is empty"); } else if (lexeme.charAt(0) != '\n') { - throw parser.syntax("Invalid text block: text block must start with a new line (LF)"); + throw new RuntimeException("Text block must start with a new line"); } StringBuilder buffer = new StringBuilder(); int longestPadding = Integer.MAX_VALUE; - String[] lines = lexeme.split("\n", -1); + List lines = lines(lexeme); - for (int i = 1; i < lines.length; i++) { - int padding = computeLeadingWhitespace(lines[i], i == lines.length - 1); + for (int i = 1; i < lines.size(); i++) { + int padding = computeLeadingWhitespace(lines.get(i), i == lines.size() - 1); if (padding > -1 && padding < longestPadding) { longestPadding = padding; } } - for (int i = 1; i < lines.length; i++) { - String line = lines[i]; + for (int i = 1; i < lines.size(); i++) { + CharSequence line = lines.get(i); - if (!line.isEmpty()) { - String formattedLine = createTextBlockLine(line, longestPadding); + if (line.length() > 0) { + CharSequence formattedLine = createTextBlockLine(line, longestPadding); if (formattedLine != null) { buffer.append(formattedLine); } } - if (i < lines.length - 1) { + if (i < lines.size() - 1) { buffer.append('\n'); } } - return buffer.toString(); + return buffer; + } + + private static List lines(final CharSequence text) { + List lines = new ArrayList<>(); + int mark = 0; + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '\n') { + lines.add(text.subSequence(mark, i)); + mark = i + 1; + } + } + lines.add(text.subSequence(mark, text.length())); + return lines; } /** @@ -251,8 +248,8 @@ private static String formatTextBlock(IdlModelParser parser, String lexeme) { * @param isLastLine Whether or not this is the last line. * @return Returns the last whitespace index starting at 0 or -1. */ - private static int computeLeadingWhitespace(String line, boolean isLastLine) { - if (line.isEmpty()) { + private static int computeLeadingWhitespace(CharSequence line, boolean isLastLine) { + if (line.length() == 0) { return -1; } @@ -277,7 +274,7 @@ private static int computeLeadingWhitespace(String line, boolean isLastLine) { * @param longestPadding The leading whitespace to remove. * @return Returns the line or null if no line is emitted. */ - private static String createTextBlockLine(String line, int longestPadding) { + private static CharSequence createTextBlockLine(CharSequence line, int longestPadding) { int startPosition = Math.min(longestPadding, line.length()); int endPosition = line.length() - 1; @@ -286,7 +283,7 @@ private static String createTextBlockLine(String line, int longestPadding) { endPosition--; } - return endPosition >= startPosition ? line.substring(startPosition, endPosition + 1) : null; + return endPosition >= startPosition ? line.subSequence(startPosition, endPosition + 1) : null; } private static boolean isValidNormalCharacter(char c, boolean isTextBlock) { @@ -294,11 +291,11 @@ private static boolean isValidNormalCharacter(char c, boolean isTextBlock) { // QuotedChar grammar: // https://smithy.io/2.0/spec/idl.html#grammar-token-smithy-QuotedChar return c == '\t' - || c == '\n' - || c == '\r' - || (c >= 0x20 && c <= 0x21) // space - "!" - || (isTextBlock && c == 0x22) // DQUOTE is allowed in text_block - || (c >= 0x23 && c <= 0x5b) // "#" - "[" - || c >= 0x5d; // "]"+ + || c == '\n' + || c == '\r' + || (c >= 0x20 && c <= 0x21) // space - "!" + || (isTextBlock && c == 0x22) // DQUOTE is allowed in text_block + || (c >= 0x23 && c <= 0x5b) // "#" - "[" + || c >= 0x5d; // "]"+ } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java new file mode 100644 index 00000000000..1ff479ea523 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlToken.java @@ -0,0 +1,73 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Represents a parsed token from the Smithy IDL. + */ +@SmithyUnstableApi +public enum IdlToken { + + SPACE(" ", true), + NEWLINE("\\n", true), + COMMA(",", true), + COMMENT("//", true), + DOC_COMMENT("///", false), + AT("@", false), + STRING("\"", false), + TEXT_BLOCK("\"\"\"", false), + COLON(":", false), + WALRUS(":=", false), + IDENTIFIER("", false), + DOT(".", false), + POUND("#", false), + DOLLAR("$", false), + NUMBER("", false), + LBRACE("{", false), + RBRACE("}", false), + LBRACKET("[", false), + RBRACKET("]", false), + LPAREN("(", false), + RPAREN(")", false), + EQUAL("=", false), + EOF("", false), + ERROR("", false); + + private final String exampleLexeme; + private final boolean isWhitespace; + + IdlToken(String exampleLexeme, boolean isWhitespace) { + this.exampleLexeme = exampleLexeme; + this.isWhitespace = isWhitespace; + } + + public String getDebug() { + return getDebug(exampleLexeme); + } + + public String getDebug(CharSequence lexeme) { + if (lexeme.length() > 0) { + return this + "('" + lexeme + "')"; + } + return toString(); + } + + public boolean isWhitespace() { + return isWhitespace; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java new file mode 100644 index 00000000000..f3a5d61c942 --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTokenizer.java @@ -0,0 +1,825 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.CharBuffer; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Function; +import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.utils.SimpleParser; +import software.amazon.smithy.utils.SmithyBuilder; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Iterates over a Smithy IDL model as a series of tokens. + */ +@SmithyUnstableApi +public final class IdlTokenizer implements Iterator { + + /** Only allow nesting up to 64 arrays/objects in node values. */ + private static final int MAX_NESTING_LEVEL = 64; + + private final SimpleParser parser; + private final String filename; + private final Function stringTable; + + private IdlToken currentTokenType; + private int currentTokenStart = -1; + private int currentTokenEnd = -1; + private int currentTokenLine = -1; + private int currentTokenColumn = -1; + private Number currentTokenNumber; + private CharSequence currentTokenStringSlice; + private String currentTokenError; + private final Deque docCommentLines = new ArrayDeque<>(); + + private IdlTokenizer(Builder builder) { + this.filename = builder.filename; + this.stringTable = builder.stringTable; + this.parser = new SimpleParser(SmithyBuilder.requiredState("model", builder.model), MAX_NESTING_LEVEL); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder implements SmithyBuilder { + private String filename = SourceLocation.NONE.getFilename(); + private CharSequence model; + private Function stringTable = CharSequence::toString; + + private Builder() { } + + /** + * The filename to associate with each token and errors. + * + *

Defaults to the filename of {@link SourceLocation#NONE}. + * + * @param filename Filename where the model comes from. + * @return Returns builder. + */ + public Builder filename(String filename) { + this.filename = Objects.requireNonNull(filename); + return this; + } + + /** + * Sets the required model IDL content to parse. + * + * @param model Model IDL content. + * @return Returns the builder. + */ + public Builder model(CharSequence model) { + this.model = model; + return this; + } + + /** + * Allows the use of a custom string table, used to convert {@link CharSequence}s into Strings. + * + *

Uses {@link CharSequence#toString()} by default. A string table cache is used when the tokenizer is + * used by {@link ModelAssembler}. + * + * @param stringTable String table to use. + * @return Returns the builder. + */ + public Builder stringTable(Function stringTable) { + this.stringTable = Objects.requireNonNull(stringTable); + return this; + } + + @Override + public IdlTokenizer build() { + return new IdlTokenizer(this); + } + } + + /** + * Get a borrowed slice of the input being parsed. + * + * @param start Start position to get, inclusive. + * @param end End position to stop at, exclusive. + * @return Returns the slice. + */ + CharSequence getInput(int start, int end) { + return CharBuffer.wrap(parser.input(), start, end); + } + + /** + * Intern a string and cache the interned value for subsequent retrieval. + * + *

This method should only be used with strings that are frequently used, for example, object keys, shape + * properties, namespaces, keywords (e.g., true, false, null, namespace, use, string, structure, trait IDs, etc). + * + * @param sequence Characters to convert to a string. + * @return Returns the String representation of {@code sequence}. + */ + public String internString(CharSequence sequence) { + return stringTable.apply(sequence); + } + + /** + * Increase the nesting level of the tokenizer. + */ + public void increaseNestingLevel() { + try { + parser.increaseNestingLevel(); + } catch (RuntimeException e) { + throw syntax("Parser exceeded maximum allowed depth of " + MAX_NESTING_LEVEL, getCurrentTokenLocation()); + } + } + + /** + * Decrease the nesting level of the tokenizer. + */ + public void decreaseNestingLevel() { + parser.decreaseNestingLevel(); + } + + /** + * Get the current position of the tokenizer. + * + * @return Returns the absolute position. + */ + public int getPosition() { + return parser.position(); + } + + /** + * Get the current line number of the tokenizer, starting at 1. + * + * @return Get the current line number. + */ + public int getLine() { + return parser.line(); + } + + /** + * Get the current column number of the tokenizer, starting at 1. + * + * @return Get the current column number. + */ + public int getColumn() { + return parser.column(); + } + + /** + * Get the current {@link IdlToken}. + * + * @return Return the current token type. + */ + public IdlToken getCurrentToken() { + if (currentTokenType == null) { + next(); + } + return currentTokenType; + } + + /** + * Get the line of the current token. + * + * @return Return the line of the current token. + */ + public int getCurrentTokenLine() { + getCurrentToken(); + return currentTokenLine; + } + + /** + * Get the column of the current token. + * + * @return Return the column of the current token. + */ + public int getCurrentTokenColumn() { + getCurrentToken(); + return currentTokenColumn; + } + + /** + * Get the start position of the current token. + * + * @return Return the start position of the current token. + */ + public int getCurrentTokenStart() { + getCurrentToken(); + return currentTokenStart; + } + + /** + * Get the span, or length, of the current token. + * + * @return Return the current token span. + */ + public int getCurrentTokenSpan() { + getCurrentToken(); + return currentTokenEnd - currentTokenStart; + } + + /** + * Get the source location of the current token. + * + * @return Return the current token source location. + */ + public SourceLocation getCurrentTokenLocation() { + getCurrentToken(); + return new SourceLocation(filename, currentTokenLine, currentTokenColumn); + } + + /** + * Get the lexeme of the current token. + * + * @return Returns the lexeme of the current token. + */ + public CharSequence getCurrentTokenLexeme() { + getCurrentToken(); + return CharBuffer.wrap(parser.input(), currentTokenStart, currentTokenEnd); + } + + /** + * If the current token is a string or text block, get the parsed content as a CharSequence. + * If the current token is an identifier, the lexeme of the identifier is returned. + * + * @return Returns the parsed string content associated with the current token. + * @throws ModelSyntaxException if the current token is not a string, text block, or identifier. + */ + public CharSequence getCurrentTokenStringSlice() { + getCurrentToken(); + if (currentTokenStringSlice != null) { + return currentTokenStringSlice; + } else if (currentTokenType == IdlToken.IDENTIFIER) { + return getCurrentTokenLexeme(); + } else { + throw syntax("The current token must be string or identifier but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + } + + /** + * If the current token is a number, get the associated parsed number. + * + * @return Returns the parsed number associated with the current token. + * @throws ModelSyntaxException if the current token is not a number. + */ + public Number getCurrentTokenNumberValue() { + getCurrentToken(); + if (currentTokenNumber == null) { + throw syntax("The current token must be number but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + return currentTokenNumber; + } + + /** + * If the current token is an error, get the error message associated with the token. + * + * @return Returns the associated error message. + * @throws ModelSyntaxException if the current token is not an error. + */ + public String getCurrentTokenError() { + getCurrentToken(); + if (currentTokenType != IdlToken.ERROR) { + throw syntax("The current token must be an error but found: " + + currentTokenType.getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + return currentTokenError == null ? "" : currentTokenError; + } + + @Override + public boolean hasNext() { + return currentTokenType != IdlToken.EOF; + } + + @Override + public IdlToken next() { + currentTokenStringSlice = null; + currentTokenNumber = null; + currentTokenColumn = parser.column(); + currentTokenLine = parser.line(); + currentTokenStart = parser.position(); + currentTokenEnd = currentTokenStart; + int c = parser.peek(); + + switch (c) { + case SimpleParser.EOF: + if (currentTokenType == IdlToken.EOF) { + throw new NoSuchElementException("Expected another token but traversed beyond EOF"); + } + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.EOF; + case ' ': + case '\t': + return tokenizeSpace(); + case '\r': + case '\n': + return tokenizeNewline(); + case ',': + return singleCharToken(IdlToken.COMMA); + case '@': + return singleCharToken(IdlToken.AT); + case '$': + return singleCharToken(IdlToken.DOLLAR); + case '.': + return singleCharToken(IdlToken.DOT); + case '{': + return singleCharToken(IdlToken.LBRACE); + case '}': + return singleCharToken(IdlToken.RBRACE); + case '[': + return singleCharToken(IdlToken.LBRACKET); + case ']': + return singleCharToken(IdlToken.RBRACKET); + case '(': + return singleCharToken(IdlToken.LPAREN); + case ')': + return singleCharToken(IdlToken.RPAREN); + case '#': + return singleCharToken(IdlToken.POUND); + case '=': + return singleCharToken(IdlToken.EQUAL); + case ':': + return parseColon(); + case '"': + return parseString(); + case '/': + return parseComment(); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return parseNumber(); + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + return parseIdentifier(); + default: + currentTokenError = "Unexpected character: '" + ((char) c) + '\''; + return singleCharToken(IdlToken.ERROR); + } + } + + /** + * Skip spaces. + */ + public void skipSpaces() { + getCurrentToken(); + while (currentTokenType == IdlToken.SPACE) { + next(); + } + } + + /** + * Skip until the current token is not {@link IdlToken#SPACE}, {@link IdlToken#COMMA}, {@link IdlToken#COMMENT}, + * or {@link IdlToken#NEWLINE}. + */ + public void skipWs() { + getCurrentToken(); + while (currentTokenType.isWhitespace()) { + next(); + } + } + + /** + * Skip until the current token is not {@link IdlToken#SPACE}, {@link IdlToken#COMMA}, {@link IdlToken#COMMENT}, + * or {@link IdlToken#NEWLINE}, or {@link IdlToken#DOC_COMMENT}. + */ + public void skipWsAndDocs() { + getCurrentToken(); + while ((currentTokenType.isWhitespace() || currentTokenType == IdlToken.DOC_COMMENT)) { + next(); + } + } + + /** + * Check if the current token is an identifier, and if its lexeme starts with the given character {@code c}. + * + *

This method does not throw if the current token is not an identifier. + * + * @return Returns true if the token is an identifier that starts with {@code c}. + */ + public boolean doesCurrentIdentifierStartWith(char c) { + getCurrentToken(); + return currentTokenType == IdlToken.IDENTIFIER + && currentTokenEnd > currentTokenStart + && parser.input().charAt(currentTokenStart) == c; + } + + /** + * Expects that the lexeme of the current token is the same as the given characters. + * + * @param chars Characters to compare the current lexeme against. + * @return Returns the current lexeme. + * @throws ModelSyntaxException if the current lexeme is not equal to the given characters. + */ + public CharSequence expectCurrentLexeme(CharSequence chars) { + CharSequence lexeme = getCurrentTokenLexeme(); + boolean isError = lexeme.length() != chars.length(); + if (!isError) { + for (int i = 0; i < chars.length(); i++) { + if (lexeme.charAt(i) != chars.charAt(i)) { + isError = true; + break; + } + } + } + if (isError) { + throw syntax("Expected `" + chars + "`, but found `" + lexeme + "`", getCurrentTokenLocation()); + } + return lexeme; + } + + /** + * Assert that the current token is {@code token}. + * + *

The tokenizer is not advanced after validating the current token.

+ * + * @param token Token to expect. + * @throws ModelSyntaxException if the current token is unexpected. + */ + public void expect(IdlToken token) { + getCurrentToken(); + + if (currentTokenType != token) { + throw syntax(createExpectMessage(token), getCurrentTokenLocation()); + } + } + + /** + * Assert that the current token is one of {@code tokens}. + * + *

The tokenizer is not advanced after validating the current token.

+ * + * @param tokens Assert that the current token is one of these tokens. + * @return Returns the current token. + * @throws ModelSyntaxException if the current token is unexpected. + */ + public IdlToken expect(IdlToken... tokens) { + getCurrentToken(); + + if (currentTokenType == IdlToken.ERROR) { + throw syntax(createExpectMessage(tokens), getCurrentTokenLocation()); + } + + for (IdlToken token : tokens) { + if (currentTokenType == token) { + return token; + } + } + + throw syntax(createExpectMessage(tokens), getCurrentTokenLocation()); + } + + private String createExpectMessage(IdlToken... tokens) { + StringBuilder result = new StringBuilder(); + if (currentTokenType == IdlToken.ERROR) { + result.append(getCurrentTokenError()); + } else if (tokens.length == 1) { + result.append("Expected ") + .append(tokens[0].getDebug()) + .append(" but found ") + .append(getCurrentToken().getDebug(getCurrentTokenLexeme())); + } else { + result.append("Expected one of "); + for (IdlToken token : tokens) { + result.append(token.getDebug()).append(", "); + } + result.delete(result.length() - 2, result.length()); + result.append("; but found ").append(getCurrentToken().getDebug(getCurrentTokenLexeme())); + } + return result.toString(); + } + + /** + * Expect that one or more spaces are found at the current token, and skip over them. + * + * @throws ModelSyntaxException if the current token is not a space. + */ + public void expectAndSkipSpaces() { + if (getCurrentToken() != IdlToken.SPACE) { + throw syntax("Expected one or more spaces, but found " + + getCurrentToken().getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + skipSpaces(); + } + + private ModelSyntaxException syntax(String message, SourceLocation location) { + return new ModelSyntaxException("Syntax error at line " + location.getLine() + ", column " + + location.getColumn() + ": " + message, location); + } + + /** + * Expect that one or more whitespace characters or documentation comments are found at the current token, and + * skip over them. + * + * @throws ModelSyntaxException if the current token is not whitespace. + */ + public void expectAndSkipWhitespace() { + if (!getCurrentToken().isWhitespace()) { + throw syntax("Expected one or more whitespace characters, but found " + + getCurrentToken().getDebug(getCurrentTokenLexeme()), getCurrentTokenLocation()); + } + skipWsAndDocs(); + } + + /** + * Expects that the current token is a newline or is whitespaces or comments followed by a newline. + * + *

This method mimics the {@code br} production from the Smithy grammar. When this method is called, any + * previously parsed and buffered documentation comments are discarded. + * + * @throws ModelSyntaxException if the current token is not a newline or followed by a newline. + */ + public void expectAndSkipBr() { + if (getCurrentToken() == IdlToken.NEWLINE) { + clearDocCommentLinesForBr(); + next(); + skipWs(); + } else { + int line = getLine(); + clearDocCommentLinesForBr(); + skipWs(); + if (line == getLine() && hasNext()) { + throw syntax("Expected a line break, but found " + + getCurrentToken().getDebug(getCurrentTokenLexeme()), + getCurrentTokenLocation()); + } + } + } + + private void clearDocCommentLinesForBr() { + // No known cases should encounter this code path unless we add error correction to the parser. + if (!docCommentLines.isEmpty()) { + docCommentLines.clear(); + } + } + + private IdlToken singleCharToken(IdlToken type) { + parser.skip(); + currentTokenEnd = parser.position(); + return currentTokenType = type; + } + + private IdlToken tokenizeNewline() { + parser.skip(); // this will \n and \r\n. + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.NEWLINE; + } + + private IdlToken tokenizeSpace() { + parser.consumeUntilNoLongerMatches(c -> c == ' ' || c == '\t'); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.SPACE; + } + + private IdlToken parseColon() { + parser.skip(); + + if (parser.peek() == '=') { + parser.skip(); + currentTokenType = IdlToken.WALRUS; + } else { + currentTokenType = IdlToken.COLON; + } + + currentTokenEnd = parser.position(); + return currentTokenType; + } + + private IdlToken parseComment() { + // first "/". + parser.expect('/'); + + // A standalone forward slash is an error. + if (parser.peek() != '/') { + currentTokenError = "Expected a '/' to follow '/' to form a comment."; + return singleCharToken(IdlToken.ERROR); + } + + // Skip the next "/". + parser.expect('/'); + + IdlToken type = IdlToken.COMMENT; + + // Three "///" is a documentation comment. + if (parser.peek() == '/') { + parser.expect('/'); + type = IdlToken.DOC_COMMENT; + // When capturing the doc comment lexeme, skip a single leading space if found. + if (parser.peek() == ' ') { + parser.skip(); + } + int lineStart = parser.position(); + parser.consumeRemainingCharactersOnLine(); + docCommentLines.add(getInput(lineStart, parser.position())); + } else { + parser.consumeRemainingCharactersOnLine(); + } + + // Include the newline in the comment and doc comment lexeme. + if (parser.expect('\r', '\n', SimpleParser.EOF) == '\r' && parser.peek() == '\n') { + parser.skip(); + } + + currentTokenEnd = parser.position(); + return currentTokenType = type; + } + + /** + * Removes any buffered documentation comment lines, and returns a concatenated string value. + * + * @return Returns the combined documentation comment string for the given lines. Returns null if no lines. + */ + String removePendingDocCommentLines() { + if (docCommentLines.isEmpty()) { + return null; + } else { + StringBuilder result = new StringBuilder(); + result.append(docCommentLines.removeFirst()); + while (!docCommentLines.isEmpty()) { + result.append('\n').append(docCommentLines.removeFirst()); + } + return result.toString(); + } + } + + private IdlToken parseNumber() { + try { + String lexeme = ParserUtils.parseNumber(parser); + if (lexeme.contains("e") || lexeme.contains("E") || lexeme.contains(".")) { + double value = Double.parseDouble(lexeme); + if (Double.isFinite(value)) { + currentTokenNumber = value; + } else { + currentTokenNumber = new BigDecimal(lexeme); + } + } else { + try { + currentTokenNumber = Long.parseLong(lexeme); + } catch (NumberFormatException e) { + currentTokenNumber = new BigInteger(lexeme); + } + } + + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.NUMBER; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + // Strip off the leading error message information if present. + if (e.getMessage().startsWith("Syntax error")) { + currentTokenError = e.getMessage().substring(e.getMessage().indexOf(':') + 1).trim(); + } else { + currentTokenError = e.getMessage(); + } + return currentTokenType = IdlToken.ERROR; + } + } + + private IdlToken parseIdentifier() { + ParserUtils.consumeIdentifier(parser); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.IDENTIFIER; + } + + private IdlToken parseString() { + parser.skip(); // skip first quote. + + if (parser.peek() == '"') { + parser.skip(); // skip second quote. + if (parser.peek() == '"') { // A third consecutive quote is a TEXT_BLOCK. + parser.skip(); + return parseTextBlock(); + } else { + // Empty string. + currentTokenEnd = parser.position(); + currentTokenStringSlice = ""; + return currentTokenType = IdlToken.STRING; + } + } + + try { + // Parse the contents of a quoted string. + currentTokenStringSlice = parseQuotedTextAndTextBlock(false); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.STRING; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + currentTokenError = "Error parsing quoted string: " + e.getMessage(); + return currentTokenType = IdlToken.ERROR; + } + } + + private IdlToken parseTextBlock() { + try { + currentTokenStringSlice = parseQuotedTextAndTextBlock(true); + currentTokenEnd = parser.position(); + return currentTokenType = IdlToken.TEXT_BLOCK; + } catch (RuntimeException e) { + currentTokenEnd = parser.position(); + currentTokenError = "Error parsing text block: " + e.getMessage(); + return currentTokenType = IdlToken.ERROR; + } + } + + // Parses both quoted_text and text_block + private CharSequence parseQuotedTextAndTextBlock(boolean triple) { + int start = parser.position(); + + while (!parser.eof()) { + char next = parser.peek(); + if (next == '"' && (!triple || (parser.peek(1) == '"' && parser.peek(2) == '"'))) { + // Found closing quotes of quoted_text and/or text_block + break; + } + parser.skip(); + if (next == '\\') { + parser.skip(); + } + } + + // Strip the ending '"'. + CharSequence result = parser.borrowSliceFrom(start); + parser.expect('"'); + + if (triple) { + parser.expect('"'); + parser.expect('"'); + } + + return IdlStringLexer.scanStringContents(result, triple); + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java index 04f8a089a03..88fcda647cc 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -22,134 +22,225 @@ import software.amazon.smithy.model.SourceLocation; import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.node.NullNode; +import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.traits.Trait; final class IdlTraitParser { - private IdlTraitParser() {} + // A pending trait that also doesn't yet have a resolved trait shape ID. + static final class Result { + private final CharSequence traitName; + private final Node value; + private final TraitType traitType; - // trait_statements = *(ws trait) - static List parseTraits(IdlModelParser parser) { - List entries = new ArrayList<>(); - while (parser.peek() == '@') { - entries.add(parseTraitValue(parser)); - parser.ws(); + Result(CharSequence traitName, Node value, TraitType traitType) { + this.traitName = traitName; + this.value = value; + this.traitType = traitType; } - return entries; + CharSequence getTraitName() { + return traitName; + } + + Node getValue() { + return value; + } + + TraitType getTraitType() { + return traitType; + } + } + + enum TraitType { + VALUE, ANNOTATION, DOC_COMMENT + } + + private IdlTraitParser() { } + + /** + * Assumes that the tokenizer is potentially before a shape or member definition, and parses out documentation + * comments and applied traits into a list of {@link Result} values that can later be turned into instances of + * {@link Trait}s to apply to shapes. + * + * @param tokenizer Tokenizer to consume and advance. + * @param resolver Forward reference resolver. + * @return Return the parsed traits. + */ + static List parseDocsAndTraitsBeforeShape(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + tokenizer.skipWs(); + + Result docComment = null; + + // Mark the position of where documentation comments start if on a doc comment. + if (tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT) { + SourceLocation documentationLocation = tokenizer.getCurrentTokenLocation(); + tokenizer.skipWsAndDocs(); + docComment = parseDocComment(tokenizer, documentationLocation); + } else { + tokenizer.skipWsAndDocs(); + } + + // Parse traits, if any. + tokenizer.skipWsAndDocs(); + List traits = expectAndSkipTraits(tokenizer, resolver); + if (docComment != null) { + traits.add(docComment); + } + tokenizer.skipWsAndDocs(); + + return traits; + } + + private static Result parseDocComment(IdlTokenizer tokenizer, SourceLocation location) { + String result = tokenizer.removePendingDocCommentLines(); + if (result == null) { + return null; + } else { + Node value = new StringNode(result, location); + return new Result(DocumentationTrait.ID.toString(), value, TraitType.DOC_COMMENT); + } + } + + /** + * Parse all traits before a shape or member, or inside an {@code apply} block. + * + * @param tokenizer Tokenizer to consume and advance. + * @param resolver Forward reference resolver. + * @return Returns the parsed traits. + */ + static List expectAndSkipTraits(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { + List results = new ArrayList<>(); + while (tokenizer.getCurrentToken() == IdlToken.AT) { + results.add(expectAndSkipTrait(tokenizer, resolver)); + tokenizer.skipWsAndDocs(); + } + return results; } - static IdlModelParser.TraitEntry parseTraitValue(IdlModelParser parser) { + /** + * Parses a single trait: "@" trait-id [(trait-body)]. + * + * @param tokenizer Tokenizer to consume and advance. + * @param resolver Forward reference resolver. + * @return Returns the parsed trait. + */ + static Result expectAndSkipTrait(IdlTokenizer tokenizer, IdlReferenceResolver resolver) { // "@" shape_id - SourceLocation location = parser.currentLocation(); - parser.expect('@'); - String id = ParserUtils.parseShapeId(parser); + SourceLocation location = tokenizer.getCurrentTokenLocation(); + tokenizer.expect(IdlToken.AT); + tokenizer.next(); + CharSequence id = IdlShapeIdParser.expectAndSkipShapeId(tokenizer); // No (): it's an annotation trait. - if (parser.peek() != '(') { - return new IdlModelParser.TraitEntry(id, new NullNode(location), true); + if (tokenizer.getCurrentToken() != IdlToken.LPAREN) { + return new Result(id, new NullNode(location), TraitType.ANNOTATION); } - parser.expect('('); - parser.ws(); + tokenizer.next(); + tokenizer.skipWsAndDocs(); // (): it's also an annotation trait. - if (parser.peek() == ')') { - parser.expect(')'); - return new IdlModelParser.TraitEntry(id, new NullNode(location), true); + if (tokenizer.getCurrentToken() == IdlToken.RPAREN) { + tokenizer.next(); + return new Result(id, new NullNode(location), TraitType.ANNOTATION); } // The trait has a value between the '(' and ')'. - Node value = parseTraitValueBody(parser, location); - parser.ws(); - parser.expect(')'); + Node value = parseTraitValueBody(tokenizer, resolver, location); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.RPAREN); + tokenizer.next(); - return new IdlModelParser.TraitEntry(id, value, false); + return new Result(id, value, TraitType.VALUE); } - private static Node parseTraitValueBody(IdlModelParser parser, SourceLocation location) { - char c = parser.peek(); - switch (c) { - case '{': - case '[': - // {} and [] are always node values. - return IdlNodeParser.parseNode(parser, location); - case '"': { - // Text blocks are always node values. - if (IdlNodeParser.peekTextBlock(parser)) { - return IdlNodeParser.parseTextBlock(parser, location); + private static Node parseTraitValueBody( + IdlTokenizer tokenizer, + IdlReferenceResolver resolver, + SourceLocation location + ) { + tokenizer.expect(IdlToken.LBRACE, IdlToken.LBRACKET, IdlToken.TEXT_BLOCK, IdlToken.STRING, + IdlToken.NUMBER, IdlToken.IDENTIFIER); + + switch (tokenizer.getCurrentToken()) { + case LBRACE: + case LBRACKET: + Node result = IdlNodeParser.expectAndSkipNode(tokenizer, resolver, location); + tokenizer.skipWsAndDocs(); + return result; + case TEXT_BLOCK: + Node textBlockResult = new StringNode(tokenizer.getCurrentTokenStringSlice().toString(), location); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + return textBlockResult; + case NUMBER: + Number number = tokenizer.getCurrentTokenNumberValue(); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + return new NumberNode(number, location); + case STRING: + String stringValue = tokenizer.getCurrentTokenStringSlice().toString(); + StringNode stringNode = new StringNode(stringValue, location); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + if (tokenizer.getCurrentToken() == IdlToken.COLON) { + tokenizer.next(); + tokenizer.skipWsAndDocs(); + return parseStructuredTrait(tokenizer, resolver, stringNode); + } else { + return stringNode; } - // It's a quoted string, so check if it's a KVP key or a node_value. - String key = IdlTextParser.parseQuotedString(parser); - return parseTraitValueBodyIdentifierOrQuotedString(parser, location, key, false); - } default: { - // Parser numbers. - if (c == '-' || ParserUtils.isDigit(c)) { - return parser.parseNumberNode(location); + case IDENTIFIER: + default: + String identifier = tokenizer.internString(tokenizer.getCurrentTokenLexeme()); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + if (tokenizer.getCurrentToken() == IdlToken.COLON) { + tokenizer.next(); + tokenizer.skipWsAndDocs(); + return parseStructuredTrait(tokenizer, resolver, new StringNode(identifier, location)); } else { - // Parse unquoted strings or possibly a structured trait. - String key = ParserUtils.parseIdentifier(parser); - return parseTraitValueBodyIdentifierOrQuotedString(parser, location, key, true); + return IdlNodeParser.parseIdentifier(resolver, identifier, location); } - } } } - private static Node parseTraitValueBodyIdentifierOrQuotedString( - IdlModelParser parser, - SourceLocation location, - String key, - boolean unquoted + private static ObjectNode parseStructuredTrait( + IdlTokenizer tokenizer, + IdlReferenceResolver resolver, + StringNode firstKey ) { - parser.ws(); - - // If the next character is ':', this it's a KVP. - if (parser.peek() == ':') { - parser.expect(':'); - parser.ws(); - return parseStructuredTrait(parser, new StringNode(key, location)); - } else if (unquoted) { - // It's a node_value that's either a keyword or shape ID. - return IdlNodeParser.parseNodeTextWithKeywords(parser, location, key); - } else { - // It's a quoted string node_value. - return new StringNode(key, location); - } - } - - private static ObjectNode parseStructuredTrait(IdlModelParser parser, StringNode startingKey) { + tokenizer.increaseNestingLevel(); Map entries = new LinkedHashMap<>(); - Node firstValue = IdlNodeParser.parseNode(parser); + Node firstValue = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + // This put call can be done safely without checking for duplicates, // as it's always the first member of the trait. - entries.put(startingKey, firstValue); - parser.ws(); - - while (!parser.eof() && parser.peek() != ')') { - char c = parser.peek(); - if (ParserUtils.isIdentifierStart(c) || c == '"') { - parseTraitStructureKvp(parser, entries); - } else { - throw parser.syntax("Unexpected object key character: '" + c + '\''); + entries.put(firstKey, firstValue); + tokenizer.skipWsAndDocs(); + + while (tokenizer.getCurrentToken() != IdlToken.RPAREN) { + tokenizer.expect(IdlToken.IDENTIFIER, IdlToken.STRING); + String key = tokenizer.internString(tokenizer.getCurrentTokenStringSlice()); + StringNode keyNode = new StringNode(key, tokenizer.getCurrentTokenLocation()); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + tokenizer.expect(IdlToken.COLON); + tokenizer.next(); + tokenizer.skipWsAndDocs(); + Node nextValue = IdlNodeParser.expectAndSkipNode(tokenizer, resolver); + Node previous = entries.put(keyNode, nextValue); + if (previous != null) { + throw new ModelSyntaxException("Duplicate member of trait: '" + keyNode.getValue() + '\'', keyNode); } + tokenizer.skipWsAndDocs(); } - return new ObjectNode(entries, startingKey.getSourceLocation()); - } - - private static void parseTraitStructureKvp(IdlModelParser parser, Map entries) { - SourceLocation keyLocation = parser.currentLocation(); - String key = IdlNodeParser.parseNodeObjectKey(parser); - StringNode nextKey = new StringNode(key, keyLocation); - parser.ws(); - parser.expect(':'); - parser.ws(); - Node nextValue = IdlNodeParser.parseNode(parser); - parser.ws(); - Node previous = entries.put(nextKey, nextValue); - if (previous != null) { - throw parser.syntax("Duplicate member of trait: '" + nextKey.getValue() + '\''); - } + tokenizer.decreaseNestingLevel(); + return new ObjectNode(entries, firstKey.getSourceLocation()); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperation.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperation.java index d712dd00578..9f0086b2814 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperation.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperation.java @@ -22,8 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Function; +import java.util.function.BiFunction; import software.amazon.smithy.model.FromSourceLocation; import software.amazon.smithy.model.SourceLocation; import software.amazon.smithy.model.node.Node; @@ -198,13 +197,13 @@ void addModifier(ShapeModifier modifier) { static final class ForwardReference extends LoadOperation { final String namespace; final String name; - private final BiConsumer> consumer; + private final BiFunction receiver; - ForwardReference(String namespace, String name, BiConsumer> consumer) { + ForwardReference(String namespace, String name, BiFunction receiver) { super(Version.UNKNOWN); this.namespace = namespace; this.name = name; - this.consumer = consumer; + this.receiver = receiver; } @Override @@ -212,8 +211,8 @@ void accept(Visitor visitor) { visitor.forwardReference(this); } - void resolve(ShapeId id, Function typeProvider) { - consumer.accept(id, typeProvider); + ValidationEvent resolve(ShapeId id, ShapeType type) { + return receiver.apply(id, type); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperationProcessor.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperationProcessor.java index 00a938982c0..4a01bcacc5f 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperationProcessor.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/LoadOperationProcessor.java @@ -151,27 +151,34 @@ private void resolveForwardReferences() { if (reference.namespace == null) { // Assume smithy.api if there is no namespace. This can happen in metadata and control sections. ShapeId absolute = ShapeId.fromOptionalNamespace(Prelude.NAMESPACE, reference.name); - reference.resolve(absolute, shapeMap::getShapeType); + resolveReference(reference, absolute, shapeMap.getShapeType(absolute)); } else { detectAndEmitForwardReference(reference); } } } + private void resolveReference(LoadOperation.ForwardReference reference, ShapeId id, ShapeType type) { + ValidationEvent event = reference.resolve(id, type); + if (event != null) { + events.add(event); + } + } + private void detectAndEmitForwardReference(LoadOperation.ForwardReference reference) { Objects.requireNonNull(reference.namespace); ShapeId inNamespace = ShapeId.fromOptionalNamespace(reference.namespace, reference.name); ShapeType inNamespaceType = shapeMap.getShapeType(inNamespace); if (inNamespaceType != null) { - reference.resolve(inNamespace, test -> inNamespaceType); + resolveReference(reference, inNamespace, inNamespaceType); } else { // Try to find a prelude shape by ID if no ID exists in the namespace with this name. ShapeId preludeId = ShapeId.fromOptionalNamespace(Prelude.NAMESPACE, reference.name); if (prelude != null && prelude.getShapeIds().contains(preludeId)) { - reference.resolve(preludeId, test -> prelude.expectShape(test).getType()); + resolveReference(reference, preludeId, prelude.expectShape(preludeId).getType()); } else { - reference.resolve(inNamespace, test -> null); + resolveReference(reference, inNamespace, null); } } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java index 54f1d18b490..53c2abab1bc 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelAssembler.java @@ -104,6 +104,7 @@ public final class ModelAssembler { private final Map properties = new HashMap<>(); private boolean disablePrelude; private Consumer validationEventListener = DEFAULT_EVENT_LISTENER; + private StringTable stringTable; // Lazy initialization holder class idiom to hold a default trait factory. static final class LazyTraitFactoryHolder { @@ -130,6 +131,7 @@ public ModelAssembler copy() { assembler.properties.putAll(properties); assembler.disableValidation = disableValidation; assembler.validationEventListener = validationEventListener; + assembler.stringTable = stringTable; return assembler; } @@ -540,10 +542,14 @@ public ValidatedResult assemble() { } } + if (stringTable == null) { + stringTable = new StringTable(); + } + // Load model files into the processor. for (Map.Entry> entry : inputStreamModels.entrySet()) { try { - ModelLoader.load(traitFactory, properties, entry.getKey(), processor, entry.getValue()); + ModelLoader.load(traitFactory, properties, entry.getKey(), processor, entry.getValue(), stringTable); } catch (SourceException e) { processor.accept(new LoadOperation.Event(ValidationEvent.fromSourceException(e))); } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java index 47e7d4fef7f..ef575a777ce 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ModelLoader.java @@ -21,6 +21,7 @@ import java.net.URLConnection; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Logger; import software.amazon.smithy.model.SourceException; @@ -61,14 +62,15 @@ static void load( Map properties, String filename, Consumer operationConsumer, - Supplier contentSupplier + Supplier contentSupplier, + Function stringTable ) { try (InputStream inputStream = contentSupplier.get()) { if (filename.endsWith(".smithy")) { String contents = IoUtils.toUtf8String(inputStream); - new IdlModelParser(filename, contents).parse(operationConsumer); + new IdlModelLoader(filename, contents, stringTable).parse(operationConsumer); } else if (filename.endsWith(".jar")) { - loadJar(traitFactory, properties, filename, operationConsumer); + loadJar(traitFactory, properties, filename, operationConsumer, stringTable); } else if (filename.endsWith(".json") || filename.equals(SourceLocation.NONE.getFilename())) { // Assume it's JSON if there's a N/A filename. loadParsedNode(Node.parse(inputStream, filename), operationConsumer); @@ -104,7 +106,8 @@ private static void loadJar( TraitFactory traitFactory, Map properties, String filename, - Consumer operationConsumer + Consumer operationConsumer, + Function stringTable ) { URL manifestUrl = ModelDiscovery.createSmithyJarManifestUrl(filename); LOGGER.fine(() -> "Loading Smithy model imports from JAR: " + manifestUrl); @@ -123,7 +126,7 @@ private static void loadJar( } catch (IOException e) { throw throwIoJarException(model, e); } - }); + }, stringTable); } catch (IOException e) { throw throwIoJarException(model, e); } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java new file mode 100644 index 00000000000..54ff56859ea --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/StringTable.java @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import java.util.Arrays; +import java.util.function.Function; + +/** + * This is a simple, not thread-safe, caching string table that converts CharSequence to String objects. + * + *

The implementation uses an FNV-1a hash, and collisions simply overwrite the previously cached value. + */ +final class StringTable implements Function { + + private static final int FNV_OFFSET_BIAS = 0x811c9dc5; + private static final int FNV_PRIME = 0x1000193; + + private final String[] table; + private final int sizeBits; + private final int size; + private final int sizeMask; + + StringTable() { + // Defaults to 1024 entries. + this(11); + } + + StringTable(int sizeBits) { + if (sizeBits <= 0) { + throw new IllegalArgumentException("Cache sizeBits must be >= 1"); + } else if (sizeBits >= 17) { + throw new IllegalArgumentException("Refusing to create a cache with " + (1 << 17) + " entries"); + } + + this.sizeBits = sizeBits; + this.size = (1 << sizeBits); + this.sizeMask = size - 1; + this.table = new String[size]; + Arrays.fill(table, ""); + } + + @Override + public String apply(CharSequence chars) { + int idx = localIdxFromHash(chars); + String[] arr = table; + String text = arr[idx]; + + // On a cache hit, return the value if it matches. Otherwise, overwrite this value. + if (textEquals(chars, text)) { + return text; + } else { + String value = chars.toString(); + arr[idx] = value; + return value; + } + } + + private int localIdxFromHash(CharSequence chars) { + return getFnvHashCode(chars) & sizeMask; + } + + private static int getFnvHashCode(CharSequence text) { + int hashCode = FNV_OFFSET_BIAS; + int end = text.length(); + + for (int i = 0; i < end; i++) { + hashCode = (hashCode ^ text.charAt(i)) * FNV_PRIME; + } + + return hashCode; + } + + private static boolean textEquals(CharSequence left, String right) { + if (left.length() != right.length()) { + return false; + } + for (int i = 0; i < left.length(); i++) { + if (left.charAt(i) != right.charAt(i)) { + return false; + } + } + return true; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java index b7556be2b44..4fa749e2df3 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java @@ -157,7 +157,7 @@ private InternalSelector createSelector() { @Override public SelectorSyntaxException syntax(String message) { - return new SelectorSyntaxException(message, expression(), position(), line(), column()); + return new SelectorSyntaxException(message, input().toString(), position(), line(), column()); } private InternalSelector parseVariable() { @@ -232,7 +232,7 @@ private InternalSelector parseSelectorFunction() { if (selectors.size() != 1) { throw new SelectorSyntaxException( "The :not function requires a single selector argument", - expression(), functionPosition, line(), column()); + input().toString(), functionPosition, line(), column()); } return new NotSelector(selectors.get(0)); case "test": @@ -243,14 +243,14 @@ private InternalSelector parseSelectorFunction() { if (selectors.size() != 1) { throw new SelectorSyntaxException( "The :in function requires a single selector argument", - expression(), functionPosition, line(), column()); + input().toString(), functionPosition, line(), column()); } return new InSelector(selectors.get(0)); case "root": if (selectors.size() != 1) { throw new SelectorSyntaxException( "The :root function requires a single selector argument", - expression(), functionPosition, line(), column()); + input().toString(), functionPosition, line(), column()); } InternalSelector root = new RootSelector(selectors.get(0), roots.size()); roots.add(selectors.get(0)); @@ -259,15 +259,15 @@ private InternalSelector parseSelectorFunction() { if (selectors.size() > 2) { throw new SelectorSyntaxException( "The :topdown function accepts 1 or 2 selectors, but found " + selectors.size(), - expression(), functionPosition, line(), column()); + input().toString(), functionPosition, line(), column()); } return new TopDownSelector(selectors); case "each": - LOGGER.warning("The `:each` selector function has been renamed to `:is`: " + expression()); + LOGGER.warning("The `:each` selector function has been renamed to `:is`: " + input()); return IsSelector.of(selectors); default: LOGGER.warning(String.format("Unknown function name `%s` found in selector: %s", - name, expression())); + name, input())); return (context, shape, next) -> InternalSelector.Response.CONTINUE; } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java b/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java index 003c4ab1932..5adf313034c 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java @@ -185,10 +185,10 @@ private Optional createTemplatedEvent(Selector.ShapeMatch match * an {@link AttributeValue} and returns a String. */ private static final class MessageTemplate { - private final String template; - private final List> parts; + private final CharSequence template; + private final List> parts; - private MessageTemplate(String template, List> parts) { + private MessageTemplate(CharSequence template, List> parts) { this.template = template; this.parts = parts; } @@ -205,7 +205,7 @@ private MessageTemplate(String template, List> */ private String expand(AttributeValue value) { StringBuilder builder = new StringBuilder(); - for (Function part : parts) { + for (Function part : parts) { builder.append(part.apply(value)); } return builder.toString(); @@ -213,7 +213,7 @@ private String expand(AttributeValue value) { @Override public String toString() { - return template; + return template.toString(); } } @@ -225,7 +225,7 @@ public String toString() { */ private static final class MessageTemplateParser extends SimpleParser { private int mark = 0; - private final List> parts = new ArrayList<>(); + private final List> parts = new ArrayList<>(); private MessageTemplateParser(String expression) { super(expression); @@ -250,7 +250,7 @@ MessageTemplate parse() { } addLiteralPartIfNecessary(); - return new MessageTemplate(expression(), parts); + return new MessageTemplate(input(), parts); } @Override @@ -260,8 +260,8 @@ public RuntimeException syntax(String message) { } private void addLiteralPartIfNecessary() { - String slice = sliceFrom(mark); - if (!slice.isEmpty()) { + CharSequence slice = borrowSliceFrom(mark); + if (slice.length() > 0) { parts.add(ignoredAttribute -> slice); } mark = position(); diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/AstModelLoaderTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/AstModelLoaderTest.java index 51d1d29a792..b52305116e1 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/AstModelLoaderTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/AstModelLoaderTest.java @@ -27,7 +27,7 @@ public class AstModelLoaderTest { @Test public void failsToLoadPropertiesFromV1() { ValidatedResult model = Model.assembler() - .addImport(getClass().getResource("invalid/properties-v2-only.json")) + .addImport(getClass().getResource("invalid/version/properties-v2-only.json")) .assemble(); assertEquals(1, model.getValidationEvents(Severity.ERROR).size()); assertTrue(model.getValidationEvents(Severity.ERROR).get(0).getMessage() diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java index 2a3c8e4cab5..cf4f5ff9470 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java @@ -17,7 +17,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -35,6 +34,7 @@ import org.opentest4j.AssertionFailedError; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.SourceLocation; +import software.amazon.smithy.model.shapes.EnumShape; import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ResourceShape; @@ -158,7 +158,7 @@ public void limitsRecursion() { Model.assembler().addUnparsedModel("/foo.smithy", nodeBuilder.toString()).assemble().unwrap(); }); - assertThat(e.getMessage(), containsString("Parser exceeded the maximum allowed depth of")); + assertThat(e.getMessage(), containsString("Parser exceeded maximum allowed depth of")); } @Test @@ -285,7 +285,7 @@ public void loadsServicesWithNonconflictingUnitTypes() { @Test public void emitsVersionWhenNotSet() { List operations = new ArrayList<>(); - IdlModelParser parser = new IdlModelParser("foo.smithy", "namespace smithy.example\n"); + IdlModelLoader parser = new IdlModelLoader("foo.smithy", "namespace smithy.example\n", CharSequence::toString); parser.parse(operations::add); assertThat(operations, hasSize(1)); @@ -313,4 +313,30 @@ public void defaultValueSugaringDoesNotEatSubsequentDocumentation() { StringShape stringShape = model.expectShape(ShapeId.from("smithy.example#MyString"), StringShape.class); assertEquals(0, stringShape.getAllTraits().size()); } + + @Test + public void setsCorrectLocationForEnum() { + Model model = Model.assembler() + .addImport(getClass().getResource("valid/enums/enums.smithy")) + .addImport(getClass().getResource("valid/enums/enum-docs.smithy")) + .assemble() + .unwrap(); + + EnumShape enumWithoutValueTraits = model.expectShape(ShapeId.from("smithy.example#EnumWithoutValueTraits"), + EnumShape.class); + MemberShape barMember = enumWithoutValueTraits.getMember("BAR").orElseThrow(AssertionFailedError::new); + + assertThat(enumWithoutValueTraits.getSourceLocation().getLine(), is(5)); + assertThat(enumWithoutValueTraits.getSourceLocation().getColumn(), is(1)); + assertThat(barMember.getSourceLocation().getLine(), is(7)); + assertThat(barMember.getSourceLocation().getColumn(), is(5)); + + EnumShape foo = model.expectShape(ShapeId.from("smithy.example#Foo"), EnumShape.class); + MemberShape fooBarMember = foo.getMember("BAR").orElseThrow(AssertionFailedError::new); + + assertThat(foo.getSourceLocation().getLine(), is(5)); + assertThat(foo.getSourceLocation().getColumn(), is(1)); + assertThat(fooBarMember.getSourceLocation().getLine(), is(7)); + assertThat(fooBarMember.getSourceLocation().getColumn(), is(5)); + } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java deleted file mode 100644 index aca97f3b434..00000000000 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlTextParserTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package software.amazon.smithy.model.loader; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.stream.Stream; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.smithy.model.node.StringNode; - -public class IdlTextParserTest { - @ParameterizedTest - @MethodSource("validTextProvider") - public void parsesText(String input, String lexeme) { - IdlModelParser parser = new IdlModelParser("/foo", input); - StringNode result = IdlNodeParser.parseNode(parser).expectStringNode(); - assertThat(result.getValue(), equalTo(lexeme)); - } - - @ParameterizedTest - @MethodSource("invalidTextProvider") - public void throwsForInvalidText(String invalidInput) { - IdlModelParser parser = new IdlModelParser("/foo", invalidInput); - assertThrows(ModelSyntaxException.class, () -> IdlNodeParser.parseNode(parser).expectStringNode()); - } - - private static Stream validTextProvider() { - return Stream.of( - Arguments.of("\"foo\"", "foo"), - Arguments.of("\"foo\\\\bar\"", "foo\\bar"), - Arguments.of("\"\t\"", "\t"), - Arguments.of("\"\r\"", "\n"), - Arguments.of("\"\n\"", "\n"), - Arguments.of("\"\\t\"", "\t"), - Arguments.of("\"\\r\"", "\r"), - Arguments.of("\"\\n\"", "\n"), - Arguments.of("\"\\b\"", "\b"), - Arguments.of("\"\\f\"", "\f"), - Arguments.of("\"\\uffff\"", "\uffff"), - Arguments.of("\"\\u000A\"", "\n"), - Arguments.of("\"\\u000D\\u000A\"", "\r\n"), - Arguments.of("\"\r\\u000A\r\n\"", "\n\n\n"), - Arguments.of("\"\\\"\"", "\""), - Arguments.of("\"foo\\\\\"", "foo\\"), - Arguments.of("\"\\/\"", "/"), - Arguments.of("\"foo\\\nbaz\"", "foobaz"), - Arguments.of("\"foo\\\rbaz\"", "foobaz"), - Arguments.of("\"foo\\\r\nbaz\"", "foobaz"), - Arguments.of("\"\\\r\"", ""), - Arguments.of("\"\\\n\"", ""), - Arguments.of("\"\\\"\"", "\""), - Arguments.of("\"\\ud83d\\ude00\"", "\uD83D\uDE00"), - - // Text blocks - Arguments.of("\"\"\"\nf\"\"\"", "f"), - Arguments.of("\"\"\"\nfoo\"\"\"", "foo"), - Arguments.of("\"\"\"\nfoo\\\"\"\"\"", "foo\""), - Arguments.of("\"\"\"\nf\n foo\n\"\"\"", "f\n foo\n"), - Arguments.of("\"\"\"\n f\n foo\n\"\"\"", "f\n foo\n"), - Arguments.of("\"\"\"\n foo\nf\n\"\"\"", " foo\nf\n"), - Arguments.of("\"\"\"\n foo\n f\n\"\"\"", " foo\nf\n"), - Arguments.of("\"\"\"\n foo\n baz\"\"\"", "foo\nbaz"), - Arguments.of("\"\"\"\n\n\n\"\"\"", "\n\n"), - Arguments.of("\"\"\"\n foo\n baz\n \"\"\"", "foo\nbaz\n"), - Arguments.of("\"\"\"\n foo\n baz\n \"\"\"", "foo\n baz\n"), - Arguments.of("\"\"\"\n foo\\n bar\n baz\"\"\"", "foo\n bar\nbaz"), - Arguments.of("\"\"\"\n{ \"foo\": \"bar\" }\"\"\"", "{ \"foo\": \"bar\" }"), - Arguments.of("\"\"\"\n \"a\" \"\"\"", "\"a\""), - Arguments.of("\"\"\"\n \" \"\"\"", "\""), - // Empty lines and lines with only ws do not contribute to incidental ws. - Arguments.of("\"\"\"\n\n foo\n \n\n \n \"\"\"", "\nfoo\n\n\n\n"), - // If the last line is offset to the right, it's discarded since it's all whitespace. - Arguments.of("\"\"\"\n foo\n \"\"\"", "foo\n"), - Arguments.of("\"\"\"\n Foo\\\n Baz\"\"\"", "FooBaz"), - Arguments.of("\"\"\"\r Foo\\\r Baz\"\"\"", "FooBaz"), - Arguments.of("\"\"\"\r\n Foo\\\r\n Baz\"\"\"", "FooBaz")); - } - - private static Stream invalidTextProvider() { - return Stream.of( "\"\b\"", "\"\"\"", "\"\\\"", "\"\\'\""); - } -} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/StringTableTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/StringTableTest.java new file mode 100644 index 00000000000..5a3175d128f --- /dev/null +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/StringTableTest.java @@ -0,0 +1,65 @@ +package software.amazon.smithy.model.loader; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; + +import java.nio.CharBuffer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class StringTableTest { + @Test + public void cachesAndReturnsStringValues() { + StringTable table = new StringTable(); + + CharBuffer originalFoo = CharBuffer.wrap(new char[]{'f', 'o', 'o'}); + String internedFoo = table.apply(originalFoo); + + assertThat(table.apply(originalFoo), equalTo(internedFoo)); + assertThat(table.apply(originalFoo), sameInstance(internedFoo)); + } + + @Test + public void overwritePreviousValuesWhenFull() { + StringTable table = new StringTable(1); // 2 entries + + CharBuffer originalFoo = CharBuffer.wrap(new char[]{'f', 'o', 'o'}); + CharBuffer originalFoo1 = CharBuffer.wrap(new char[]{'f', 'o', 'o', '1'}); + CharBuffer originalFoo2 = CharBuffer.wrap(new char[]{'f', 'o', 'o', '2'}); + + String internedFoo = table.apply(originalFoo); + + assertThat(internedFoo, sameInstance(table.apply(originalFoo))); + assertThat(internedFoo, equalTo(originalFoo.toString())); + assertThat(internedFoo, not(sameInstance(originalFoo.toString()))); + + String internedFoo1 = table.apply(originalFoo1); + String internedFoo2 = table.apply(originalFoo2); + + assertThat(internedFoo1, sameInstance(table.apply(originalFoo1))); + assertThat(internedFoo1, equalTo(originalFoo1.toString())); + assertThat(internedFoo1, not(sameInstance(originalFoo1.toString()))); + + assertThat(internedFoo2, sameInstance(table.apply(originalFoo2))); + assertThat(internedFoo2, equalTo(originalFoo2.toString())); + assertThat(internedFoo2, not(sameInstance(originalFoo2.toString()))); + + // The cache is now full. Overwrite an entry and store 'foo', causing the previously interned instance to + // differ from the newly computed instance. + String nextInternedFoo = table.apply(originalFoo); + assertThat(nextInternedFoo, equalTo(originalFoo.toString())); + assertThat(nextInternedFoo, not(sameInstance(internedFoo))); + } + + @Test + public void doesNotCreateTooBigOfCache() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new StringTable(17)); + } + + @Test + public void doesNotCreateTooSmallOfCache() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new StringTable(0)); + } +} diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java new file mode 100644 index 00000000000..dab187dd1e1 --- /dev/null +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/TokenizerTest.java @@ -0,0 +1,524 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TokenizerTest { + @Test + public void tokenizesIdentifierFollowedByQuote() { + String contents = "metadata\"foo\"=\"bar\""; + IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.STRING)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(9)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.EQUAL)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(14)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.STRING)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(15)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.EOF)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(20)); + } + + @Test + public void tokenizesSingleCharacterLexemes() { + String contents = "\t\r\n\r,@$.:{}()[]# "; + IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.SPACE)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(1)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.NEWLINE)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + assertThat(tokenizer.getCurrentTokenSpan(), is(2)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.NEWLINE)); + assertThat(tokenizer.getCurrentTokenLine(), is(2)); + assertThat(tokenizer.getCurrentTokenColumn(), is(1)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.COMMA)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(1)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.AT)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.DOLLAR)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(3)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.DOT)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(4)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.COLON)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(5)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.LBRACE)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(6)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.RBRACE)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(7)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.LPAREN)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(8)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.RPAREN)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(9)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.LBRACKET)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(10)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.RBRACKET)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(11)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.POUND)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(12)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.SPACE)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(13)); + assertThat(tokenizer.getCurrentTokenSpan(), is(1)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.EOF)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(14)); + } + + @Test + public void tokenizesMultipleSpacesAndTabsIntoSingleLexeme() { + String contents = " \t "; + IdlTokenizer tokenizer = IdlTokenizer.builder().model(contents).build(); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.SPACE)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(1)); + assertThat(tokenizer.getCurrentTokenSpan(), is(5)); + assertThat(tokenizer.getCurrentTokenLexeme().toString(), equalTo(" \t ")); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.EOF)); + assertThat(tokenizer.getCurrentTokenLine(), is(1)); + assertThat(tokenizer.getCurrentTokenColumn(), is(6)); + } + + @Test + public void throwsWhenAccessingErrorAndNoError() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("a").build(); + + Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenError); + } + + @Test + public void storesErrorForInvalidSyntax() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("!").build(); + + assertThat(tokenizer.next(), is(IdlToken.ERROR)); + assertThat(tokenizer.getCurrentTokenError(), equalTo("Unexpected character: '!'")); + } + + @Test + public void throwsWhenAccessingNumberAndNoNumber() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("a").build(); + + Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenNumberValue); + } + + @Test + public void storesCurrentTokenNumber() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("10").build(); + + assertThat(tokenizer.next(), is(IdlToken.NUMBER)); + assertThat(tokenizer.getCurrentTokenNumberValue(), equalTo(10L)); + } + + @Test + public void throwsWhenAccessingStringValue() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("10").build(); + + Assertions.assertThrows(ModelSyntaxException.class, tokenizer::getCurrentTokenStringSlice); + } + + @Test + public void storesCurrentTokenString() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"hello\"").build(); + + assertThat(tokenizer.next(), is(IdlToken.STRING)); + assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("hello")); + } + + @Test + public void storesCurrentTokenStringForIdentifier() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("hello").build(); + + assertThat(tokenizer.next(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("hello")); + } + + @Test + public void throwsWhenTooNested() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("").build(); + + for (int i = 0; i < 64; i++) { + tokenizer.increaseNestingLevel(); + } + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, tokenizer::increaseNestingLevel); + + assertThat(e.getMessage(), + startsWith("Syntax error at line 1, column 1: Parser exceeded maximum allowed depth of 64")); + } + + @Test + public void tokenizerInternsStrings() { + List tracked = new ArrayList<>(); + Function table = c -> { + tracked.add(c); + return c.toString(); + }; + + IdlTokenizer tokenizer = IdlTokenizer.builder() + .model("foo") + .stringTable(table) + .build(); + + tokenizer.internString("hi"); + + assertThat(tracked, contains("hi")); + } + + @Test + public void skipSpaces() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model(" hi").build(); + + tokenizer.skipSpaces(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenColumn(), is(5)); + } + + @Test + public void skipsExpectedSpaces() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model(" hi").build(); + + tokenizer.expectAndSkipSpaces(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenColumn(), is(5)); + } + + @Test + public void failsWhenExpectedSpacesNotThere() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("abc").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, tokenizer::expectAndSkipSpaces); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one or more spaces, but " + + "found IDENTIFIER('abc')")); + } + + @Test + public void skipWhitespace() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n hi").build(); + + tokenizer.skipWs(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void expectAndSkipWhitespace() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n hi").build(); + + tokenizer.expectAndSkipWhitespace(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenLine(), is(3)); + assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void throwsWhenExpectedWhitespaceNotFound() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("hi").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + tokenizer::expectAndSkipWhitespace); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one or more whitespace " + + "characters, but found IDENTIFIER('hi')")); + } + + @Test + public void skipDocsAndWhitespace() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model(" \n\n /// Docs\n/// Docs\n\n hi").build(); + + tokenizer.skipWsAndDocs(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenLine(), is(6)); + assertThat(tokenizer.getCurrentTokenColumn(), is(2)); + } + + @Test + public void expectsAndSkipsBr() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("\n Hi").build(); + + tokenizer.expectAndSkipBr(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.IDENTIFIER)); + assertThat(tokenizer.getCurrentTokenLine(), is(2)); + assertThat(tokenizer.getCurrentTokenColumn(), is(3)); + } + + @Test + public void throwsWhenBrNotFound() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + tokenizer::expectAndSkipBr); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected a line break, but " + + "found IDENTIFIER('Hi')")); + } + + @Test + public void expectCurrentTokenLexeme() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + tokenizer.expectCurrentLexeme("Hi"); + } + + @Test + public void throwsWhenCurrentTokenLexemeUnexpected() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, () -> { + tokenizer.expectCurrentLexeme("Bye"); + }); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected `Bye`, but found `Hi`")); + assertThat(e.getSourceLocation().getLine(), is(1)); + assertThat(e.getSourceLocation().getColumn(), is(1)); + } + + @Test + public void throwsWhenCurrentTokenLexemeUnexpectedAndSameLength() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("A").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, () -> { + tokenizer.expectCurrentLexeme("B"); + }); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected `B`, but found `A`")); + assertThat(e.getSourceLocation().getLine(), is(1)); + assertThat(e.getSourceLocation().getColumn(), is(1)); + } + + @Test + public void tokenDoesNotStartWith() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + assertThat(tokenizer.doesCurrentIdentifierStartWith('B'), is(false)); + } + + @Test + public void tokenDoesStartWith() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + assertThat(tokenizer.doesCurrentIdentifierStartWith('H'), is(true)); + } + + @Test + public void tokenDoesNotStartWithBecauseNotIdentifier() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"Hi\"").build(); + + assertThat(tokenizer.doesCurrentIdentifierStartWith('H'), is(false)); + } + + @Test + public void expectsSingleTokenType() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + tokenizer.expect(IdlToken.IDENTIFIER); + } + + @Test + public void failsForSingleExpectedToken() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + () -> tokenizer.expect(IdlToken.NUMBER)); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected NUMBER but " + + "found IDENTIFIER('Hi')")); + } + + @Test + public void expectsMultipleTokenTypes() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + tokenizer.expect(IdlToken.STRING, IdlToken.IDENTIFIER); + } + + @Test + public void failsForMultipleExpectedTokens() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("Hi").build(); + + ModelSyntaxException e = Assertions.assertThrows(ModelSyntaxException.class, + () -> tokenizer.expect(IdlToken.NUMBER, IdlToken.LBRACE)); + + assertThat(e.getMessage(), startsWith("Syntax error at line 1, column 1: Expected one of NUMBER, LBRACE('{'); " + + "but found IDENTIFIER('Hi')")); + } + + @Test + public void failsWhenSingleForwardSlashFound() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("/").build(); + + tokenizer.next(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.ERROR)); + assertThat(tokenizer.getCurrentTokenError(), equalTo("Expected a '/' to follow '/' to form a comment.")); + } + + @Test + public void throwsWhenTraversingPastEof() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("").build(); + + assertThat(tokenizer.next(), is(IdlToken.EOF)); + + Assertions.assertThrows(NoSuchElementException.class, tokenizer::next); + } + + @Test + public void returnsCapturedDocsInRange() { + IdlTokenizer tokenizer = IdlTokenizer + .builder() + .model("/// Hi\n" + + "/// There\n" + + "/// 123\n" + + "/// 456\n") + .build(); + + tokenizer.skipWsAndDocs(); + String lines = tokenizer.removePendingDocCommentLines(); + + assertThat(lines, equalTo("Hi\nThere\n123\n456")); + assertThat(tokenizer.removePendingDocCommentLines(), nullValue()); + } + + @Test + public void tokenizesStringWithNewlines() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"hi\nthere\"").build(); + + tokenizer.next(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.STRING)); + assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("hi\nthere")); + assertThat(tokenizer.getCurrentTokenLexeme().toString(), equalTo("\"hi\nthere\"")); + assertThat(tokenizer.getCurrentTokenSpan(), is(10)); + + tokenizer.next(); + assertThat(tokenizer.getCurrentToken(), is(IdlToken.EOF)); + assertThat(tokenizer.getCurrentTokenLine(), is(2)); + assertThat(tokenizer.getCurrentTokenColumn(), is(7)); + } + + @Test + public void tokenizesEmptyStrings() { + IdlTokenizer tokenizer = IdlTokenizer.builder().model("\"\"").build(); + + tokenizer.next(); + + assertThat(tokenizer.getCurrentToken(), is(IdlToken.STRING)); + assertThat(tokenizer.getCurrentTokenStringSlice().toString(), equalTo("")); + assertThat(tokenizer.getCurrentTokenLexeme().toString(), equalTo("\"\"")); + assertThat(tokenizer.getCurrentTokenSpan(), is(2)); + } +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-list-member-names.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-list-member-names.errors index fdc1f145ce8..5f2b51f3107 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-list-member-names.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-list-member-names.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 7, column 5 near `member`: Expected: '}', but found 'm' | Model +[ERROR] -: Syntax error at line 7, column 5: Expected RBRACE('}') but found IDENTIFIER('member') | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-map-member-names.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-map-member-names.errors index 4bc1d3fc896..776102a47a1 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-map-member-names.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-map-member-names.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 7, column 5 near `key`: Expected: '}', but found 'k' | Model +[ERROR] -: Syntax error at line 7, column 5: Expected RBRACE('}') but found IDENTIFIER('key') | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-resource-binding.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-resource-binding.errors index 204d34dcff6..612a9cf2264 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-resource-binding.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-resource-binding.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 10, column 22 near `, | Model +[ERROR] -: Duplicate member: 'create' | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-service-binding.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-service-binding.errors index ca924439a59..c3c8dbd9f02 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-service-binding.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-service-binding.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 7, column 26 near `, | Model +[ERROR] -: Duplicate member: 'version' | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-set-member-names.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-set-member-names.errors index fdc1f145ce8..5f2b51f3107 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-set-member-names.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-set-member-names.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 7, column 5 near `member`: Expected: '}', but found 'm' | Model +[ERROR] -: Syntax error at line 7, column 5: Expected RBRACE('}') but found IDENTIFIER('member') | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-structure-member-names.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-structure-member-names.errors index 39349cf2d31..a9ef513b047 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-structure-member-names.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-structure-member-names.errors @@ -1 +1 @@ -[ERROR] com.foo#Struct: Parse error at line 7, column 8 near `: `: Duplicate member of com.foo#Struct: 'foo' | Model +[ERROR] com.foo#Struct: Syntax error at line 7, column 5: Duplicate member of `com.foo#Struct`: 'foo' | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-member-names.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-member-names.errors index 93895560df3..8e1663aad16 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-member-names.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-member-names.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 8, column 1 | Model +[ERROR] -: Duplicate member of trait: 'message' | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-object-keys.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-object-keys.errors index a445b9a5410..cda150522a5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-object-keys.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/dupe-trait-object-keys.errors @@ -1 +1 @@ -[ERROR] -: Parse error at line 8, column 21 near `, | Model +[ERROR] -: Duplicate member: 'value' | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key1.smithy deleted file mode 100644 index e9167d54e77..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key1.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 3, column 9 near `: `: Expected: ')', but found ':' -namespace com.foo -@foo(100: true) -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key2.smithy deleted file mode 100644 index 221ae40fdd4..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key2.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 3, column 8 near `: `: Expected: ')', but found ':' -namespace com.foo -@foo({}: true) -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key3.smithy deleted file mode 100644 index 367bd7a48c0..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-invalid-object-key3.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 3, column 17 near `{}`: Unexpected object key character: '{' -namespace com.foo -@foo(foo: true, {}: true) -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list1.smithy deleted file mode 100644 index 2232a3c7b1d..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list1.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 5, column 1 near `[EOF]`: Expected: ']', but found '[EOF]' | Model -namespace com.foo -@tags(["foo", "bar" -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list2.smithy deleted file mode 100644 index 75d5edb8dd7..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list2.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 5, column 1 near `[EOF]`: Expected: ']', but found '[EOF]' -namespace com.foo -@tags(["foo", "bar", -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list3.smithy deleted file mode 100644 index e28933d9803..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list3.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 5, column 1 near `[EOF]`: Expected: ']', but found '[EOF]' -namespace com.foo -@tags([ -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list4.smithy deleted file mode 100644 index 8767143ede1..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-list4.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 4, column 1 near `[EOF]`: Expected: ']', but found '[EOF]' -namespace com.foo -@tags([[[[[[[[[[[[ diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object1.smithy deleted file mode 100644 index 30ecdab8327..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object1.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 8 near `MyString`: Expected: ')', but found 'M' -namespace com.foo -@foo( -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object2.smithy deleted file mode 100644 index 8584c90e7d2..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object2.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 1 near `string`: Expected: ')', but found 's' -namespace com.foo -@foo("bar" -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object3.smithy deleted file mode 100644 index 0e93fd78bee..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object3.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 5, column 1 near `[EOF]`: Expected: ':', but found '[EOF]' -namespace com.foo -@foo(bar: -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object4.smithy deleted file mode 100644 index c1cbbe0a501..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object4.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 8 near `MyString`: Expected: ':', but found 'M' | Model -namespace com.foo -@foo(bar: "baz" -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object5.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object5.smithy deleted file mode 100644 index 49e5d095a57..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object5.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 8 near `MyString`: Expected: ':', but found 'M' -namespace com.foo -@foo(bar: "baz", -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object6.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object6.smithy deleted file mode 100644 index 1de722abe96..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/annotation-unclosed-object6.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 1 near `string`: Expected: ':', but found 's' -namespace com.foo -@foo(bar: "baz", "bam" -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-block-missing-closing.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-block-missing-closing.smithy index dca731c4115..35de73f05a1 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-block-missing-closing.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-block-missing-closing.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 1 near ``: Expected: '}' +// Syntax error at line 7, column 1: Expected RBRACE('}') but found EOF | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-newline.smithy index 5657545dd42..75a1cc97297 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-newline.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 29 near `apply`: Expected a line break | Model +// Syntax error at line 5, column 29: Expected a line break, but found IDENTIFIER('apply') | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-space.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-space.smithy index 832b2746ea5..735de88a866 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-space.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-space.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 10 near `{ `: Expected one or more whitespace characters | Model +// Syntax error at line 5, column 10: Expected one or more whitespace characters, but found LBRACE('{') $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-trait-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-trait-value.smithy index 38e1baa5fea..b5d83d70b37 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-trait-value.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply/apply-missing-trait-value.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 1 near `string`: Expected: '@', but found 's' +// Syntax error at line 5, column 1: Expected one of AT('@'), LBRACE('{'); but found IDENTIFIER('string') | Model namespace com.foo apply SomeShape // comment so spaces aren't eaten diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-colon-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-colon-newline.smithy index c0b7cf37cfc..9da193541d9 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-colon-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-colon-newline.smithy @@ -1,3 +1,3 @@ -// Parse error at line 2, column 10 +// Syntax error at line 2, column 10 $version: "2.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-missing-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-missing-value.smithy index aa20ed498d2..5b0b392d9ad 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-missing-value.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-missing-value.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 10 +// Syntax error at line 2, column 10 $version: diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-newline.smithy index 027407e0584..557b5505674 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-newline.smithy @@ -1,3 +1,3 @@ -// Parse error at line 2, column 9 +// Syntax error at line 2, column 9 $version : "2.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-statement-before-others.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-statement-before-others.smithy index 506bb803d4a..70d78d413f7 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-statement-before-others.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-statement-before-others.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 1 near `$v`: Expected a valid identifier character, but found '$' +// Syntax error at line 4, column 1: Expected IDENTIFIER but found DOLLAR('$') | Model namespace foo.baz $version: "2.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-version-defined-twice.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-version-defined-twice.smithy index 2da0ac76891..88683cc5cb0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-version-defined-twice.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-version-defined-twice.smithy @@ -1,4 +1,4 @@ -// Parse error at line 3, column 11 near `"2`: Duplicate control statement `version` +// Syntax error at line 3, column 11: Duplicate control statement `version` $version: "2.0" $version: "2.0" namespace com.baz diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key.smithy index 4a36800254d..0e845caba47 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 2 near `!f`: Expected a valid identifier character, but found '!' +// Syntax error at line 2, column 2: Unexpected character: '!' | Model $!foo: "1.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key2.smithy index 5cbc356109c..26960e625aa 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key2.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-invalid-key2.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 5 near `.b`: Expected: ':', but found '.' +// Syntax error at line 2, column 5: Expected COLON(':') but found DOT('.') | Model $foo.bar: "1.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-no-colon.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-no-colon.smithy index 79dff552c16..3c72dc088fc 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-no-colon.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/control-with-no-colon.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 10 near `"1`: Expected: ':', but found '"' +// Syntax error at line 2, column 10: Expected COLON(':') but found STRING('"1.0"') | Model $version "1.0" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/no-newline-after-control.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/no-newline-after-control.smithy index b7d5f6b1388..4aebf9887a5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/no-newline-after-control.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/control/no-newline-after-control.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 17 near `$x`: Expected a line break | Model +// Syntax error at line 2, column 17: Expected a line break, but found DOLLAR('$') | Model $version: "2.0" $xyz: "100" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-in-v1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-in-v1.smithy index ab12782a73d..6cc4196f2bb 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-in-v1.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-in-v1.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 17 near `= `: @default assignment is only supported in IDL version 2 or later | Model +// Syntax error at line 6, column 17: @default assignment is only supported in IDL version 2 or later | Model $version: "1.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-incomplete-node.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-incomplete-node.smithy index 687b21ba109..2a9d91db3e9 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-incomplete-node.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-incomplete-node.smithy @@ -1,4 +1,4 @@ -// Parse error at line 8, column 1 near ``: Expected: ']' +// Syntax error at line 8, column 1: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found EOF | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-structure-close.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-structure-close.smithy index d7b195a5bc3..b554cb4fd7e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-structure-close.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-structure-close.smithy @@ -1,4 +1,4 @@ -// Parse error at line 8, column 1 near ``: Expected: '}' +// Syntax error at line 8, column 1: Expected RBRACE('}') but found EOF | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-value.smithy index eacda9b6d3d..86cb919ad03 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-value.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-missing-value.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 18 +// Syntax error at line 7, column 18 $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-with-newline-before-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-with-newline-before-value.smithy index ee969789727..8753f539132 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-with-newline-before-value.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/default-with-newline-before-value.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 18 +// Syntax error at line 7, column 18 $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/missing-newline-after-assignment.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/missing-newline-after-assignment.smithy index ba716c99a15..17e24a2abd9 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/missing-newline-after-assignment.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/missing-newline-after-assignment.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 24 near `baz`: Expected a line break | Model +// Syntax error at line 6, column 24: Expected a line break, but found IDENTIFIER('baz') | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/unions-do-not-support-defaults.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/unions-do-not-support-defaults.smithy index 34487a654f7..81832fcaa69 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/unions-do-not-support-defaults.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/defaults/unions-do-not-support-defaults.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 17 near `= ` +// Syntax error at line 7, column 17: Expected IDENTIFIER but found EQUAL('=') | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/elided-union-member-from-resource.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/elided-union-member-from-resource.smithy index 124cb5bbd7b..37e0200522b 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/elided-union-member-from-resource.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/elided-union-member-from-resource.smithy @@ -1,4 +1,4 @@ -// Parse error at line 12, column 15 near `for`: Expected: '{', but found 'f' +// Syntax error at line 12, column 15: Expected LBRACE('{') but found IDENTIFIER('for') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/for-after-with.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/for-after-with.smithy similarity index 71% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/for-after-with.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/for-after-with.smithy index 5617a02753f..a948001e0c0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/for-after-with.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/for-after-with.smithy @@ -1,4 +1,4 @@ -// Parse error at line 17, column 41 near `for`: Expected: '{', but found 'f' +// Syntax error at line 17, column 41: Expected LBRACE('{') but found IDENTIFIER('for') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/for-referencing-non-resource.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/for-referencing-non-resource.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/for-referencing-non-resource.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/elision/for-referencing-non-resource.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/missing-newline-after-assignment.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/missing-newline-after-assignment.smithy deleted file mode 100644 index 13b78a67c68..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/missing-newline-after-assignment.smithy +++ /dev/null @@ -1,7 +0,0 @@ -// Parse error at line 6, column 13 near `BAZ`: Expected a line break | Model -$version: "2.0" -namespace smithy.example - -intEnum Foo { - BAR = 1 BAZ = 2 -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/enum-with-bad-values.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-with-bad-values.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/enum-with-bad-values.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-with-bad-values.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/enum-with-newline-before-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-with-newline-before-value.smithy similarity index 66% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/enum-with-newline-before-value.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-with-newline-before-value.smithy index af2857d75d2..bb2a5ed8dc2 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/enum-with-newline-before-value.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-with-newline-before-value.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 10 +// Syntax error at line 6, column 10 $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-without-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-without-member.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-without-member.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/enum-without-member.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/idl-enums-in-v1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/idl-enums-in-v1.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/idl-enums-in-v1.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/idl-enums-in-v1.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/int-enum-without-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/int-enum-without-member.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/int-enum-without-member.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/int-enum-without-member.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-bad-value.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-bad-value.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-bad-value.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-bad-value.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-float.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-float.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-float.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-float.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-long.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-long.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enum-values/intEnum-with-long.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/intEnum-with-long.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/missing-newline-after-assignment.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/missing-newline-after-assignment.smithy new file mode 100644 index 00000000000..592b89a6a4e --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/enums/missing-newline-after-assignment.smithy @@ -0,0 +1,7 @@ +// Syntax error at line 6, column 13: Expected a line break, but found IDENTIFIER('BAZ') | Model +$version: "2.0" +namespace smithy.example + +intEnum Foo { + BAR = 1 BAZ = 2 +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-id-after-type.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-id-after-type.smithy deleted file mode 100644 index f7eaf593d23..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-id-after-type.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 8 near `(N`: Expected a valid identifier character, but found '(' | Model -namespace com.foo - -string (Nope) diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-name-but-eof.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-name-but-eof.smithy deleted file mode 100644 index 6a66f022631..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/expected-shape-name-but-eof.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 7 near `\n`: Expected one or more spaces | Model -namespace com.foo - -string diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-error.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-error.smithy index bd49f6fd677..86f26709e7f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-error.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-error.smithy @@ -1,4 +1,4 @@ -// Expected: '[', but found '=' +// Syntax error at line 6, column 12: Expected COLON(':') but found WALRUS(':=') $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-name-override.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-name-override.smithy index cf09aec7143..77df7208659 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-name-override.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-name-override.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 14 near `CustomName`: Expected: '{', but found 'C' +// Syntax error at line 6, column 14: Expected LBRACE('{') but found IDENTIFIER('CustomName') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-single-error.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-single-error.smithy index f6cdc6769f0..945e4c38c79 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-single-error.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-single-error.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 12 near `= ` +// Syntax error at line 6, column 12: Expected COLON(':') but found EQUAL('=') $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-structure-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-structure-member.smithy index 6d968beb4ec..eff28262c8f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-structure-member.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/inline-structure-member.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 14 near `= `: Expected a valid identifier character, but found '=' +// Syntax error at line 6, column 13: Expected COLON(':') but found WALRUS(':=') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/walrus-operator-in-node.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/walrus-operator-in-node.smithy index 1ab74a1009a..42e75f83382 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/walrus-operator-in-node.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/inline-io/walrus-operator-in-node.smithy @@ -1,4 +1,4 @@ -// The `:=` syntax may only be used when defining inline operation input and output shapes. +// Syntax error at line 5, column 13: Expected COLON(':') but found WALRUS(':=') | Model $version: "2.0" metadata foo = { diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-shape-id-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-shape-id-key.smithy deleted file mode 100644 index 68d066383eb..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-shape-id-key.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 3, column 6 near `.b`: Expected: ':', but found '.' -metadata foo = { - foo.bar#baz: "hello" -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-text-block-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-text-block-key.smithy deleted file mode 100644 index ac82a0bf814..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-object-text-block-key.smithy +++ /dev/null @@ -1,5 +0,0 @@ -// Parse error at line 3, column 5 near `"\n`: Expected: ':', but found '"' -metadata foo = { - """ - Key""": "hello" -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-unquoted-shape-id.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-unquoted-shape-id.smithy deleted file mode 100644 index 439d6d14f90..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-unquoted-shape-id.smithy +++ /dev/null @@ -1,5 +0,0 @@ -// Parse error at line 3, column 19 near `#\n`: Expected a valid identifier character, but found '#' - -metadata abc = aa$# - -namespace foo.baz diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-invalid-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-invalid-member.smithy deleted file mode 100644 index 628749b997e..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-invalid-member.smithy +++ /dev/null @@ -1,6 +0,0 @@ -// Parse error at line 5, column 3 near `foo`: Expected: '}', but found 'f' | Model -namespace com.foo - -list MyList { - foo: smithy.api#String, -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-empty-members.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-empty-members.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/list-empty-members.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-empty-members.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-invalid-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-invalid-member.smithy new file mode 100644 index 00000000000..17c1dcb6915 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/list-invalid-member.smithy @@ -0,0 +1,6 @@ +// Syntax error at line 5, column 3: Expected RBRACE('}') but found IDENTIFIER('foo') | Model +namespace com.foo + +list MyList { + foo: smithy.api#String, +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/no-sets-in-v2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/no-sets-in-v2.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/no-sets-in-v2.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/no-sets-in-v2.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-empty-members.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-empty-members.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-empty-members.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-empty-members.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-invalid-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-invalid-member.smithy new file mode 100644 index 00000000000..c02d3ae2def --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/lists/set-invalid-member.smithy @@ -0,0 +1,6 @@ +// Syntax error at line 5, column 3: Expected RBRACE('}') but found IDENTIFIER('foo') | Model +namespace com.foo + +set MySet { + foo: smithy.api#String, +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters1.smithy deleted file mode 100644 index d68680f76fb..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters1.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 4, column 1 near `[EOF]`: Expected: '}', but found '[EOF]' -namespace com.foo -map MyMap { diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters2.smithy deleted file mode 100644 index 078e67514bc..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters2.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 4, column 1 near ``: Expected: '}', but found '' -namespace com.foo -map MyMap { key: SomeShape, diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters3.smithy deleted file mode 100644 index aa7e4cbda33..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters3.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 4, column 1 near `[EOF]`: Expected: '}', but found '[EOF]' -namespace com.foo -map MyMap { key: SomeShape, value: AnotherShape diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters4.smithy deleted file mode 100644 index 4a85142bad5..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-unclosed-parameters4.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 4, column 1 near `[EOF]`: Expected: '}', but found '[EOF]' -namespace com.foo -map MyMap { key: smithy.api#String, value: smithy.api#String diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-empty-members.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-empty-members.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-empty-members.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-empty-members.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-invalid-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-invalid-member.smithy similarity index 54% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-invalid-member.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-invalid-member.smithy index 9836767a472..3532be6051e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/map-invalid-member.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-invalid-member.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 3 near `fuzz`: Expected: '}', but found 'f' | Model +// Syntax error at line 7, column 3: Expected RBRACE('}') but found IDENTIFIER('fuzz') | Model namespace com.foo map MyMap { diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters1.smithy new file mode 100644 index 00000000000..616b779fe89 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters1.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 4, column 1: Expected RBRACE('}') but found EOF | Model +namespace com.foo +map MyMap { diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters2.smithy new file mode 100644 index 00000000000..489cea8522a --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters2.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 4, column 1: Expected RBRACE('}') but found EOF | Model +namespace com.foo +map MyMap { key: SomeShape, diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters3.smithy new file mode 100644 index 00000000000..17dfa066d58 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters3.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 4, column 1: Expected RBRACE('}') but found EOF | Model +namespace com.foo +map MyMap { key: SomeShape, value: AnotherShape diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters4.smithy new file mode 100644 index 00000000000..b90f7fcc4ed --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/maps/map-unclosed-parameters4.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 4, column 1: Expected RBRACE('}') but found EOF | Model +namespace com.foo +map MyMap { key: smithy.api#String, value: smithy.api#String diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-shape-id-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-shape-id-key.smithy new file mode 100644 index 00000000000..0faad422f5f --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-shape-id-key.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 3, column 6: Expected COLON(':') but found DOT('.') | Model +metadata foo = { + foo.bar#baz: "hello" +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-text-block-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-text-block-key.smithy new file mode 100644 index 00000000000..5ecb7af8fc7 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-object-text-block-key.smithy @@ -0,0 +1,5 @@ +// Syntax error at line 3, column 3: Expected one of RBRACE('}'), STRING('"'), IDENTIFIER; but found TEXT_BLOCK('"""\n Key"""') | Model +metadata foo = { + """ + Key""": "hello" +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-unquoted-shape-id.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-unquoted-shape-id.smithy new file mode 100644 index 00000000000..9c78acc887c --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/invalid-unquoted-shape-id.smithy @@ -0,0 +1,5 @@ +// Syntax error at line 3, column 19: Expected IDENTIFIER but found POUND('#') | Model + +metadata abc = aa$# + +namespace foo.baz diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-multiple-lines.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-multiple-lines.smithy index 799fda5d8d1..c4853f8b8ff 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-multiple-lines.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-multiple-lines.smithy @@ -1,4 +1,4 @@ -// Parse error at line 3, column 16 near `//`: Expected a valid identifier character, but found '/' | Model +// Syntax error at line 3, column 16: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found COMMENT('// this is not allowed\n') | Model $version: "2.0" -metadata foo = // this is not allows +metadata foo = // this is not allowed "bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-must-come-before-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-must-come-before-namespace.smithy index b4d81dec171..1d714b02f08 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-must-come-before-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-must-come-before-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 9 near ` foo`: Metadata statements must appear before a namespace statement +// Syntax error at line 5, column 1: Unknown shape type: metadata | Model $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-with-invalid-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-with-invalid-key.smithy index c618dc2266e..0e04ae2044f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-with-invalid-key.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/metadata-with-invalid-key.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 13 near `.b`: Expected: '=', but found '.' +// Syntax error at line 2, column 13: Expected EQUAL('=') but found DOT('.') | Model metadata foo.bar = "baz" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/no-newline-after-metadata.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/no-newline-after-metadata.smithy index e7e8df4e133..c55addc27d5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/no-newline-after-metadata.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/no-newline-after-metadata.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 22 near `metadata`: Expected a line break | Model +// Syntax error at line 3, column 22: Expected a line break, but found IDENTIFIER('metadata') | Model $version: "2.0" metadata foo = "bar" metadata baz = "bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/not-newline-after-metadata.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/not-newline-after-metadata.smithy index 6501ecb95b4..b4679acc1a0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/not-newline-after-metadata.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/not-newline-after-metadata.smithy @@ -1,4 +1,4 @@ -// Parse error at line 3, column 9 +// Syntax error at line 3, column 9 $version: "2.0" metadata foo = "bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy index 992ebaa7854..ffbdc455fda 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-quoted.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 9 near `"f`: Expected one or more spaces | Model +// Syntax error at line 3, column 9: Expected one or more spaces, but found STRING('"foo"') | Model $version: "2.0" metadata"foo"="bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy index 2b3b2a43180..039ecd8791a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/metadata/space-after-metadata-unquoted.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 9 near `foo`: Expected one or more spaces | Model +// Syntax error at line 3, column 1: Expected `metadata`, but found `metadatafoo` | Model $version: "2.0" metadatafoo"="bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/dangling-with.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/dangling-with.smithy index b8319bb95cb..d9657200a36 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/dangling-with.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/dangling-with.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 1 near ``: Expected: '[', but found '' +// Syntax error at line 6, column 1: Expected LBRACKET('[') but found EOF | Model $version: "2" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/invalid-statement-after-structure-name.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/invalid-statement-after-structure-name.smithy index 3a6a787a3cb..acc716f8ac1 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/invalid-statement-after-structure-name.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/invalid-statement-after-structure-name.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 15 near `Baz`: Expected: '{', but found 'B' +// Syntax error at line 5, column 15: Expected LBRACE('{') but found IDENTIFIER('Baz') | Model $version: "2" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy index f038cf9b8f6..25fb65ece73 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/missing-rest-of-with-word.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 18 near ` `: Expected: 'h', but found ' ' +// Syntax error at line 5, column 15: Expected `with`, but found `wit` | Model $version: "2" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-explicit-1-0.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-explicit-1-0.smithy index 3de5b363575..f06993d509c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-explicit-1-0.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-explicit-1-0.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 19 near ` `: Mixins can only be used with Smithy version 2 or later. Attempted to use mixins with version `1.0`. +// Syntax error at line 5, column 15: Mixins can only be used with Smithy version 2 or later. Attempted to use mixins with version `1.0`. | Model $version: "1.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-implicit-1-0.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-implicit-1-0.smithy index df8a67024d4..0d68ffeceb6 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-implicit-1-0.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/mixins-usage-in-implicit-1-0.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 19 near ` `: Mixins can only be used with Smithy version 2 or later. Attempted to use mixins with version `1.0`. +// Syntax error at line 4, column 15: Mixins can only be used with Smithy version 2 or later. Attempted to use mixins with version `1.0`. | Model namespace smithy.example structure Foo with [Bar] {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-but-no-ids.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-but-no-ids.smithy index f9fd01569ba..88df3fd18d5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-but-no-ids.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-but-no-ids.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 21 near `] `: Expected a valid identifier character, but found ']' +// Syntax error at line 5, column 21: Expected IDENTIFIER but found RBRACKET(']') | Model $version: "2" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigDecimal.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigDecimal.smithy index fef40fbe38f..788d1d5a68c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigDecimal.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigDecimal.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ bigDecimal MixinBigdecimal bigDecimal MixedBigdecimal with [MixinBigdecimal] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigInteger.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigInteger.smithy index 8a3d2ac9fb8..7021232f22e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigInteger.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-bigInteger.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ bigInteger MixinBiginteger bigInteger MixedBiginteger with [MixinBiginteger] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-blob.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-blob.smithy index 915f9d2f338..452eb2d9cfe 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-blob.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-blob.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ blob MixinBlob blob MixedBlob with [MixinBlob] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-boolean.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-boolean.smithy index f27c986d456..86818b7fe3c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-boolean.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-boolean.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ boolean MixinBoolean boolean MixedBoolean with [MixinBoolean] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-byte.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-byte.smithy index ece46a413e4..9584020c449 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-byte.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-byte.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ byte MixinByte byte MixedByte with [MixinByte] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-document.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-document.smithy index 8bf58a2fa98..332ecd22132 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-document.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-document.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ document MixinDocument document MixedDocument with [MixinDocument] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-double.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-double.smithy index ef3ba335ebc..828cb8896e7 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-double.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-double.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ double MixinDouble double MixedDouble with [MixinDouble] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-float.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-float.smithy index a00e813883e..092e1874e6a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-float.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-float.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ float MixinFloat float MixedFloat with [MixinFloat] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-integer.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-integer.smithy index f653c969bc8..18532f83651 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-integer.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-integer.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ integer MixinInteger integer MixedInteger with [MixinInteger] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-list.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-list.smithy index f2ed6af4d5c..533dff8235c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-list.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-list.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-long.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-long.smithy index e2ab79e32b9..a64e87ee7d1 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-long.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-long.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ long MixinLong long MixedLong with [MixinLong] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-map.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-map.smithy index 35d1311774e..e417ed4a1f2 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-map.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-map.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-resource.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-resource.smithy index 48037ed3e5b..00b0c0ca056 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-resource.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-resource.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-service.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-service.smithy index 9fdd1087145..ee3c2a98734 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-service.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-service.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-set.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-set.smithy deleted file mode 100644 index 35d1311774e..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-set.smithy +++ /dev/null @@ -1,13 +0,0 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' -$version: "2.0" - -namespace smithy.example - -map MixedMap - with [MixinMap] {} - -@mixin -map MixinMap { - key: String - value: String -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-short.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-short.smithy index 314e7f0e5c6..48c22c97a94 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-short.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-short.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ short MixinShort short MixedShort with [MixinShort] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-string.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-string.smithy index 88df5d1ee88..c1b57e255b0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-string.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-string.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-structure.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-structure.smithy index a99c82c8996..2db3a31debb 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-structure.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-structure.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-timestamp.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-timestamp.smithy index 2a5bd835eb6..c4fc55100be 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-timestamp.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-timestamp.smithy @@ -1,4 +1,4 @@ -// Parse error at line 10, column 9 near ` `: Unexpected shape type: with +// Syntax error at line 10, column 5: Unknown shape type: with | Model $version: "2.0" namespace smithy.example @@ -8,4 +8,3 @@ timestamp MixinTimestamp timestamp MixedTimestamp with [MixinTimestamp] - \ No newline at end of file diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-union.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-union.smithy index 9aa858348e0..29a1a21e9ec 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-union.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/mixins/with-on-next-line-union.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 5 near `with`: Expected: '{', but found 'w' +// Syntax error at line 7, column 5: Expected LBRACE('{') but found IDENTIFIER('with') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-shapes.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-shapes.smithy index 206d458cfb9..3d9bc526c43 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-shapes.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-shapes.smithy @@ -1,2 +1,2 @@ -// A namespace must be defined before a use statement or shapes +// Syntax error at line 2, column 1: Expected a namespace definition but found IDENTIFIER('string') | Model string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-traits.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-traits.smithy index 8752cefc316..f2be04fb46e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-traits.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-before-traits.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 1 near `@i`: Expected a namespace definition, but found unexpected syntax +// Syntax error at line 2, column 1: Expected a namespace definition but found AT('@') | Model @invalid diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-defined-twice.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-defined-twice.smithy index c2d1e62624e..1a023578cb8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-defined-twice.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-defined-twice.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 10 near ` com`: Only a single namespace can be declared per/file +// Syntax error at line 4, column 1: Unknown shape type: namespace | Model $version: "2.0" namespace com.foo namespace com.baz diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-multiple-lines.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-multiple-lines.smithy index 5f1eb7212d5..ef5a5e43b7b 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-multiple-lines.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-multiple-lines.smithy @@ -1,4 +1,4 @@ -// Parse error at line 3, column 14 near `//`: Expected a valid identifier character, but found '/' | Model +// Syntax error at line 3, column 14: Expected IDENTIFIER but found COMMENT('// Spaces are fine. The comment is fine. But the newline is not fine.\n') | Model $version: "2.0" namespace // Spaces are fine. The comment is fine. But the newline is not fine. com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-no-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-no-newline.smithy index 75938812851..150ba5de7cb 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-no-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-no-newline.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 19 near `string`: Expected a line break | Model +// Syntax error at line 3, column 19: Expected a line break, but found IDENTIFIER('string') | Model $version: "2.0" namespace com.foo string Foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-syntax-error.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-syntax-error.smithy index f633626b2dd..a209fafb1e5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-syntax-error.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/namespace-syntax-error.smithy @@ -1,2 +1,2 @@ -// Expected a valid identifier character, but found '\n' +// Syntax error at line 2, column 30: Expected IDENTIFIER but found NEWLINE('\n') | Model namespace not.valid.trailing. diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string1.smithy index 590bb422ea2..5984fee4489 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string1.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string1.smithy @@ -1,3 +1,3 @@ -// Parse error at line 4, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 4, column 1: Expected: '"', but found '[EOF]' namespace abc.def @foo(" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string2-invalid-single-quote.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string2-invalid-single-quote.smithy index d6df3e0a36d..4fc8e26db96 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string2-invalid-single-quote.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string2-invalid-single-quote.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 6 near `'\n`: Expected a valid identifier character, but found ''' +// Syntax error at line 3, column 6: Unexpected character: ''' | Model namespace abc.def @foo(' diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string3.smithy index 854cef14559..acf84e2e599 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string3.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/namespace/unclosed-string3.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 6, column 1: Expected: '"', but found '[EOF]' namespace abc.def @foo(" hello diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/decimal-with-no-digit-after.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/decimal-with-no-digit-after.smithy index ce3e104d361..f5fa1a9fa59 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/decimal-with-no-digit-after.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/decimal-with-no-digit-after.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 9 near `)\n`: Invalid number '1.': '.' must be followed by a digit +// Syntax error at line 7, column 7: Invalid number '1.': '.' must be followed by a digit | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-no-digit.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-no-digit.smithy index a05f4d2f634..2d6d8bd4362 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-no-digit.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-no-digit.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 11 near `)\n`: Invalid number '1.0e': 'e', '+', and '-' must be followed by a digit +// Syntax error at line 7, column 7: Invalid number '1.0e': 'e', '+', and '-' must be followed by a digit | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-sign-then-no-digit.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-sign-then-no-digit.smithy index 97f051e9653..1d4d8f3493c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-sign-then-no-digit.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/exponent-with-sign-then-no-digit.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 12 near `)\n`: Invalid number '1.0e+': 'e', '+', and '-' must be followed by a digit +// Syntax error at line 7, column 7: Invalid number '1.0e+': 'e', '+', and '-' must be followed by a digit | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/leading-decimal.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/leading-decimal.smithy index 92e96064ac9..19556cdbb33 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/leading-decimal.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/leading-decimal.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 7 near `.1`: Expected a valid identifier character, but found '.' +// Syntax error at line 7, column 7: Expected one of LBRACE('{'), LBRACKET('['), TEXT_BLOCK('"""'), STRING('"'), NUMBER, IDENTIFIER; but found DOT('.') | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/negative-with-no-digit.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/negative-with-no-digit.smithy index c3ea4cdcd73..e3d54245168 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/negative-with-no-digit.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/numbers/negative-with-no-digit.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 8 near `)\n`: Invalid number '-': '-' must be followed by a digit +// Syntax error at line 7, column 7: Invalid number '-': '-' must be followed by a digit | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/input-defined-twice.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/input-defined-twice.smithy index 8f8da89df03..15868da17e8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/input-defined-twice.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/input-defined-twice.smithy @@ -1,4 +1,4 @@ -// Parse error at line 8, column 10 near `: `: Duplicate operation input property for com.foo#GetFoo +// Syntax error at line 8, column 5: Duplicate operation input property for `com.foo#GetFoo` $version: "2.0" namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/missing-ws-after-input.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/missing-ws-after-input.smithy index 4de38001f49..1f6a1f51378 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/missing-ws-after-input.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/operations/missing-ws-after-input.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 29 near `: `: Expected one or more whitespace characters +// Syntax error at line 6, column 29: Expected one or more whitespace characters, but found COLON(':') $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/absolute-shape-name.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/absolute-shape-name.smithy new file mode 100644 index 00000000000..1b3d562e8c0 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/absolute-shape-name.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 11: Expected a line break, but found DOT('.') +namespace com.foo + +string com.foo#Bam diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-id-after-type.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-id-after-type.smithy new file mode 100644 index 00000000000..968a7c33636 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-id-after-type.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 8: Expected IDENTIFIER but found LPAREN('(') | Model +namespace com.foo + +string (Nope) diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy new file mode 100644 index 00000000000..22cbce4d2b5 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/expected-shape-name-but-eof.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 7: Expected one or more spaces, but found NEWLINE('\n') | Model +namespace com.foo + +string diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy index 90ea9d09bff..16282aa804e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/invalid-newline-after-shape-type.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 10 +// Syntax error at line 5, column 10: Expected one or more spaces, but found NEWLINE('\n') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-string.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-string.smithy index 82da8cd2c39..db30e1917a5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-string.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-string.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 10 near `\n`: Unexpected shape type: stringFoo | Model +// Syntax error at line 5, column 1: Unknown shape type: stringFoo | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-structure.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-structure.smithy index 6e700617002..7fd67b1b72d 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-structure.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/missing-space-after-structure.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 13 near `{}`: Unexpected shape type: structureFoo | Model +// Syntax error at line 5, column 1: Unknown shape type: structureFoo | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax.smithy new file mode 100644 index 00000000000..c5d6dc85832 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 14: Expected LBRACE('{') but found DOT('.') | Model +namespace com.foo + +structure Not...Valid {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax2.smithy new file mode 100644 index 00000000000..c9c79e71498 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax2.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 11: Unexpected character: '%' | Model +namespace com.foo + +structure %Invalid {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax3.smithy new file mode 100644 index 00000000000..2266a2d41d5 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/shape-name-syntax3.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 12: Expected a valid identifier character, but found ' ' +namespace com.foo + +structure _ {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/statement-unknown.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/statement-unknown.smithy similarity index 74% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/statement-unknown.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/statement-unknown.smithy index 0d14b0b178f..74f222c23a6 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/statement-unknown.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/statement-unknown.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 8 near ` String`: Unexpected shape type: invalid +// Syntax error at line 7, column 1: Unknown shape type: invalid | Model // ^ The list of expected tokens goes on, but it's been trimmed here to help reduce the // number of time this test will need to be updated if/when the syntax changes. namespace foo.bar diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/syntax-error-token.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/syntax-error-token.smithy new file mode 100644 index 00000000000..045a682e1e1 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/parse-errors/syntax-error-token.smithy @@ -0,0 +1,2 @@ +// Syntax error at line 2, column 1: Expected a namespace definition but found ERROR('!') | Model +!!!!!!!!!Nope!!!!!!!!!! diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-invalid-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-invalid-member.smithy deleted file mode 100644 index 03dc2c061c2..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/set-invalid-member.smithy +++ /dev/null @@ -1,6 +0,0 @@ -// Parse error at line 5, column 3 near `foo`: Expected: '}', but found 'f' | Model -namespace com.foo - -set MySet { - foo: smithy.api#String, -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax.smithy deleted file mode 100644 index 27fb21cc45c..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 14 near `..`: Expected: '{', but found '.' -namespace com.foo - -structure Not...Valid {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax2.smithy deleted file mode 100644 index 2f91da0e3f2..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax2.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 11 near `%I`: Expected a valid identifier character, but found '%' -namespace com.foo - -structure %Invalid {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax3.smithy deleted file mode 100644 index a86ff4b1cdb..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/shape-name-syntax3.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 4, column 12 near ` `: Expected a valid identifier character, but found ' ' -namespace com.foo - -structure _ {} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-apostrophe.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-apostrophe.smithy index a109d5da605..e5fd7008236 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-apostrophe.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-apostrophe.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 20 near `)\n`: Invalid escape found in string: `\'` +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid escape found in string: `\'` | Model namespace smithy.example @documentation("\'") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-space.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-space.smithy index 33f2ed786c4..51251bc98c8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-space.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-escape-space.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 20 near `)\n`: Invalid escape found in string: `\ ` +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid escape found in string: `\ ` | Model namespace smithy.example @documentation("\ ") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape.smithy index abaebb5141e..388a7b5e04f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 21 near `)\n`: Invalid unclosed unicode escape found in string +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid unclosed unicode escape found in string | Model namespace smithy.example @documentation("\ua") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape2.smithy index d9a16a2c35e..b1f8430d615 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape2.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape2.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 22 near `)\n`: Invalid unclosed unicode escape found in string +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid unclosed unicode escape found in string | Model namespace smithy.example @documentation("\uaa") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape3.smithy index 1687cc0cfdc..717956077d7 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape3.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape3.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 23 near `)\n`: Invalid unclosed unicode escape found in string +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid unclosed unicode escape found in string | Model namespace smithy.example @documentation("\uaaa") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape4.smithy index 17eaa10baa6..0b2e9770b0e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape4.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape4.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 24 near `)\n`: Invalid unicode escape character: `t` +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid unicode escape character: `t` | Model namespace smithy.example @documentation("\uaaat") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape5.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape5.smithy index eed6deed17b..98de13ddb08 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape5.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/invalid-unicode-escape5.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 24 near `)\n`: Invalid unicode escape character: `t` +// Syntax error at line 4, column 16: Error parsing quoted string: Invalid unicode escape character: `t` | Model namespace smithy.example @documentation("\uataa") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-empty.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-empty.smithy index d48fa9479f3..af30e2392d8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-empty.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-empty.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 22 near `)\n`: Invalid text block: text block is empty +// Syntax error at line 4, column 16: Error parsing text block: Text block is empty | Model namespace smithy.example @documentation("""""") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-invalid-unicode.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-invalid-unicode.smithy index 1932e70b135..6dc5b785510 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-invalid-unicode.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-invalid-unicode.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 11 near `)\n`: Invalid unclosed unicode escape found in string +// Syntax error at line 4, column 16: Error parsing text block: Invalid unclosed unicode escape found in string | Model namespace smithy.example @documentation(""" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-missing-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-missing-newline.smithy index 7e2cba8e0da..a1787df9821 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-missing-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/text-block-missing-newline.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 24 near `)\n`: Invalid text block: text block must start with a new line (LF) +// Syntax error at line 4, column 16: Error parsing text block: Text block must start with a new line | Model namespace smithy.example @documentation("""Hi""") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string.smithy index 5ee51f473a2..3cdbd3bb18c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 6, column 1: Expected: '"', but found '[EOF]' namespace smithy.example @documentation(") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string2.smithy index 7c4268c07e7..52e98bcea7f 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string2.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string2.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 6, column 1: Expected: '"', but found '[EOF]' namespace smithy.example @documentation("\") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string3.smithy index 1f21e758b70..349a2600ed7 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string3.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-string3.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 6, column 1: Expected: '"', but found '[EOF]' namespace smithy.example @documentation("\\\") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-text-block.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-text-block.smithy index 3714e936b9b..49996690c9e 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-text-block.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/strings/unclosed-text-block.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 1 near `[EOF]`: Expected: '"', but found '[EOF]' +// Syntax error at line 7, column 1: Expected: '"', but found '[EOF]' namespace smithy.example @documentation(""" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/syntax-error-token.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/syntax-error-token.smithy deleted file mode 100644 index a6a212f460e..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/syntax-error-token.smithy +++ /dev/null @@ -1,2 +0,0 @@ -// Parse error at line 2, column 1 near `!!`: Expected a namespace definition, but found unexpected syntax -!!!!!!!!!Nope!!!!!!!!!! diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-name-syntax.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-name-syntax.smithy deleted file mode 100644 index 523e519b1ba..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-name-syntax.smithy +++ /dev/null @@ -1,5 +0,0 @@ -// Parse error at line 4, column 10 near `#f`: Expected a valid identifier character, but found '#' -namespace foo.bar - -@invalid##foo..bar -string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-on-eof.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-on-eof.smithy deleted file mode 100644 index 0c70a29e1d6..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-on-eof.smithy +++ /dev/null @@ -1,4 +0,0 @@ -// Parse error at line 5, column 1 near `[EOF]`: Expected a valid identifier character, but found '[EOF]' -namespace com.foo - -@deprecated diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-trailing.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-trailing.smithy deleted file mode 100644 index 03fe954efdc..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/trait-trailing.smithy +++ /dev/null @@ -1,8 +0,0 @@ -// Parse error at line 8, column 1 near `}\n`: Expected a valid identifier character, but found '}' -namespace com.foo - -structure Foo { - baz: String, - - @deprecated -} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-apply.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-apply.smithy deleted file mode 100644 index 5a3960584c3..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-apply.smithy +++ /dev/null @@ -1,5 +0,0 @@ -// Parse error at line 5, column 6 near ` Foo`: Unexpected shape type: apply -namespace com.foo - -@deprecated -apply Foo @deprecated diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-metadata.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-metadata.smithy deleted file mode 100644 index 29bc32e2336..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-metadata.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Expected a namespace definition, but found unexpected syntax -@deprecated -metadata foo = "bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-namespace.smithy deleted file mode 100644 index c2672eff96b..00000000000 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/trait-targets/traits-on-namespace.smithy +++ /dev/null @@ -1,3 +0,0 @@ -// Parse error at line 2, column 1 near `@d`: Expected a namespace definition, but found unexpected syntax -@deprecated -namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key1.smithy new file mode 100644 index 00000000000..2b3a5105de4 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key1.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 3, column 9: Expected RPAREN(')') but found COLON(':') | Model +namespace com.foo +@foo(100: true) +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key2.smithy new file mode 100644 index 00000000000..61f730c26d2 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key2.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 3, column 8: Expected RPAREN(')') but found COLON(':') | Model +namespace com.foo +@foo({}: true) +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key3.smithy new file mode 100644 index 00000000000..4f960f34a80 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-invalid-object-key3.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 3, column 17: Expected one of IDENTIFIER, STRING('"'); but found LBRACE('{') | Model +namespace com.foo +@foo(foo: true, {}: true) +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list1.smithy new file mode 100644 index 00000000000..ca03d073f7f --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list1.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 5, column 1: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found EOF | Model +namespace com.foo +@tags(["foo", "bar" +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list2.smithy new file mode 100644 index 00000000000..3c29b82cac4 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list2.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 5, column 1: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found EOF | Model +namespace com.foo +@tags(["foo", "bar", +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list3.smithy new file mode 100644 index 00000000000..5b8468d8478 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list3.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 5, column 1: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found EOF | Model +namespace com.foo +@tags([ +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list4.smithy new file mode 100644 index 00000000000..3fffd8a6d07 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-list4.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 4, column 1: Expected one of STRING('"'), TEXT_BLOCK('"""'), NUMBER, IDENTIFIER, LBRACE('{'), LBRACKET('['); but found EOF | Model +namespace com.foo +@tags([[[[[[[[[[[[ diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object1.smithy new file mode 100644 index 00000000000..87564f75948 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object1.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 8: Expected RPAREN(')') but found IDENTIFIER('MyString') | Model +namespace com.foo +@foo( +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object2.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object2.smithy new file mode 100644 index 00000000000..03a18d70631 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object2.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 1: Expected RPAREN(')') but found IDENTIFIER('string') | Model +namespace com.foo +@foo("bar" +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object3.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object3.smithy new file mode 100644 index 00000000000..1646fc912f8 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object3.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 5, column 1: Expected COLON(':') but found EOF | Model +namespace com.foo +@foo(bar: +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object4.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object4.smithy new file mode 100644 index 00000000000..27c459eeabd --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object4.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 8: Expected COLON(':') but found IDENTIFIER('MyString') | Model +namespace com.foo +@foo(bar: "baz" +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object5.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object5.smithy new file mode 100644 index 00000000000..d875aafd681 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object5.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 8: Expected COLON(':') but found IDENTIFIER('MyString') | Model +namespace com.foo +@foo(bar: "baz", +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object6.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object6.smithy new file mode 100644 index 00000000000..5c5cf216f2b --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/annotation-unclosed-object6.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 4, column 1: Expected COLON(':') but found IDENTIFIER('string') | Model +namespace com.foo +@foo(bar: "baz", "bam" +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-shape-id-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-shape-id-key.smithy similarity index 51% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-shape-id-key.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-shape-id-key.smithy index 38d12baa73a..6d3ac8643c0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-shape-id-key.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-shape-id-key.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 30 near `.e`: Expected: ')', but found '.' +// Syntax error at line 4, column 30: Expected RPAREN(')') but found DOT('.') | Model namespace smithy.example @externalDocumentation(smithy.example#foo: "bar") diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-textblock-key.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-textblock-key.smithy similarity index 51% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-textblock-key.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-textblock-key.smithy index a37825c7a79..57d3fc71c22 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-trait-textblock-key.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/invalid-trait-textblock-key.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 11 near `: `: Expected: ')', but found ':' +// Syntax error at line 6, column 11: Expected RPAREN(')') but found COLON(':') | Model namespace smithy.example @externalDocumentation( diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-name-syntax.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-name-syntax.smithy new file mode 100644 index 00000000000..edbf0404fea --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-name-syntax.smithy @@ -0,0 +1,5 @@ +// Syntax error at line 4, column 10: Expected IDENTIFIER but found POUND('#') | Model +namespace foo.bar + +@invalid##foo..bar +string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-on-eof.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-on-eof.smithy new file mode 100644 index 00000000000..45dfc18f1d9 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-on-eof.smithy @@ -0,0 +1,4 @@ +// Syntax error at line 5, column 1: Expected IDENTIFIER but found EOF | Model +namespace com.foo + +@deprecated diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-trailing.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-trailing.smithy new file mode 100644 index 00000000000..39532fdb30f --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/trait-trailing.smithy @@ -0,0 +1,8 @@ +// Syntax error at line 8, column 1: Expected IDENTIFIER but found RBRACE('}') | Model +namespace com.foo + +structure Foo { + baz: String, + + @deprecated +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-apply.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-apply.smithy new file mode 100644 index 00000000000..5e040c3e505 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-apply.smithy @@ -0,0 +1,5 @@ +// Syntax error at line 5, column 1: Unknown shape type: apply | Model +namespace com.foo + +@deprecated +apply Foo @deprecated diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-metadata.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-metadata.smithy new file mode 100644 index 00000000000..a1fb58572cc --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-metadata.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 2, column 1: Expected a namespace definition but found AT('@') | Model +@deprecated +metadata foo = "bar" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-namespace.smithy new file mode 100644 index 00000000000..5e3cb4997e0 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/traits/traits-on-namespace.smithy @@ -0,0 +1,3 @@ +// Syntax error at line 2, column 1: Expected a namespace definition but found AT('@') | Model +@deprecated +namespace com.foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-shape.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-shape.smithy index 7162f6c8a13..d4e79b4faf8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-shape.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-shape.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 4 near ` smithy`: A use statement must come before any shape definition +// Syntax error at line 6, column 1: Unknown shape type: use | Model namespace smithy.example string MyString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-traitdef.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-traitdef.smithy index 839ffb813ae..a2e41afa87b 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-traitdef.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-after-traitdef.smithy @@ -1,4 +1,4 @@ -// Parse error at line 7, column 4 near ` smithy`: A use statement must come before any shape definition +// Syntax error at line 7, column 1: Unknown shape type: use | Model namespace smithy.example @trait diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-empty-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-empty-namespace.smithy index 0c3b86e4f87..284e25281a7 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-empty-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-empty-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 5 near `#\n`: Expected a valid identifier character, but found '#' +// Syntax error at line 6, column 5: Expected IDENTIFIER but found POUND('#') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-invalid-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-invalid-namespace.smithy index ebf32caa735..a75f7deb762 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-invalid-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-invalid-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 6 near `$#`: Expected: '#', but found '$' +// Syntax error at line 6, column 6: Expected POUND('#') but found DOLLAR('$') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-after-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-after-namespace.smithy index fa4b489b5e5..37bba86862d 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-after-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-after-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 13 near `\n[EOF]`: Expected a valid identifier character, but found '\n' +// Syntax error at line 6, column 13: Expected IDENTIFIER but found NEWLINE('\n') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-namespace.smithy index eabf9f622e2..ffd21d59ca0 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 15 near `\n[EOF]`: Expected: '#', but found '\n' +// Syntax error at line 6, column 15: Expected POUND('#') but found NEWLINE('\n') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-newline.smithy index 6edf7a7f93d..dd5c0adfee4 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-newline.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 17 near `use`: Expected a line break | Model +// Syntax error at line 5, column 17: Expected a line break, but found IDENTIFIER('use') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-space.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-space.smithy index 53c0f5f292b..0c1a64f3b11 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-space.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-missing-space.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 4 near `com` +// Syntax error at line 5, column 1: Unknown shape type: usecom | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines-comment.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines-comment.smithy index 1a90d9ae29b..be8364318b5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines-comment.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines-comment.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 5 near `//`: Expected a valid identifier character, but found '/' | Model +// Syntax error at line 5, column 5: Expected IDENTIFIER but found COMMENT('// Invalid\n') | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines.smithy index 1376f723f9d..6e6e37d8bbd 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-multiple-lines.smithy @@ -1,4 +1,4 @@ -// Parse error at line 5, column 4 +// Syntax error at line 5, column 4 $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-must-come-after-namespace.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-must-come-after-namespace.smithy index 89c75b093f7..0d7df7d7140 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-must-come-after-namespace.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-must-come-after-namespace.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 1 near `use`: A namespace must be defined before a use statement or shapes +// Syntax error at line 4, column 1: Expected a namespace definition but found IDENTIFIER('use') | Model $version: "2.0" use smithy.example#Foo diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-statement-invalid-id.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-statement-invalid-id.smithy index 21682abc935..fe617bab86a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-statement-invalid-id.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-statement-invalid-id.smithy @@ -1,4 +1,4 @@ -// Parse error at line 6, column 16 near `!\n`: Expected a valid identifier character, but found '!' +// Syntax error at line 6, column 16: Unexpected character: '!' | Model $version: "2.0" namespace smithy.example diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-with-member.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-with-member.smithy new file mode 100644 index 00000000000..df8b3f3d2a7 --- /dev/null +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/use/use-with-member.smithy @@ -0,0 +1,5 @@ +// Use statements cannot use members +$version: "2.0" +namespace smithy.example + +use com.foo#Bar$baz diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/idl-int-enums-in-v1.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/idl-int-enums-in-v1.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/idl-int-enums-in-v1.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/idl-int-enums-in-v1.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-version-type.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/invalid-version-type.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/invalid-version-type.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/invalid-version-type.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/properties-v2-only.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/properties-v2-only.json similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/properties-v2-only.json rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/properties-v2-only.json diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/properties-v2-only.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/properties-v2-only.smithy similarity index 100% rename from smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/properties-v2-only.smithy rename to smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/properties-v2-only.smithy diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/unsupported-version.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/unsupported-version.smithy index b08be376642..235a664364a 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/unsupported-version.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/unsupported-version.smithy @@ -1,2 +1,2 @@ -// Parse error at line 2, column 16 near `\n[EOF]`: Unsupported Smithy version number: 999 +// Syntax error at line 2, column 16: Unsupported Smithy version number: 999 $version: "999" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/version-set-more-than-once.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/version-set-more-than-once.smithy index 9203d46aeba..4b819ca9aac 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/version-set-more-than-once.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/version/version-set-more-than-once.smithy @@ -1,3 +1,3 @@ -// Parse error at line 3, column 11 near `"2`: Duplicate control statement `version` +// Syntax error at line 3, column 11: Duplicate control statement `version` $version: "2" $version: "2" diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.json index 3c6a38a00c2..0cfe8bcb8e5 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.json +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.json @@ -4,6 +4,7 @@ "smithy.example#Features": { "type": "string", "traits": { + "smithy.api#documentation": "Hello", "smithy.api#enum": [ { "name": "X", diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.smithy index cd63ce414c3..c1894e38c97 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments-ignored-after-shape.smithy @@ -6,6 +6,7 @@ $version: "2.0" namespace smithy.example +/// Hello @enum([ /// Invalid! { name: "X", value: "X"} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json index 5e21ea10d7c..2c94286f7b8 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json @@ -1,5 +1,5 @@ { - "smithy": "1.0", + "smithy": "2.0", "shapes": { "smithy.example#NotDocumented": { "type": "string" @@ -16,7 +16,8 @@ "foo": { "target": "smithy.api#String", "traits": { - "smithy.api#documentation": "Docs on member!" + "smithy.api#documentation": "Docs on member!", + "smithy.api#default": "hi" } }, "baz": { diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy index 0c21f8518b4..4c930f39475 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy @@ -14,7 +14,7 @@ string MyString /// Structure structure MyStructure { /// Docs on member! - foo: String, + foo: String = "hi", /// Docs on another member! @required diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.json b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.json index 6aa6195df64..e33035572e2 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.json +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.json @@ -18,6 +18,9 @@ "smithy.api#enumValue": "there" } } + }, + "traits": { + "smithy.api#documentation": "Hello!" } } } diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.smithy index 6b64079681f..e6e990fcc5c 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/enums/enum-docs.smithy @@ -1,6 +1,7 @@ $version: "2.0" namespace smithy.example +/// Hello! enum Foo { /// Hello1 BAR = "hi" diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java index f4925e9d500..abce9b502d6 100644 --- a/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java @@ -91,7 +91,7 @@ private interface Operation { void apply(Sink sink, AbstractCodeWriter writer) throws IOException; // Writes literal segments of the input string. - static Operation stringSlice(String source, int start, int end) { + static Operation stringSlice(CharSequence source, int start, int end) { return (sink, writer) -> Sink.writeString(sink, source, start, end); } @@ -432,7 +432,7 @@ private void parseArgument(int pendingTextStart) { } else { // Output any pending captured text before the output of the expression. if (pendingTextStart > -1) { - pushOperation(Operation.stringSlice(parser.expression(), pendingTextStart, parser.position() - 1)); + pushOperation(Operation.stringSlice(parser.input(), pendingTextStart, parser.position() - 1)); } pushOperation(parseNormalArgument()); } @@ -456,7 +456,7 @@ private void parseBracedArgument(int pendingTextStart) { parser.skip(); if (pendingTextStart > -1) { while (startPosition > pendingTextStart - && Character.isWhitespace(parser.expression().charAt(startPosition - 1))) { + && Character.isWhitespace(parser.input().charAt(startPosition - 1))) { startPosition--; } } @@ -474,7 +474,7 @@ private void parseBracedArgument(int pendingTextStart) { // Output any pending captured text before the output of the expression. if (pendingTextStart > -1) { - pushOperation(Operation.stringSlice(parser.expression(), pendingTextStart, startPosition)); + pushOperation(Operation.stringSlice(parser.input(), pendingTextStart, startPosition)); } Operation operation = parseNormalArgument(); @@ -583,16 +583,16 @@ private void handleConditionalOnLine(int pendingTextStart, int startPosition, in if (parser.peek() != '\r' && parser.peek() != '\n' && parser.peek() != Character.MIN_VALUE) { // If the expression is not followed directly by a newline, then all leading text. if (pendingTextStart > -1) { - pushOperation(Operation.stringSlice(parser.expression(), pendingTextStart, startPosition)); + pushOperation(Operation.stringSlice(parser.input(), pendingTextStart, startPosition)); } } else if (isAllLeadingWhitespaceOnLine(startPosition, startColumn)) { if (pendingTextStart > -1) { - pushOperation(Operation.stringSlice(parser.expression(), pendingTextStart, + pushOperation(Operation.stringSlice(parser.input(), pendingTextStart, startPosition - startColumn)); } parser.skip(); } else if (pendingTextStart > -1) { - pushOperation(Operation.stringSlice(parser.expression(), pendingTextStart, startPosition)); + pushOperation(Operation.stringSlice(parser.input(), pendingTextStart, startPosition)); } } diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java index 4a107f23c18..28215a91295 100644 --- a/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/MediaType.java @@ -55,7 +55,7 @@ private MediaType(String value, String type, String subtype, Map public static MediaType from(String value) { Parser parser = new Parser(value); parser.parse(); - return new MediaType(parser.expression(), parser.type, parser.subtype, parser.parameters); + return new MediaType(parser.input().toString(), parser.type, parser.subtype, parser.parameters); } /** diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java index fcdefd02f52..a6da9af9140 100644 --- a/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java @@ -15,22 +15,24 @@ package software.amazon.smithy.utils; +import java.nio.CharBuffer; import java.util.Objects; import java.util.function.Predicate; /** * A simple expression parser that can be extended to implement parsers - * for small domain specific languages. + * for small domain-specific languages. * - *

This parser consumes characters of an in-memory string while tracking + *

This parser internally consumes characters of a {@link CharSequence} while tracking * the current 0-based position, 1-based line, and 1-based column. * Expectations can be made on the parser to require specific characters, * and when those expectations are not met, a syntax exception is thrown. */ public class SimpleParser { - private final String expression; - private final int length; + public static final char EOF = Character.MIN_VALUE; + + private final CharSequence input; private final int maxNestingLevel; private int position = 0; private int line = 1; @@ -42,7 +44,7 @@ public class SimpleParser { * * @param expression Expression to parser. */ - public SimpleParser(String expression) { + public SimpleParser(CharSequence expression) { this(expression, 0); } @@ -53,12 +55,11 @@ public SimpleParser(String expression) { * {@code maxParsingLevel} to 0 disables the enforcement of a * maximum parsing level. * - * @param expression Expression to parse that must not be null. + * @param input Input to parse that must not be null. * @param maxNestingLevel The maximum allowed nesting level of the parser. */ - public SimpleParser(String expression, int maxNestingLevel) { - this.expression = Objects.requireNonNull(expression, "expression must not be null"); - this.length = expression.length(); + public SimpleParser(CharSequence input, int maxNestingLevel) { + this.input = Objects.requireNonNull(input, "expression must not be null"); if (maxNestingLevel < 0) { throw new IllegalArgumentException("maxNestingLevel must be >= 0"); @@ -68,12 +69,17 @@ public SimpleParser(String expression, int maxNestingLevel) { } /** - * Gets the expression being parsed. + * Gets the input being parsed as a CharSequence. * - * @return Returns the expression being parsed. + * @return Returns the input being parsed. */ + public final CharSequence input() { + return input; + } + + @Deprecated public final String expression() { - return expression; + return input.toString(); } /** @@ -115,7 +121,7 @@ public final void rewind(int position, int line, int column) { * @return Returns true if the parser has reached the end. */ public final boolean eof() { - return position >= length; + return position >= input.length(); } /** @@ -140,11 +146,11 @@ public final char peek() { */ public final char peek(int offset) { int target = position + offset; - if (target >= length || target < 0) { - return Character.MIN_VALUE; + if (target >= input.length() || target < 0) { + return EOF; } - return expression.charAt(target); + return input.charAt(target); } /** @@ -170,7 +176,7 @@ public final char expect(char token) { */ public final String peekSingleCharForMessage() { char peek = peek(); - return peek == Character.MIN_VALUE ? "[EOF]" : String.valueOf(peek); + return peek == EOF ? "[EOF]" : String.valueOf(peek); } /** @@ -205,14 +211,14 @@ public final char expect(char... tokens) { * @return Returns the created syntax error. */ public RuntimeException syntax(String message) { - return new RuntimeException("Syntax error at line " + line() + " column " + column() + ": " + message); + return new RuntimeException("Syntax error at line " + line() + ", column " + column() + ": " + message); } /** * Skip 0 or more whitespace characters (that is, ' ', '\t', '\r', and '\n'). */ public void ws() { - while (!eof() && isWhitespace(peek())) { + while (isWhitespace(peek())) { skip(); } } @@ -225,7 +231,7 @@ private boolean isWhitespace(char c) { * Skip 0 or more spaces (that is, ' ' and '\t'). */ public void sp() { - while (!eof() && isSpace(peek())) { + while (isSpace(peek())) { skip(); } } @@ -264,7 +270,7 @@ public void skip() { return; } - switch (expression.charAt(position)) { + switch (input.charAt(position)) { case '\r': if (peek(1) == '\n') { position++; @@ -301,15 +307,37 @@ public void consumeRemainingCharactersOnLine() { } /** - * Gets a slice of the expression starting from the given 0-based - * character position, read all the way through to the current - * position of the parser. + * Copies a slice of the expression into a string, starting from the given 0-based character position, + * read all the way through to the current position of the parser. * * @param start Position to slice from, ending at the current position. - * @return Returns the slice of the expression from {@code start} to {@link #position}. + * @return Returns the copied slice of the expression from {@code start} to {@link #position}. */ public final String sliceFrom(int start) { - return expression().substring(start, position); + return input.subSequence(start, position).toString(); + } + + /** + * Gets a zero-copy slice of the expression starting from the given 0-based character position, read all the + * way through to the current position of the parser. + * + * @param start Position to slice from, ending at the current position. + * @return Returns the zero-copy slice of the expression from {@code start} to {@link #position}. + */ + public final CharSequence borrowSliceFrom(int start) { + return CharBuffer.wrap(input, start, position); + } + + /** + * Gets a zero-copy slice of the expression starting from the given 0-based character position, read all the + * way through to the current position of the parser minus {@code removeRight}. + * + * @param start Position to slice from, ending at the current position. + * @param removeRight Number of characters to omit before the current position. + * @return Returns the zero-copy slice of the expression from {@code start} to {@link #position}. + */ + public final CharSequence borrowSliceFrom(int start, int removeRight) { + return CharBuffer.wrap(input, start, position - removeRight); } /** diff --git a/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java b/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java index c474b0d30c6..1b9c0bd5d19 100644 --- a/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java +++ b/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java @@ -14,7 +14,7 @@ public class SimpleParserTest { public void simpleParserBasics() { SimpleParser p = new SimpleParser("foo", 10); - assertThat(p.expression(), equalTo("foo")); + assertThat(p.input().toString(), equalTo("foo")); assertThat(p.line(), equalTo(1)); assertThat(p.column(), equalTo(1)); assertThat(p.position(), equalTo(0)); @@ -31,7 +31,7 @@ public void expectThrowsWhenNotMatching() { new SimpleParser("foo").expect('!'); }); - assertThat(e.getMessage(), equalTo("Syntax error at line 1 column 1: Expected: '!', but found 'f'")); + assertThat(e.getMessage(), equalTo("Syntax error at line 1, column 1: Expected: '!', but found 'f'")); } @Test @@ -41,7 +41,7 @@ public void expectThrowsWhenNotMatchingOneOfMany() { }); assertThat(e.getMessage(), equalTo( - "Syntax error at line 1 column 1: Found 'f', but expected one of the following tokens: '!' '?'")); + "Syntax error at line 1, column 1: Found 'f', but expected one of the following tokens: '!' '?'")); } @Test @@ -51,7 +51,7 @@ public void expectThrowsWhenNotMatchingOneOfManyAndDueToEof() { }); assertThat(e.getMessage(), equalTo( - "Syntax error at line 1 column 1: Found '[EOF]', but expected one of the following tokens: '!' '?'")); + "Syntax error at line 1, column 1: Found '[EOF]', but expected one of the following tokens: '!' '?'")); } @Test