Update Smithy to 1.23.1 to support IDL 2 (#1623)

CHANGELOG.next.toml

@@ -1,4 +1,5 @@
 # Example changelog entries
+# 
 # [[aws-sdk-rust]]
 # message = "Fix typos in module documentation for generated crates"
 # references = ["smithy-rs#920"]
@@ -47,6 +48,36 @@ references = ["smithy-rs#1598"]
 meta = { "breaking" = true, "tada" = true, "bug" = false, "target" = "client" }
 author = "Velfi"
 
+[[smithy-rs]]
+message = "Update Smithy dependency to 1.23.1. Models using version 2.0 of the IDL are now supported."
+references = ["smithy-rs#1623"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "ogudavid"
+
+[[smithy-rs]]
+message = "Fix detecting sensitive members through their target shape having the @sensitive trait applied."
+references = ["smithy-rs#1623"]
+meta = { "breaking" = false, "tada" = false, "bug" = true }
+author = "ogudavid"
+
+[[smithy-rs]]
+message = "Remove @sensitive trait tests which applied trait to member. The ability to mark members with @sensitive was removed in Smithy 1.22."
+references = ["smithy-rs#1623"]
+meta = { "breaking" = true, "tada" = false, "bug" = false }
+author = "ogudavid"
+
+[[smithy-rs]]
+message = "Fix SetShape matching needing to occur before ListShape since it is now a subclass. Sets were deprecated in Smithy 1.22."
+references = ["smithy-rs#1623"]
+meta = { "breaking" = false, "tada" = false, "bug" = false }
+author = "ogudavid"
+
+[[smithy-rs]]
+message = "Fix Union shape test data having an invalid empty union. Break fixed from Smithy 1.21 to Smithy 1.22."
+references = ["smithy-rs#1623"]
+meta = { "breaking" = false, "tada" = false, "bug" = false }
+author = "ogudavid"
+
 [[smithy-rs]]
 message = "Add codegen version to generated package metadata"
 references = ["smithy-rs#1612"]

codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerHttpSensitivityGenerator.kt

@@ -132,7 +132,7 @@ class ServerHttpSensitivityGenerator(
                     it.direction == RelationshipDirection.DIRECTED
                 }
                 .filter {
-                    it.hasTrait<SensitiveTrait>()
+                    isDirectedRelationshipSensitive<SensitiveTrait>(it)
                 }.mapNotNull {
                     it as? MemberShape
                 }
@@ -294,7 +294,7 @@ class ServerHttpSensitivityGenerator(
                 it.direction == RelationshipDirection.DIRECTED && !it.shape.hasTrait<A>()
             }
             .filter {
-                it.hasTrait<A>()
+                isDirectedRelationshipSensitive<A>(it)
             }
             .flatMap {
                 Walker(model)
@@ -303,13 +303,20 @@ class ServerHttpSensitivityGenerator(
                         it.direction == RelationshipDirection.DIRECTED
                     }
                     .filter {
-                        it.hasTrait<B>()
+                        isDirectedRelationshipSensitive<B>(it)
                     }.mapNotNull {
                         it as? MemberShape
                     }
             }
     }
 
+    internal inline fun <reified A : Trait> isDirectedRelationshipSensitive(partnerShape: Shape): Boolean {
+        return partnerShape.hasTrait<A>() || (
+            partnerShape.asMemberShape().isPresent() &&
+                model.expectShape(partnerShape.asMemberShape().get().getTarget()).hasTrait<A>()
+            )
+    }
+
     // Find member shapes with trait `T` contained in a shape enjoying `SensitiveTrait`.
     // [trait|sensitive] ~> [trait|T]
     internal inline fun <reified T : Trait> findSensitiveBound(rootShape: Shape): List<MemberShape> {

codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocolTestGenerator.kt

@@ -658,7 +658,6 @@ class ServerProtocolTestGenerator(
             FailingTest(RestJson, "RestJsonHttpWithEmptyStructurePayload", TestType.Request),
             FailingTest(RestJson, "RestJsonHttpResponseCodeDefaultsToModeledCode", TestType.Response),
 
-            FailingTest(RestJson, "RestJsonWithPayloadExpectsImpliedAccept", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonBodyMalformedBlobInvalidBase64_case1", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonBodyMalformedBlobInvalidBase64_case2", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonWithBodyExpectsApplicationJsonContentType", TestType.MalformedRequest),
@@ -673,8 +672,6 @@ class ServerProtocolTestGenerator(
             FailingTest(RestJson, "RestJsonBodyTimestampDefaultRejectsMalformedEpochSeconds_case5", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonBodyTimestampDefaultRejectsMalformedEpochSeconds_case7", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonBodyTimestampDefaultRejectsMalformedEpochSeconds_case9", TestType.MalformedRequest),
-            FailingTest(RestJson, "RestJsonPathTimestampDefaultRejectsDifferent8601Formats_case13", TestType.MalformedRequest),
-            FailingTest(RestJson, "RestJsonQueryTimestampDefaultRejectsDifferent8601Formats_case13", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonMalformedUnionNoFieldsSet", TestType.MalformedRequest),
             FailingTest(RestJson, "RestJsonMalformedSetDuplicateBlobs", TestType.MalformedRequest),
 

codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/ServerHttpSensitivityGeneratorTest.kt

@@ -65,36 +65,6 @@ class ServerHttpSensitivityGeneratorTest {
         assertEquals(members, listOf("code"))
     }
 
-    @Test
-    fun `find inner sensitive`() {
-        val model = """
-            namespace test
-
-            operation Secret {
-                input: Input,
-            }
-
-            structure Input {
-                @required
-                @sensitive
-                @httpHeader("header-a")
-                headerA: String,
-
-                @required
-                @httpHeader("header-b")
-                headerB: String
-            }
-        """.asSmithyModel()
-
-        val operation = model.operationShapes.toList()[0]
-        val generator = ServerHttpSensitivityGenerator(model, operation, TestRuntimeConfig)
-
-        val inputShape = operation.inputShape(model)
-        val members: List<String> = generator.findSensitiveBound<HttpHeaderTrait>(inputShape).map(MemberShape::getMemberName)
-
-        assertEquals(members, listOf("headerA"))
-    }
-
     @Test
     fun `find nested sensitive`() {
         val model = """
@@ -142,18 +112,9 @@ class ServerHttpSensitivityGeneratorTest {
                 @httpQuery("query_a")
                 queryA: String,
 
-                @sensitive
-                nestedA: NestedA,
-
                 nestedB: NestedB
             }
 
-            structure NestedA {
-                @required
-                @httpQuery("query_b")
-                queryB: String
-            }
-
             @sensitive
             structure NestedB {
                 @required
@@ -167,7 +128,7 @@ class ServerHttpSensitivityGeneratorTest {
         val input = generator.input()!!
         val querySensitivity = generator.findQuerySensitivity(input)
         assertEquals(querySensitivity.allKeysSensitive, false)
-        assertEquals((querySensitivity as ServerHttpSensitivityGenerator.QuerySensitivity.NotSensitiveMapValue).queryKeys, listOf("query_c", "query_b"))
+        assertEquals((querySensitivity as ServerHttpSensitivityGenerator.QuerySensitivity.NotSensitiveMapValue).queryKeys, listOf("query_c"))
 
         val testProject = TestWorkspace.testProject(serverTestSymbolProvider(model))
         testProject.lib { writer ->
@@ -178,7 +139,6 @@ class ServerHttpSensitivityGeneratorTest {
                 rustTemplate(
                     """
                     assert_eq!(closure("query_a"), #{SmithyHttpServer}::logging::sensitivity::uri::QueryMarker { key: false, value: false });
-                    assert_eq!(closure("query_b"), #{SmithyHttpServer}::logging::sensitivity::uri::QueryMarker { key: false, value: true });
                     assert_eq!(closure("query_c"), #{SmithyHttpServer}::logging::sensitivity::uri::QueryMarker { key: false, value: true });
                     """,
                     *codegenScope,
@@ -250,9 +210,11 @@ class ServerHttpSensitivityGeneratorTest {
                 queryMap: QueryMap,
             }
 
+            @sensitive
+            string SensitiveKey
+
             map QueryMap {
-                @sensitive
-                key: String,
+                key: SensitiveKey,
                 value: String
             }
 
@@ -297,10 +259,12 @@ class ServerHttpSensitivityGeneratorTest {
                 queryMap: QueryMap,
             }
 
+            @sensitive
+            string SensitiveValue
+
             map QueryMap {
                 key: String,
-                @sensitive
-                value: String
+                value: SensitiveValue
             }
 
         """.asSmithyModel()
@@ -343,18 +307,9 @@ class ServerHttpSensitivityGeneratorTest {
                 @httpHeader("header-a")
                 headerA: String,
 
-                @sensitive
-                nestedA: NestedA,
-
                 nestedB: NestedB
             }
 
-            structure NestedA {
-                @required
-                @httpHeader("header-b")
-                headerB: String
-            }
-
             @sensitive
             structure NestedB {
                 @required
@@ -367,7 +322,7 @@ class ServerHttpSensitivityGeneratorTest {
 
         val inputShape = operation.inputShape(model)
         val headerData = generator.findHeaderSensitivity(inputShape)
-        assertEquals(headerData.headerKeys, listOf("header-c", "header-b"))
+        assertEquals(headerData.headerKeys, listOf("header-c"))
         assertEquals((headerData as ServerHttpSensitivityGenerator.HeaderSensitivity.NotSensitiveMapValue).prefixHeader, null)
 
         val testProject = TestWorkspace.testProject(serverTestSymbolProvider(model))
@@ -380,8 +335,6 @@ class ServerHttpSensitivityGeneratorTest {
                     """
                     let name = #{Http}::header::HeaderName::from_static("header-a");
                     assert_eq!(closure(&name), #{SmithyHttpServer}::logging::sensitivity::headers::HeaderMarker { value: false, key_suffix: None });
-                    let name = #{Http}::header::HeaderName::from_static("header-b");
-                    assert_eq!(closure(&name), #{SmithyHttpServer}::logging::sensitivity::headers::HeaderMarker { value: true, key_suffix: None });
                     let name = #{Http}::header::HeaderName::from_static("header-c");
                     assert_eq!(closure(&name), #{SmithyHttpServer}::logging::sensitivity::headers::HeaderMarker { value: true, key_suffix: None });
                     """,
@@ -458,9 +411,10 @@ class ServerHttpSensitivityGeneratorTest {
                 prefix_map: PrefixMap,
             }
 
+            @sensitive
+            string SensitiveKey
             map PrefixMap {
-                @sensitive
-                key: String,
+                key: SensitiveKey,
                 value: String
             }
 
@@ -511,10 +465,12 @@ class ServerHttpSensitivityGeneratorTest {
                 prefix_map: PrefixMap,
             }
 
+            @sensitive
+            string SensitiveValue
+
             map PrefixMap {
                 key: String,
-                @sensitive
-                value: String
+                value: SensitiveValue
             }
 
         """.asSmithyModel()
@@ -560,15 +516,16 @@ class ServerHttpSensitivityGeneratorTest {
                 input: Input,
             }
 
+            @sensitive
+            string SensitiveString
+
             structure Input {
                 @required
-                @sensitive
                 @httpLabel
-                labelA: String,
+                labelA: SensitiveString,
                 @required
                 @httpLabel
-                @sensitive
-                labelB: String,
+                labelB: SensitiveString,
             }
         """.asSmithyModel()
         val operation = model.operationShapes.toList()[0]

codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/EventStreamTestTools.kt

@@ -98,7 +98,7 @@ private fun fillInBaseModel(
         MessageWithNoHeaderPayloadTraits: MessageWithNoHeaderPayloadTraits,
         SomeError: SomeError,
     }
-    structure TestStreamInputOutput { @required value: TestStream }
+    structure TestStreamInputOutput { @httpPayload @required value: TestStream }
     operation TestStreamOp {
         input: TestStreamInputOutput,
         output: TestStreamInputOutput,

codegen-test/model/rest-xml-extras.smithy

@@ -1,4 +1,4 @@
-$version: "1.0"
+$version: "2.0"
 namespace aws.protocoltests.restxml
 
 use aws.protocols#restXml
@@ -43,22 +43,20 @@ structure CreateFooInput {}
     method: "POST",
    body: """
         <PrimitiveIntDocument>
-            <requiredValue>0</requiredValue>
         </PrimitiveIntDocument>
     """,
     bodyMediaType: "application/xml",
-    params: {}
+    params: { }
 }, {
-       id: "RestXmlSerPrimitiveIntSet",
+       id: "RestXmlSerPrimitiveIntSetToDefault",
        protocol: "aws.protocols#restXml",
        documentation: "Primitive ints should not be serialized when they are unset",
        uri: "/primitive-document",
        method: "POST",
        body: """
        <PrimitiveIntDocument>
             <value>1</value>
-            <requiredValue>0</requiredValue>
-        </PrimitiveIntDocument>
+       </PrimitiveIntDocument>
        """,
        bodyMediaType: "application/xml",
        params: { value: 1 }
@@ -70,9 +68,9 @@ operation PrimitiveIntOpXml {
 }
 
 structure PrimitiveIntDocument {
-    value: PrimitiveInt,
-    @required
-    requiredValue: PrimitiveInt
+    value: PrimitiveInt
+    @default(0)
+    defaultedValue: PrimitiveInt
 }
 
 @enum([{"value": "enumvalue", "name": "V"}])

codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/protocols/InlineFunctionNamer.kt

@@ -60,11 +60,11 @@ fun ShapeId.toRustIdentifier(): String {
 private fun RustSymbolProvider.shapeFunctionName(prefix: String, shape: Shape): String {
     val symbolNameSnakeCase = toSymbol(shape).fullName.replace("::", "_").toSnakeCase()
     return prefix + "_" + when (shape) {
-        is ListShape -> "list_${shape.id.toRustIdentifier()}"
         is MapShape -> "map_${shape.id.toRustIdentifier()}"
         is MemberShape -> "member_${shape.container.toRustIdentifier()}_${shape.memberName.toSnakeCase()}"
         is OperationShape -> "operation_$symbolNameSnakeCase"
-        is SetShape -> "set_${shape.id.toRustIdentifier()}"
+        is SetShape -> "set_${shape.id.toRustIdentifier()}" // set shape check MUST come before list, it is a subclass
+        is ListShape -> "list_${shape.id.toRustIdentifier()}"
         is StructureShape -> "structure_$symbolNameSnakeCase"
         is UnionShape -> "union_$symbolNameSnakeCase"
         is DocumentShape -> "document"

codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/StructureGeneratorTest.kt

@@ -58,12 +58,13 @@ class StructureGeneratorTest {
             @sensitive
             string SecretKey
 
+            @sensitive
+            string Password
+
             structure Credentials {
                 username: String,
-                @sensitive
-                password: String,
+                password: Password,
 
-                // test that sensitive can be applied directly to a member or to the shape
                 secretKey: SecretKey
             }
 

codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/UnionGeneratorTest.kt

@@ -110,7 +110,7 @@ class UnionGeneratorTest {
             }
 
             @deprecated
-            union Bar {}
+            union Bar { x: Integer }
         """.asSmithyModel()
         val provider: SymbolProvider = testSymbolProvider(model)
         val writer = RustWriter.root()

codegen/src/test/kotlin/software/amazon/smithy/rust/codegen/generators/http/RequestBindingGeneratorTest.kt

@@ -101,10 +101,12 @@ class RequestBindingGeneratorTest {
             @httpPrefixHeaders("X-Prefix-")
             prefix: StringMap,
 
-            @sensitive
             @httpHeader("stringHeader")
-            stringHeader: String
+            stringHeader: SensitiveStringHeader
         }
+
+        @sensitive
+        string SensitiveStringHeader
     """.asSmithyModel()
     private val model = OperationNormalizer.transform(baseModel)
     private val symbolProvider = testSymbolProvider(model)