Check references via idRef in UnreferencedShapes

smithy-aws-cloudformation-traits/src/test/resources/software/amazon/smithy/aws/cloudformation/traits/errorfiles/additionalschemas-conflict.errors

@@ -1,3 +1,2 @@
 [ERROR] smithy.example#AdditionalSchemasConflictResource: The `bar` property of the generated `AdditionalSchemasConflictResource` CloudFormation resource targets multiple shapes: [smithy.api#Boolean, smithy.api#String]. Reusing member names that target different shapes can cause confusion for users of the API. This target discrepancy must either be resolved in the model or one of the members must be excluded from the conversion. | CfnResourceProperty
-[NOTE] smithy.example#AdditionalSchemasConflictProperties: The structure shape is not connected to from any service shape. | UnreferencedShape
 [WARNING] smithy.example#AdditionalSchemasConflictResource: This shape applies a trait that is unstable: aws.cloudformation#cfnResource | UnstableTrait

smithy-aws-cloudformation-traits/src/test/resources/software/amazon/smithy/aws/cloudformation/traits/errorfiles/deconflict-by-excluding.errors

@@ -1,3 +1,2 @@
-[NOTE] smithy.example#AdditionalSchemasDeconflictedProperties: The structure shape is not connected to from any service shape. | UnreferencedShape
 [WARNING] smithy.example#CreateAdditionalSchemasDeconflictedResourceRequest$bar: This shape applies a trait that is unstable: aws.cloudformation#cfnExcludeProperty | UnstableTrait
 [WARNING] smithy.example#AdditionalSchemasDeconflictedResource: This shape applies a trait that is unstable: aws.cloudformation#cfnResource | UnstableTrait

smithy-aws-cloudformation-traits/src/test/resources/software/amazon/smithy/aws/cloudformation/traits/errorfiles/invalid-additional-schemas-shape.errors

@@ -1,3 +1,2 @@
 [ERROR] smithy.example#InvalidAdditionalSchemasShapeResource: Error validating trait `aws.cloudformation#cfnResource`.additionalSchemas.0: Shape ID `smithy.example#ListShape` does not match selector `structure` | TraitValue
-[NOTE] smithy.example#ListShape: The list shape is not connected to from any service shape. | UnreferencedShape
 [WARNING] smithy.example#InvalidAdditionalSchemasShapeResource: This shape applies a trait that is unstable: aws.cloudformation#cfnResource | UnstableTrait

smithy-model/src/main/java/software/amazon/smithy/model/neighbor/IdRefShapeReferences.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.model.neighbor;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.node.Node;
+import software.amazon.smithy.model.node.StringNode;
+import software.amazon.smithy.model.selector.PathFinder;
+import software.amazon.smithy.model.selector.Selector;
+import software.amazon.smithy.model.shapes.MemberShape;
+import software.amazon.smithy.model.shapes.Shape;
+import software.amazon.smithy.model.shapes.ShapeId;
+import software.amazon.smithy.model.traits.IdRefTrait;
+import software.amazon.smithy.model.traits.Trait;
+import software.amazon.smithy.model.traits.TraitDefinition;
+
+/**
+ * Finds all shapes referenced by {@link IdRefTrait} from within trait
+ * values.
+ *
+ * <p>This works by finding all paths from {@link TraitDefinition} shapes
+ * to shapes with {@link IdRefTrait}, then using those paths to search
+ * the node values of each application of the trait to extract the {@link ShapeId}
+ * value. Because we don't have a fixed set of traits known to potentially have
+ * idRef members, this has to be done dynamically.
+ */
+final class IdRefShapeReferences {
+    private static final Selector WITH_ID_REF = Selector.parse("[trait|idRef]");
+    private final Model model;
+    private final Set<ShapeId> found = new HashSet<>();
+
+    IdRefShapeReferences(Model model) {
+        this.model = model;
+    }
+
+    Set<ShapeId> compute(Set<Shape> connected) {
+        PathFinder finder = PathFinder.create(model);
+        for (Shape traitDef : model.getShapesWithTrait(TraitDefinition.class)) {
+            if (traitDef.hasTrait(IdRefTrait.class)) {
+                // PathFinder doesn't handle the case where the trait def has the idRef
+                NodeQuery query = new NodeQuery().self();
+                addReferences(traitDef, query, connected);
+                continue;
+            }
+            List<PathFinder.Path> paths = finder.search(traitDef, WITH_ID_REF);
+            if (!paths.isEmpty()) {
+                for (PathFinder.Path path : paths) {
+                    NodeQuery query = buildNodeQuery(path);
+                    addReferences(traitDef, query, connected);
+                }
+            }
+        }
+        return found;
+    }
+
+    private void addReferences(Shape traitDef, NodeQuery query, Set<Shape> connected) {
+        model.getShapesWithTrait(traitDef.getId()).stream()
+                .filter(connected::contains)
+                .forEach(shape -> {
+                    Trait trait = shape.findTrait(traitDef.getId()).get(); // We already know the shape has the trait.
+                    Node node = trait.toNode();
+                    query.execute(node).forEach(n ->
+                            // Invalid shape ids are handled by the idRef trait validator, so ignore them here.
+                            n.asStringNode().flatMap(StringNode::asShapeId).ifPresent(found::add)
+                    );
+                });
+    }
+
+    private static NodeQuery buildNodeQuery(PathFinder.Path path) {
+        NodeQuery query = new NodeQuery();
+        // The path goes from trait definition -> shape with idRef
+        for (Relationship relationship : path) {
+            if (!relationship.getNeighborShape().isPresent()) {
+                break;
+            }
+            switch (relationship.getRelationshipType()) {
+                case MAP_KEY:
+                    query.anyMemberName();
+                    break;
+                case MAP_VALUE:
+                    query.anyMember();
+                    break;
+                case LIST_MEMBER:
+                case SET_MEMBER:
+                    query.anyElement();
+                    break;
+                case UNION_MEMBER:
+                case STRUCTURE_MEMBER:
+                    MemberShape member = (MemberShape) relationship.getNeighborShape().get();
+                    query.member(member.getMemberName());
+                    break;
+                default:
+                    // Other relationship types don't produce meaningful edges to search the node.
+                    break;
+            }
+        }
+        return query;
+    }
+}

smithy-model/src/main/java/software/amazon/smithy/model/neighbor/NodeQuery.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.model.neighbor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import software.amazon.smithy.model.node.Node;
+import software.amazon.smithy.utils.ListUtils;
+
+/**
+ * Searches {@link Node}s to find matching children. Each search
+ * condition is executed on the result of the previous search,
+ * and the results are aggregated.
+ */
+final class NodeQuery {
+    private final List<Query> queries = new ArrayList<>();
+
+    NodeQuery() {
+    }
+
+    NodeQuery self() {
+        queries.add(Stream::of);
+        return this;
+    }
+
+    NodeQuery member(String name) {
+        queries.add(node -> {
+            if (node == null || !node.isObjectNode()) {
+                return Stream.empty();
+            }
+            return node.expectObjectNode().getMember(name).map(Stream::of).orElse(Stream.empty());
+        });
+        return this;
+    }
+
+    NodeQuery anyMember() {
+        queries.add(node -> {
+            if (node == null || !node.isObjectNode()) {
+                return Stream.empty();
+            }
+            return node.expectObjectNode().getMembers().values().stream();
+        });
+        return this;
+    }
+
+    NodeQuery anyElement() {
+        queries.add(node -> {
+            if (node == null || !node.isArrayNode()) {
+                return Stream.empty();
+            }
+            return node.expectArrayNode().getElements().stream();
+        });
+        return this;
+    }
+
+    NodeQuery anyMemberName() {
+        queries.add(node -> {
+            if (node == null || !node.isObjectNode()) {
+                return Stream.empty();
+            }
+            return node.expectObjectNode().getMembers().keySet().stream();
+        });
+        return this;
+    }
+
+    List<Node> execute(Node node) {
+        if (queries.isEmpty()) {
+            return ListUtils.of();
+        }
+
+        Stream<Node> previousResult = Stream.of(node);
+        for (Query query : queries) {
+            previousResult = previousResult.flatMap(query::run);
+        }
+        return previousResult.collect(Collectors.toList());
+    }
+
+    @FunctionalInterface
+    interface Query {
+        Stream<? extends Node> run(Node node);
+    }
+}

smithy-model/src/main/java/software/amazon/smithy/model/neighbor/UnreferencedShapes.java

@@ -22,6 +22,7 @@
 import software.amazon.smithy.model.knowledge.NeighborProviderIndex;
 import software.amazon.smithy.model.loader.Prelude;
 import software.amazon.smithy.model.shapes.Shape;
+import software.amazon.smithy.model.shapes.ShapeId;
 import software.amazon.smithy.model.traits.TraitDefinition;
 import software.amazon.smithy.utils.FunctionalUtils;
 
@@ -66,6 +67,12 @@ public Set<Shape> compute(Model model) {
             connected.addAll(shapeWalker.walkShapes(trait));
         }
 
+        for (ShapeId referencedThroughIdRef : new IdRefShapeReferences(model).compute(connected)) {
+            // Referenced shapes may not exist in the model, but we don't want to throw.
+            model.getShape(referencedThroughIdRef)
+                    .ifPresent(shape -> connected.addAll(shapeWalker.walkShapes(shape)));
+        }
+
         // Any shape that wasn't identified as connected to a service is considered unreferenced.
         Set<Shape> result = new HashSet<>();
         for (Shape shape : model.toSet()) {

smithy-model/src/main/java/software/amazon/smithy/model/node/StringNode.java

@@ -168,6 +168,19 @@ public ShapeId expectShapeId() {
         }
     }
 
+    /**
+     * Gets the value of the string as a ShapeId if it is a valid Shape ID.
+     *
+     * @return Returns the optional Shape ID.
+     */
+    public Optional<ShapeId> asShapeId() {
+        try {
+            return Optional.of(ShapeId.from(getValue()));
+        } catch (ShapeIdSyntaxException e) {
+            return Optional.empty();
+        }
+    }
+
     @Override
     public boolean equals(Object other) {
         return other instanceof StringNode && value.equals(((StringNode) other).getValue());

smithy-model/src/test/java/software/amazon/smithy/model/neighbor/UnreferencedShapesTest.java

@@ -80,4 +80,29 @@ private Model createPrivateShapeModel(ShapeId id) {
                 .assemble()
                 .unwrap();
     }
+
+    @Test
+    public void checksShapeReferencesThroughIdRef() {
+        Model m = Model.assembler()
+                .addImport(getClass().getResource("through-idref.smithy"))
+                .assemble()
+                .unwrap();
+
+        Set<Shape> shapes = new UnreferencedShapes().compute(m);
+        assertThat(shapes, empty());
+    }
+
+    @Test
+    public void doesNotCheckShapeReferencesThroughIdRefOnUnconnectedTraits() {
+        Model m = Model.assembler()
+                .addImport(getClass().getResource("through-idref-unconnected.smithy"))
+                .assemble()
+                .unwrap();
+
+        Set<ShapeId> ids = new UnreferencedShapes().compute(m).stream().map(Shape::getId).collect(Collectors.toSet());
+        assertThat(ids, containsInAnyOrder(
+                ShapeId.from("com.foo#WithTrait"),
+                ShapeId.from("com.foo#Referenced")
+        ));
+    }
 }

smithy-model/src/test/resources/software/amazon/smithy/model/neighbor/through-idref-unconnected.smithy

@@ -0,0 +1,12 @@
+$version: "2.0"
+
+namespace com.foo
+
+@trait
+@idRef(failWhenMissing: true, selector: "*")
+string unconnected
+
+@unconnected(Referenced)
+structure WithTrait {}
+
+structure Referenced {}