Support vending products that are backed by binaryTargets

This adds support for vending an executable product that consists solely of a binary target that is backed by an artifact bundle. This allows vending binary executables as their own separate package, independently of the plugins that are using them.

rdar://101096803

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Dependency/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+xcuserdata/
+DerivedData/
+.swiftpm/config/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Dependency/Binaries/MyVendedSourceGenBuildTool.artifactbundle/info.json

@@ -0,0 +1,19 @@
+{
+    "schemaVersion": "1.0",
+    "artifacts": {
+        "mytool": {
+            "type": "executable",
+            "version": "1.2.3",
+            "variants": [
+                {
+                    "path": "mytool-macos/mytool",
+                    "supportedTriples": ["x86_64-apple-macosx", "arm64-apple-macosx"]
+                },
+                {
+                    "path": "mytool-linux/mytool",
+                    "supportedTriples": ["x86_64-unknown-linux-gnu"]
+                }
+            ]
+        }
+    }
+}

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Dependency/Binaries/MyVendedSourceGenBuildTool.artifactbundle/mytool-linux/mytool

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+print_usage() {
+    echo "usage: ${0##*/} [--verbose] <in> <out>"
+}
+
+# Parse arguments until we find '--' or an argument that isn't an option.
+until [ $# -eq 0 ]
+do
+    case "$1" in
+        --verbose) verbose=1; shift;;
+        --) shift; break;;
+        -*) echo "unknown option: ${1}"; print_usage; exit 1; shift;;
+        *) break;;
+    esac
+done
+
+# Print usage and leave if we don't have exactly two arguments.
+if [ $# -ne 2 ]; then
+    print_usage
+    exit 1
+fi
+
+# For our sample tool we just copy from one to the other.
+if [ $verbose != 0 ]; then
+    echo "[${0##*/}-linux] '$1' '$2'"
+fi
+
+cp "$1" "$2"

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Dependency/Binaries/MyVendedSourceGenBuildTool.artifactbundle/mytool-macos/mytool

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+print_usage() {
+    echo "usage: ${0##*/} [--verbose] <in> <out>"
+}
+
+# Parse arguments until we find '--' or an argument that isn't an option.
+until [ $# -eq 0 ]
+do
+    case "$1" in
+        --verbose) verbose=1; shift;;
+        --) shift; break;;
+        -*) echo "unknown option: ${1}"; print_usage; exit 1; shift;;
+        *) break;;
+    esac
+done
+
+# Print usage and leave if we don't have exactly two arguments.
+if [ $# -ne 2 ]; then
+    print_usage
+    exit 1
+fi
+
+# For our sample tool we just copy from one to the other.
+if [ $verbose != 0 ]; then
+    echo "[${0##*/}-macosx] '$1' '$2'"
+fi
+
+cp "$1" "$2"

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Dependency/Package.swift

@@ -0,0 +1,18 @@
+// swift-tools-version: 5.6
+import PackageDescription
+
+let package = Package(
+    name: "MyBinaryProduct",
+    products: [
+        .executable(
+            name: "MyVendedSourceGenBuildTool",
+            targets: ["MyVendedSourceGenBuildTool"]
+        ),    
+    ],
+    targets: [
+        .binaryTarget(
+            name: "MyVendedSourceGenBuildTool",
+            path: "Binaries/MyVendedSourceGenBuildTool.artifactbundle"
+        ),
+    ]
+)

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Package.swift

@@ -0,0 +1,26 @@
+// swift-tools-version: 5.6
+import PackageDescription
+
+let package = Package(
+    name: "MyBinaryToolPlugin",
+    dependencies: [
+        .package(path: "Dependency"),
+    ],
+    targets: [
+        // A local tool that uses a build tool plugin.
+        .executableTarget(
+            name: "MyLocalTool",
+            plugins: [
+                "MySourceGenBuildToolPlugin",
+            ]
+        ),
+        // The plugin that generates build tool commands to invoke MySourceGenBuildTool.
+        .plugin(
+            name: "MySourceGenBuildToolPlugin",
+            capability: .buildTool(),
+            dependencies: [
+                .product(name: "MyVendedSourceGenBuildTool", package: "Dependency"),
+            ]
+        ),
+    ]
+)

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Plugins/MySourceGenBuildToolPlugin/plugin.swift

@@ -0,0 +1,34 @@
+import PackagePlugin
+
+@main
+struct MyPlugin: BuildToolPlugin {
+
+    func createBuildCommands(context: PluginContext, target: Target) throws -> [Command] {
+        print("Hello from the Build Tool Plugin!")
+        guard let target = target as? SourceModuleTarget else { return [] }
+        let inputFiles = target.sourceFiles.filter({ $0.path.extension == "dat" })
+        return try inputFiles.map {
+            let inputFile = $0
+            let inputPath = inputFile.path
+            let outputName = inputPath.stem + ".swift"
+            let outputPath = context.pluginWorkDirectory.appending(outputName)
+            return .buildCommand(
+                displayName:
+                    "Generating \(outputName) from \(inputPath.lastComponent)",
+                executable:
+                    try context.tool(named: "mytool").path,
+                arguments: [
+                    "--verbose",
+                    "\(inputPath)",
+                    "\(outputPath)"
+                ],
+                inputFiles: [
+                    inputPath,
+                ],
+                outputFiles: [
+                    outputPath
+                ]
+            )
+        }
+    }
+}

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Sources/MyLocalTool/foo.dat

@@ -0,0 +1 @@
+let foo = "I am Foo!"

Fixtures/Miscellaneous/Plugins/BinaryToolProductPlugin/Sources/MyLocalTool/main.swift

@@ -0,0 +1 @@
+print("Generated string Foo: '\(foo)'")

Sources/Build/BuildPlan.swift

@@ -1469,7 +1469,7 @@ public final class ProductBuildDescription {
             // we will instead have generated a source file containing the redirect.
             // Support for linking tests against executables is conditional on the tools
             // version of the package that defines the executable product.
-            let executableTarget = try product.executableTarget()
+            let executableTarget = try product.executableTarget
             if executableTarget.underlyingTarget is SwiftTarget, toolsVersion >= .v5_5,
                buildParameters.canRenameEntrypointFunctionName {
                 if let flags = buildParameters.linkerFlagsForRenamingMainFunction(of: executableTarget) {
@@ -1989,8 +1989,8 @@ public class BuildPlan {
 
         var productMap: [ResolvedProduct: ProductBuildDescription] = [:]
         // Create product description for each product we have in the package graph except
-        // for automatic libraries and plugins, because they don't produce any output.
-        for product in graph.allProducts where product.type != .library(.automatic) && product.type != .plugin {
+        // for automatic libraries, plugins or products which consist solely of binary targets, because they don't produce any output.
+        for product in graph.allProducts where product.type != .library(.automatic) && product.type != .plugin && !product.targets.filter({ !($0.underlyingTarget is BinaryTarget) }).isEmpty {
             guard let package = graph.package(for: product) else {
                 throw InternalError("unknown package for \(product)")
             }

Sources/Build/LLBuildManifestBuilder.swift

@@ -553,7 +553,7 @@ extension LLBuildManifestBuilder {
             if target.type == .executable {
                 // FIXME: Optimize.
                 let _product = try plan.graph.allProducts.first {
-                    try $0.type == .executable && $0.executableTarget() == target
+                    try $0.type == .executable && $0.executableTarget == target
                 }
                 if let product = _product {
                     guard let planProduct = plan.productMap[product] else {

Sources/Commands/SwiftPackageTool.swift

@@ -1010,6 +1010,7 @@ extension SwiftPackageTool {
             try PluginCommand.run(
                 plugin: matchingPlugins[0],
                 package: packageGraph.rootPackages[0],
+                packageGraph: packageGraph,
                 options: pluginOptions,
                 arguments: arguments,
                 swiftTool: swiftTool)
@@ -1018,6 +1019,7 @@ extension SwiftPackageTool {
         static func run(
             plugin: PluginTarget,
             package: ResolvedPackage,
+            packageGraph: PackageGraph,
             options: PluginOptions,
             arguments: [String],
             swiftTool: SwiftTool
@@ -1080,29 +1082,17 @@ extension SwiftPackageTool {
 
             // Build or bring up-to-date any executable host-side tools on which this plugin depends. Add them and any binary dependencies to the tool-names-to-path map.
             var toolNamesToPaths: [String: AbsolutePath] = [:]
-            for dep in plugin.dependencies(satisfying: try swiftTool.buildParameters().buildEnvironment) {
+            for dep in try plugin.accessibleTools(packageGraph: packageGraph, fileSystem: swiftTool.fileSystem, environment: try swiftTool.buildParameters().buildEnvironment, for: try pluginScriptRunner.hostTriple) {
                 let buildOperation = try swiftTool.createBuildOperation(cacheBuildManifest: false)
                 switch dep {
-                case .product(let productRef, _):
-                    // Build the product referenced by the tool, and add the executable to the tool map.
-                    try buildOperation.build(subset: .product(productRef.name))
-                    if let builtTool = buildOperation.buildPlan?.buildProducts.first(where: { $0.product.name == productRef.name}) {
-                        toolNamesToPaths[productRef.name] = builtTool.binary
-                    }
-                case .target(let target, _):
-                    if let target = target as? BinaryTarget {
-                        // Add the executables vended by the binary target to the tool map.
-                        for exec in try target.parseArtifactArchives(for: pluginScriptRunner.hostTriple, fileSystem: swiftTool.fileSystem) {
-                            toolNamesToPaths[exec.name] = exec.executablePath
-                        }
-                    }
-                    else {                        
-                        // Build the product referenced by the tool, and add the executable to the tool map. Product dependencies are not supported within a package, so we instead find the executable that corresponds to the product. There is always one, because of autogeneration of implicit executables with the same name as the target if there isn't an explicit one.
-                        try buildOperation.build(subset: .product(target.name))
-                        if let builtTool = buildOperation.buildPlan?.buildProducts.first(where: { $0.product.name == target.name}) {
-                            toolNamesToPaths[target.name] = builtTool.binary
-                        }
+                case .builtTool(let name, _):
+                    // Build the product referenced by the tool, and add the executable to the tool map. Product dependencies are not supported within a package, so if the tool happens to be from the same package, we instead find the executable that corresponds to the product. There is always one, because of autogeneration of implicit executables with the same name as the target if there isn't an explicit one.
+                    try buildOperation.build(subset: .product(name))
+                    if let builtTool = buildOperation.buildPlan?.buildProducts.first(where: { $0.product.name == name}) {
+                        toolNamesToPaths[name] = builtTool.binary
                     }
+                case .vendedTool(let name, let path):
+                    toolNamesToPaths[name] = path
                 }
             }
 
@@ -1536,6 +1526,7 @@ extension SwiftPackageTool {
             try PluginCommand.run(
                 plugin: matchingPlugins[0],
                 package: packageGraph.rootPackages[0],
+                packageGraph: packageGraph,
                 options: pluginOptions,
                 arguments: Array( remaining.dropFirst()),
                 swiftTool: swiftTool)

Sources/PackageGraph/ResolvedProduct.swift

@@ -43,11 +43,16 @@ public final class ResolvedProduct {
     /// The main executable target of product.
     ///
     /// Note: This property is only valid for executable products.
-    public func executableTarget() throws -> ResolvedTarget {
-        guard type == .executable || type == .snippet else {
-            throw InternalError("firstExecutableModule should only be called for executable targets")
+    public var executableTarget: ResolvedTarget {
+        get throws {
+            guard type == .executable || type == .snippet else {
+                throw InternalError("`executableTarget` should only be called for executable targets")
+            }
+            guard let underlyingExecutableTarget = targets.map({ $0.underlyingTarget }).executables.first, let executableTarget = targets.first(where: { $0.underlyingTarget == underlyingExecutableTarget }) else {
+                throw InternalError("could not determine executable target")
+            }
+            return executableTarget
         }
-        return targets.first(where: { $0.type == .executable || $0.type == .snippet })!
     }
 
     public init(product: Product, targets: [ResolvedTarget]) {

Sources/PackageLoading/ManifestLoader+Validation.swift

@@ -78,10 +78,14 @@ public struct ManifestValidator {
                 }
             }
 
-            // Check that products that reference only binary targets don't define a type.
-            let areTargetsBinary = product.targets.allSatisfy { self.manifest.targetMap[$0]?.type == .binary }
-            if areTargetsBinary && product.type != .library(.automatic) {
-                diagnostics.append(.invalidBinaryProductType(productName: product.name))
+            // Check that products that reference only binary targets don't define an explicit library type.
+            if product.targets.allSatisfy({ self.manifest.targetMap[$0]?.type == .binary }) {
+                switch product.type {
+                case .library(.automatic), .executable:
+                    break
+                default:
+                    diagnostics.append(.invalidBinaryProductType(productName: product.name))
+                }
             }
         }
 
@@ -262,7 +266,7 @@ extension Basics.Diagnostic {
     }
 
     static func invalidBinaryProductType(productName: String) -> Self {
-        .error("invalid type for binary product '\(productName)'; products referencing only binary targets must have a type of 'library'")
+        .error("invalid type for binary product '\(productName)'; products referencing only binary targets must be executable or automatic library products")
     }
 
     /*static func duplicateDependency(dependencyIdentity: PackageIdentity) -> Self {

Sources/PackageLoading/PackageBuilder.swift

@@ -1290,7 +1290,7 @@ public final class PackageBuilder {
     }
 
     private func validateExecutableProduct(_ product: ProductDescription, with targets: [Target]) -> Bool {
-        let executableTargetCount = targets.filter { $0.type == .executable }.count
+        let executableTargetCount = targets.executables.count
         guard executableTargetCount == 1 else {
             if executableTargetCount == 0 {
                 if let target = targets.spm_only {

Sources/PackageModel/Product.swift

@@ -43,7 +43,7 @@ public class Product: Codable {
             throw InternalError("Targets cannot be empty")
         }
         if type == .executable {
-            guard targets.filter({ $0.type == .executable }).count == 1 else {
+            guard targets.executables.count == 1 else {
                 throw InternalError("Executable products should have exactly one executable target.")
             }
         }

Sources/PackageModel/Target.swift

@@ -638,6 +638,11 @@ public final class BinaryTarget: Target {
         }
     }
 
+    public var containsExecutable: Bool {
+        // FIXME: needs to be revisited once libraries are supported in artifact bundles
+        return self.kind == .artifactsArchive
+    }
+
     public enum Origin: Equatable, Codable {
 
         /// Represents an artifact that was downloaded from a remote URL.
@@ -799,3 +804,18 @@ public enum PluginPermission: Hashable, Codable {
         }
     }
 }
+
+public extension Sequence where Iterator.Element == Target {
+    var executables: [Target] {
+        return filter {
+            switch $0.type {
+            case .binary:
+                return ($0 as? BinaryTarget)?.containsExecutable == true
+            case .executable:
+                return true
+            default:
+                return false
+            }
+        }
+    }
+}

Sources/SPMBuildCore/PluginContextSerializer.swift

@@ -184,9 +184,7 @@ internal struct PluginContextSerializer {
         switch product.type {
 
         case .executable:
-            guard let mainExecTarget = product.targets.first(where: { $0.type == .executable }) else {
-                throw InternalError("could not determine main executable target for product \(product)")
-            }
+            let mainExecTarget = try product.executableTarget
             guard let mainExecTargetId = try serialize(target: mainExecTarget) else {
                 throw InternalError("unable to serialize main executable target \(mainExecTarget) for product \(product)")
             }