Generate all requested modes in parallel (#280)

### Motivation The generator itself is synchronous, but it can be run in one of more modes: `types`, `client`, `server`, and these runs can be parallelised. This is likely to have value to most adopters since most will use at least two modes. This is especially useful for large APIs. ### Modifications Run all the requested modes of the generator in a task group. ### Result When running with multiple modes, generation is faster. Concretely, when using the Github API and generating types and client, it cut the overall generation time by 40%. ### Test Plan CI. ### Resolves Resolves #227. Signed-off-by: Si Beaumont <[email protected]>
apple · Sep 18, 2023 · 7da0f96 · 7da0f96
1 parent a07ebd2
commit 7da0f96
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 23 deletions.
diff --git a/Sources/_OpenAPIGeneratorCore/Config.swift b/Sources/_OpenAPIGeneratorCore/Config.swift
@@ -18,7 +18,7 @@
 /// A single generator pipeline run produces exactly one file, so for
 /// generating multiple files, create multiple configuration values, each with
 /// a different generator mode.
-public struct Config {
+public struct Config: Sendable {
 
     /// The generator mode to use.
     public var mode: GeneratorMode

diff --git a/Sources/_OpenAPIGeneratorCore/Diagnostics.swift b/Sources/_OpenAPIGeneratorCore/Diagnostics.swift
@@ -15,7 +15,7 @@ import Foundation
 import OpenAPIKit
 
 /// A message emitted by the generator.
-public struct Diagnostic: Error, Codable {
+public struct Diagnostic: Error, Codable, Sendable {
 
     /// Describes the severity of a diagnostic.
     public enum Severity: String, Codable, Sendable {
@@ -327,8 +327,7 @@ struct PrintingDiagnosticCollector: DiagnosticCollector {
 }
 
 /// A diagnostic collector that prints diagnostics to standard error.
-public struct StdErrPrintingDiagnosticCollector: DiagnosticCollector {
-
+public struct StdErrPrintingDiagnosticCollector: DiagnosticCollector, Sendable {
     /// Creates a new collector.
     public init() {}
 

diff --git a/Sources/_OpenAPIGeneratorCore/FeatureFlags.swift b/Sources/_OpenAPIGeneratorCore/FeatureFlags.swift
@@ -25,7 +25,7 @@
 /// enabled unconditionally on main and the feature flag removed, and version
 /// 0.2 is tagged. (This is for pre-1.0 versioning, would be 1.0 and 2.0 after
 /// 1.0 is released.)
-public enum FeatureFlag: String, Hashable, Codable, CaseIterable {
+public enum FeatureFlag: String, Hashable, Codable, CaseIterable, Sendable {
 
     /// Support for `nullable` schemas.
     ///

diff --git a/Sources/swift-openapi-generator/GenerateCommand.swift b/Sources/swift-openapi-generator/GenerateCommand.swift
@@ -55,7 +55,7 @@ struct _GenerateCommand: AsyncParsableCommand {
     var isDryRun: Bool = false
 
     func run() async throws {
-        try generate.runGenerator(
+        try await generate.runGenerator(
             outputDirectory: outputDirectory,
             pluginSource: pluginSource,
             isDryRun: isDryRun

diff --git a/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift b/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift
@@ -29,7 +29,7 @@ extension _GenerateOptions {
         outputDirectory: URL,
         pluginSource: PluginSource?,
         isDryRun: Bool
-    ) throws {
+    ) async throws {
         let config = try loadedConfig()
         let sortedModes = try resolvedModes(config)
         let resolvedAdditionalImports = resolvedAdditionalImports(config)
@@ -41,7 +41,7 @@ extension _GenerateOptions {
                 featureFlags: resolvedFeatureFlags
             )
         }
-        let diagnostics: any DiagnosticCollector
+        let diagnostics: any DiagnosticCollector & Sendable
         let finalizeDiagnostics: () throws -> Void
         if let diagnosticsOutputPath {
             let _diagnostics = _YamlFileDiagnosticsCollector(url: diagnosticsOutputPath)
@@ -70,7 +70,7 @@ extension _GenerateOptions {
             """
         )
         do {
-            try _Tool.runGenerator(
+            try await _Tool.runGenerator(
                 doc: doc,
                 configs: configs,
                 pluginSource: pluginSource,

diff --git a/Sources/swift-openapi-generator/YamlFileDiagnosticsCollector.swift b/Sources/swift-openapi-generator/YamlFileDiagnosticsCollector.swift
@@ -21,7 +21,9 @@ struct _DiagnosticsYamlFileContent: Encodable {
 }
 
 /// A collector that writes diagnostics to a YAML file.
-class _YamlFileDiagnosticsCollector: DiagnosticCollector {
+final class _YamlFileDiagnosticsCollector: DiagnosticCollector, @unchecked Sendable {
+    /// Protects `diagnostics`.
+    private let lock = NSLock()
 
     /// A list of collected diagnostics.
     private var diagnostics: [Diagnostic] = []
@@ -36,12 +38,16 @@ class _YamlFileDiagnosticsCollector: DiagnosticCollector {
     }
 
     func emit(_ diagnostic: Diagnostic) {
+        lock.lock()
+        defer { lock.unlock() }
         diagnostics.append(diagnostic)
     }
 
     /// Finishes writing to the collector by persisting the accumulated
     /// diagnostics to a YAML file.
     func finalize() throws {
+        lock.lock()
+        defer { lock.unlock() }
         let uniqueMessages = Set(diagnostics.map(\.message)).sorted()
         let encoder = YAMLEncoder()
         encoder.options.sortKeys = true

diff --git a/Sources/swift-openapi-generator/runGenerator.swift b/Sources/swift-openapi-generator/runGenerator.swift
@@ -11,7 +11,14 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 //===----------------------------------------------------------------------===//
-import Foundation
+#if os(Linux)
+@preconcurrency import struct Foundation.URL
+@preconcurrency import struct Foundation.Data
+#else
+import struct Foundation.URL
+import struct Foundation.Data
+#endif
+import class Foundation.FileManager
 import ArgumentParser
 import _OpenAPIGeneratorCore
 
@@ -32,24 +39,30 @@ extension _Tool {
         pluginSource: PluginSource?,
         outputDirectory: URL,
         isDryRun: Bool,
-        diagnostics: any DiagnosticCollector
-    ) throws {
+        diagnostics: any DiagnosticCollector & Sendable
+    ) async throws {
         let docData: Data
         do {
             docData = try Data(contentsOf: doc)
         } catch {
             throw ValidationError("Failed to load the OpenAPI document at path \(doc.path), error: \(error)")
         }
-        for config in configs {
-            try runGenerator(
-                doc: doc,
-                docData: docData,
-                config: config,
-                outputDirectory: outputDirectory,
-                outputFileName: config.mode.outputFileName,
-                isDryRun: isDryRun,
-                diagnostics: diagnostics
-            )
+
+        try await withThrowingTaskGroup(of: Void.self) { group in
+            for config in configs {
+                group.addTask {
+                    try runGenerator(
+                        doc: doc,
+                        docData: docData,
+                        config: config,
+                        outputDirectory: outputDirectory,
+                        outputFileName: config.mode.outputFileName,
+                        isDryRun: isDryRun,
+                        diagnostics: diagnostics
+                    )
+                }
+            }
+            try await group.waitForAll()
         }
 
         // If from a BuildTool plugin, the generator will have to emit all 3 files