feat(minor): Improve metric api ergonomics (#175) (#176)

Improve ergonomics of benchmark metric selection by allowing one to write `.all` instead of `BenchmarkMetric.all`

Benchmarks/Basic/Basic+SetupTeardown.swift

@@ -1,6 +1,6 @@
 //
 //  File.swift
-//  
+//
 //
 //  Created by Joakim Hassila on 2023-04-21.
 //

Benchmarks/Basic/BenchmarkRunner+Basic.swift

@@ -24,9 +24,9 @@ let benchmarks = {
                                                                 .p99: .milliseconds(3),
                                                                 .p100: .milliseconds(1)]
 
-        thresholds = [BenchmarkMetric.wallClock: BenchmarkThresholds(absolute: absolute)]
+        thresholds = [.wallClock: .init(absolute: absolute)]
     } else {
-        thresholds = [BenchmarkMetric.wallClock: BenchmarkThresholds.relaxed]
+        thresholds = [.wallClock: .relaxed]
     }
 
     Benchmark.defaultConfiguration = .init(warmupIterations: 0,
@@ -47,7 +47,7 @@ let benchmarks = {
     }
 
     Benchmark("Scaled metrics",
-              configuration: .init(metrics: BenchmarkMetric.all + [CustomMetrics.two, CustomMetrics.one],
+              configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one],
                                    scalingFactor: .kilo)) { benchmark in
         for _ in benchmark.scaledIterations {
             blackHole(Int.random(in: benchmark.scaledIterations))
@@ -57,7 +57,7 @@ let benchmarks = {
     }
 
     Benchmark("All metrics",
-              configuration: .init(metrics: BenchmarkMetric.all, skip: true)) { _ in
+              configuration: .init(metrics: .all, skip: true)) { _ in
     }
 
     let stats = Statistics(numberOfSignificantDigits: .four)
@@ -68,7 +68,7 @@ let benchmarks = {
     }
 
     Benchmark("Statistics",
-              configuration: .init(metrics: BenchmarkMetric.arc + [.wallClock],
+              configuration: .init(metrics: .arc + [.wallClock],
                                    scalingFactor: .kilo, maxDuration: .seconds(1))) { benchmark in
         for _ in benchmark.scaledIterations {
             blackHole(stats.percentiles())

Benchmarks/DateTime/DateTime.swift

@@ -11,7 +11,7 @@ import Benchmark
 import DateTime
 
 let benchmarks = {
-    Benchmark.defaultConfiguration = .init(metrics: [.throughput, .wallClock] + BenchmarkMetric.arc,
+    Benchmark.defaultConfiguration = .init(metrics: [.throughput, .wallClock] + .arc,
                                            warmupIterations: 10,
                                            scalingFactor: .kilo,
                                            maxDuration: .seconds(1),

Benchmarks/Histogram/Histogram.swift

@@ -11,7 +11,7 @@ import Benchmark
 import Histogram
 
 let benchmarks = {
-    let metrics = [.wallClock, .throughput] + BenchmarkMetric.memory + BenchmarkMetric.arc
+    let metrics = [.wallClock, .throughput] + .memory + .arc
     Benchmark.defaultConfiguration = .init(metrics: metrics,
                                            scalingFactor: .mega,
                                            maxDuration: .seconds(1),

Package.swift

@@ -122,14 +122,14 @@ let package = Package(
 let macOSSPIBuild: Bool // Disables jemalloc for macOS SPI builds as the infrastructure doesn't have jemalloc there
 
 #if os(macOS) || os(iOS)
-if let spiBuildEnvironment = ProcessInfo.processInfo.environment["SPI_BUILD"], spiBuildEnvironment == "1" {
-    macOSSPIBuild = true
-    print("Building for SPI@macOS, disabling Jemalloc")
-} else {
-    macOSSPIBuild = false
-}
+    if let spiBuildEnvironment = ProcessInfo.processInfo.environment["SPI_BUILD"], spiBuildEnvironment == "1" {
+        macOSSPIBuild = true
+        print("Building for SPI@macOS, disabling Jemalloc")
+    } else {
+        macOSSPIBuild = false
+    }
 #else
-macOSSPIBuild = false
+    macOSSPIBuild = false
 #endif
 
 // Add Benchmark target dynamically

Plugins/BenchmarkTool/BenchmarkTool+CreateBenchmark.swift

@@ -11,11 +11,11 @@
 import SystemPackage
 
 #if canImport(Darwin)
-import Darwin
+    import Darwin
 #elseif canImport(Glibc)
-import Glibc
+    import Glibc
 #else
-#error("Unsupported Platform")
+    #error("Unsupported Platform")
 #endif
 
 extension BenchmarkTool {

Plugins/BenchmarkTool/BenchmarkTool+Export.swift

@@ -57,9 +57,9 @@ extension BenchmarkTool {
         outputPath.append(csvFile.components)
 
         print("Writing output to \(outputPath)")
-        
+
         printFailedBenchmarks()
-        
+
         do {
             let fd = try FileDescriptor.open(
                 outputPath, .writeOnly, options: [.truncate, .create], permissions: .ownerReadWrite
@@ -112,7 +112,7 @@ extension BenchmarkTool {
         outputPath.append(jsonFile.components)
 
         print("Writing output to \(outputPath)")
-        
+
         printFailedBenchmarks()
 
         do {
@@ -221,7 +221,7 @@ extension BenchmarkTool {
             }
         }
     }
-    
+
     func printFailedBenchmarks() {
         if !failedBenchmarkList.isEmpty {
             print("The following benchmarks failed: \n")

Plugins/BenchmarkTool/BenchmarkTool+Operations.swift

@@ -62,7 +62,7 @@ extension BenchmarkTool {
                 break outerloop
             case let .error(description):
                 failBenchmark(description, exitCode: .benchmarkJobFailed, "\(target)/\(benchmark.name)")
-                
+
                 benchmarkResults[BenchmarkIdentifier(target: target, name: benchmark.name)] = []
                 break outerloop
             default:

Plugins/BenchmarkTool/BenchmarkTool.swift

@@ -107,38 +107,33 @@ struct BenchmarkTool: AsyncParsableCommand {
     var benchmarkBaselines: [BenchmarkBaseline] = [] // The baselines read from disk, merged + current run if needed
     var comparisonBaseline: BenchmarkBaseline?
     var checkBaseline: BenchmarkBaseline?
-    
+
     var failedBenchmarkList: [String] = []
 
     mutating func failBenchmark(_ reason: String? = nil, exitCode: ExitCode = .genericFailure, _ failedBenchmark: String? = nil) {
         if let reason {
             print(reason)
             print("")
         }
-        
+
         // check what failed and react accordingly
         switch exitCode {
-            case .genericFailure:
-                exitBenchmark(exitCode: exitCode)
-            case .thresholdViolation:
-                exitBenchmark(exitCode: exitCode)
-            case .benchmarkJobFailed:
-                if let failedBenchmark {
-                    failedBenchmarkList.append(failedBenchmark)
-                }
-            default:
-                exitBenchmark(exitCode: exitCode)
+        case .benchmarkJobFailed:
+            if let failedBenchmark {
+                failedBenchmarkList.append(failedBenchmark)
+            }
+        default:
+            exitBenchmark(exitCode: exitCode)
         }
     }
-    
+
     func exitBenchmark(exitCode: ExitCode) {
         #if canImport(Darwin)
             Darwin.exit(exitCode.rawValue)
         #elseif canImport(Glibc)
             Glibc.exit(exitCode.rawValue)
         #endif
     }
-
 
     func printChildRunError(error: Int32, benchmarkExecutablePath: String) {
         print("Failed to run '\(command)' for \(benchmarkExecutablePath), error code [\(error)]")
@@ -314,7 +309,6 @@ struct BenchmarkTool: AsyncParsableCommand {
                 switch benchmarkCommand {
                 case .`init`:
                     fatalError("Should never come here")
-                    break
                 case .query:
                     try queryBenchmarks(benchmarkPath) // Get all available benchmarks first
                 case .list:
@@ -347,7 +341,7 @@ struct BenchmarkTool: AsyncParsableCommand {
 
         return benchmarkResults
     }
-    
+
     struct FailedBenchmark: Codable {
         let benchmarkName: String
         let failureReason: String

Sources/Benchmark/BenchmarkExecutor+Extensions.swift

@@ -77,4 +77,5 @@ extension BenchmarkExecutor {
         }
     }
 }
+
 // swiftlint:enable cyclomatic_complexity

Sources/Benchmark/BenchmarkInternals.swift

@@ -36,4 +36,5 @@ public enum BenchmarkCommandReply: Codable {
     case end // end of query for list/result
     case error(_ description: String) // error while performing operation (e.g. 'run')
 }
+
 // swiftlint:enable all

Sources/Benchmark/BenchmarkMetric+Defaults.swift

@@ -13,6 +13,9 @@ public extension BenchmarkMetric {
     /// The default collection of metrics used for a benchmark.
     ///
     /// The defaults include ``wallClock``, ``cpuTotal``, ``mallocCountTotal``, ``throughput``, and ``peakMemoryResident``.
+    ///
+    /// There is also an convenience extension on Array defined such that you can write just `.default` rather than `BenchmarkMetric.default`
+    ///
     static var `default`: [BenchmarkMetric] {
         [.wallClock,
          .cpuTotal,
@@ -100,3 +103,43 @@ public extension BenchmarkMetric {
          .retainReleaseDelta]
     }
 }
+
+// Nicer convenience extension for Array so one can write `.extended` instead of `BenchmarkMetric.extended`
+public extension [BenchmarkMetric] {
+    /// The default collection of metrics used for a benchmark.
+    ///
+    /// The defaults include ``wallClock``, ``cpuTotal``, ``mallocCountTotal``, ``throughput``, and ``peakMemoryResident``.
+    static var `default`: [BenchmarkMetric] {
+        BenchmarkMetric.default
+    }
+
+    /// A collection of extended system benchmarks.
+    static var extended: [BenchmarkMetric] {
+        BenchmarkMetric.extended
+    }
+
+    /// A collection of memory benchmarks.
+    static var memory: [BenchmarkMetric] {
+        BenchmarkMetric.memory
+    }
+
+    /// A collection of ARC metrics
+    static var arc: [BenchmarkMetric] {
+        BenchmarkMetric.arc
+    }
+
+    /// A collection of system benchmarks.
+    static var system: [BenchmarkMetric] {
+        BenchmarkMetric.system
+    }
+
+    /// A collection of disk benchmarks.
+    static var disk: [BenchmarkMetric] {
+        BenchmarkMetric.disk
+    }
+
+    /// A collection of all benchmarks supported by this library.
+    static var all: [BenchmarkMetric] {
+        BenchmarkMetric.all
+    }
+}

Sources/Benchmark/BenchmarkResult.swift

@@ -531,4 +531,5 @@ public extension BenchmarkTimeUnits {
         }
     }
 }
+
 // swiftlint:enable file_length identifier_name function_parameter_count function_body_length type_body_length

Sources/Benchmark/BenchmarkRunner.swift

@@ -162,7 +162,7 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
                     }
 
                     if debug, allMetrics {
-                        benchmark.configuration.metrics = BenchmarkMetric.all
+                        benchmark.configuration.metrics = .all
                     }
 
                     benchmark.target = benchmarkToRun.target
@@ -180,17 +180,17 @@ public struct BenchmarkRunner: AsyncParsableCommand, BenchmarkRunnerReadWrite {
                         }
                     } catch {
                         let description = """
-                                          Benchmark.setup or local benchmark setup failed:
+                        Benchmark.setup or local benchmark setup failed:
 
-                                          \(error)
+                        \(error)
 
-                                          If it is a filesystem permissioning error or if the benchmark uses networking, you may need
-                                          to give permissions or even disable SwiftPM's sandbox environment and run the benchmark using:
+                        If it is a filesystem permissioning error or if the benchmark uses networking, you may need
+                        to give permissions or even disable SwiftPM's sandbox environment and run the benchmark using:
 
-                                          swift package --allow-writing-to-package-directory benchmark
-                                          or
-                                          swift package --disable-sandbox benchmark
-                                          """
+                        swift package --allow-writing-to-package-directory benchmark
+                        or
+                        swift package --disable-sandbox benchmark
+                        """
 
                         try channel.write(.error(description))
                         return

Sources/Benchmark/BenchmarkThresholds+Defaults.swift

@@ -92,4 +92,5 @@ public extension BenchmarkThresholds {
         BenchmarkThresholds()
     }
 }
+
 // swiftlint:enable discouraged_none_name

Sources/Benchmark/Blackhole.swift

@@ -31,3 +31,8 @@
 /// ```
 @inline(never)
 public func blackHole(_: some Any) {}
+
+@inline(never)
+public func identity<T>(_ value: T) -> T {
+    value
+}