From 4b34c19013de868e59fb16d80008d51f583e0cb9 Mon Sep 17 00:00:00 2001 From: Jairon Terrero Date: Mon, 16 Sep 2024 11:53:28 -0600 Subject: [PATCH] feat(minor): Add Benchmark tags Sometimes it is convenient to run the same benchmark multiple times under slightly different scenarios. For example, benchmarking `func foo(_ n: Int)` for various parameterizations of `n`. This adds support for specifying tags for a benchmark but makes no functional changes beyond the name/identifier of a benchmark now including a parameter list when one was provided. This unlocks the potential for the benchmark exporters (such as the Influx exporter) to leverage these tags (by emitting appropriate tags/fields in the case of Influx) in their output. --- Plugins/BenchmarkTool/BenchmarkTool.swift | 2 +- Sources/Benchmark/Benchmark.swift | 39 +++++++++++++++++++---- Sources/Benchmark/BenchmarkExecutor.swift | 2 ++ Sources/Benchmark/BenchmarkResult.swift | 3 ++ Tests/BenchmarkTests/BenchmarkTests.swift | 13 ++++++++ 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Plugins/BenchmarkTool/BenchmarkTool.swift b/Plugins/BenchmarkTool/BenchmarkTool.swift index af26196b..068409ae 100644 --- a/Plugins/BenchmarkTool/BenchmarkTool.swift +++ b/Plugins/BenchmarkTool/BenchmarkTool.swift @@ -268,7 +268,7 @@ struct BenchmarkTool: AsyncParsableCommand { // run each benchmark for the target as a separate process try benchmarks.forEach { benchmark in - if try shouldIncludeBenchmark(benchmark.name) { + if try shouldIncludeBenchmark(benchmark.baseName) { let results = try runChild(benchmarkPath: benchmark.executablePath!, benchmarkCommand: command, benchmark: benchmark) { [self] result in diff --git a/Sources/Benchmark/Benchmark.swift b/Sources/Benchmark/Benchmark.swift index 94ba1287..79f2be93 100644 --- a/Sources/Benchmark/Benchmark.swift +++ b/Sources/Benchmark/Benchmark.swift @@ -10,10 +10,10 @@ import Dispatch -// swiftlint: disable file_length +// swiftlint:disable file_length /// Defines a benchmark -public final class Benchmark: Codable, Hashable { +public final class Benchmark: Codable, Hashable { // swiftlint:disable:this type_body_length #if swift(>=5.8) @_documentation(visibility: internal) #endif @@ -69,8 +69,28 @@ public final class Benchmark: Codable, Hashable { #endif public static var benchmarks: [Benchmark] = [] // Bookkeeping of all registered benchmarks + /// The name of the benchmark without any of the tags appended + public var baseName: String + /// The name used for display purposes of the benchmark (also used for matching when comparing to baselines) - public var name: String + public var name: String { + get { + if configuration.tags.isEmpty { + return baseName + } else { + return baseName + + " (" + + configuration.tags + .sorted(by: { $0.key < $1.key }) + .map({ "\($0.key): \($0.value)" }) + .joined(separator: ", ") + + ")" + } + } + set { + baseName = newValue + } + } /// The reason for a benchmark failure, not set if successful public var failureReason: String? @@ -119,6 +139,7 @@ public final class Benchmark: Codable, Hashable { /// Hook for setting defaults for a whole benchmark suite public static var defaultConfiguration: Configuration = .init(metrics: BenchmarkMetric.default, + tags: [:], timeUnits: .automatic, warmupIterations: 1, scalingFactor: .one, @@ -131,7 +152,7 @@ public final class Benchmark: Codable, Hashable { var measurementCompleted = false // Keep track so we skip multiple 'end of measurement' enum CodingKeys: String, CodingKey { - case name + case baseName = "name" case target case executablePath case configuration @@ -168,7 +189,7 @@ public final class Benchmark: Codable, Hashable { return nil } target = "" - self.name = name + self.baseName = name self.configuration = configuration self.closure = closure self.setup = setup @@ -193,7 +214,7 @@ public final class Benchmark: Codable, Hashable { return nil } target = "" - self.name = name + self.baseName = name self.configuration = configuration asyncClosure = closure self.setup = setup @@ -362,6 +383,8 @@ public extension Benchmark { struct Configuration: Codable { /// Defines the metrics that should be measured for the benchmark public var metrics: [BenchmarkMetric] + /// Specifies the parameters used to define the benchmark. + public var tags: [String: String] /// Override the automatic detection of timeunits for metrics related to time to a specific /// one (auto should work for most use cases) public var timeUnits: BenchmarkTimeUnits @@ -386,6 +409,7 @@ public extension Benchmark { public var teardown: BenchmarkTeardownHook? public init(metrics: [BenchmarkMetric] = defaultConfiguration.metrics, + tags: [String: String] = defaultConfiguration.tags, timeUnits: BenchmarkTimeUnits = defaultConfiguration.timeUnits, warmupIterations: Int = defaultConfiguration.warmupIterations, scalingFactor: BenchmarkScalingFactor = defaultConfiguration.scalingFactor, @@ -397,6 +421,7 @@ public extension Benchmark { setup: BenchmarkSetupHook? = nil, teardown: BenchmarkTeardownHook? = nil) { self.metrics = metrics + self.tags = tags self.timeUnits = timeUnits self.warmupIterations = warmupIterations self.scalingFactor = scalingFactor @@ -411,6 +436,7 @@ public extension Benchmark { // swiftlint:disable nesting enum CodingKeys: String, CodingKey { case metrics + case tags case timeUnits case warmupIterations case scalingFactor @@ -447,3 +473,4 @@ public extension Benchmark { @_optimize(none) // Used after tip here: https://forums.swift.org/t/compiler-swallows-blackhole/64305/10 - see also https://github.com/apple/swift/commit/1fceeab71e79dc96f1b6f560bf745b016d7fcdcf static func blackHole(_: some Any) {} } +// swiftlint:enable file_length diff --git a/Sources/Benchmark/BenchmarkExecutor.swift b/Sources/Benchmark/BenchmarkExecutor.swift index 8b2abb96..f1da29f2 100644 --- a/Sources/Benchmark/BenchmarkExecutor.swift +++ b/Sources/Benchmark/BenchmarkExecutor.swift @@ -444,6 +444,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length scalingFactor: benchmark.configuration.scalingFactor, warmupIterations: benchmark.configuration.warmupIterations, thresholds: benchmark.configuration.thresholds?[metric], + tags: benchmark.configuration.tags, statistics: value) results.append(result) } @@ -457,6 +458,7 @@ struct BenchmarkExecutor { // swiftlint:disable:this type_body_length scalingFactor: benchmark.configuration.scalingFactor, warmupIterations: benchmark.configuration.warmupIterations, thresholds: benchmark.configuration.thresholds?[metric], + tags: benchmark.configuration.tags, statistics: value) results.append(result) } diff --git a/Sources/Benchmark/BenchmarkResult.swift b/Sources/Benchmark/BenchmarkResult.swift index a0ba6255..09a6a855 100644 --- a/Sources/Benchmark/BenchmarkResult.swift +++ b/Sources/Benchmark/BenchmarkResult.swift @@ -161,12 +161,14 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { scalingFactor: BenchmarkScalingFactor, warmupIterations: Int, thresholds: BenchmarkThresholds? = nil, + tags: [String: String] = [:], statistics: Statistics) { self.metric = metric self.timeUnits = timeUnits == .automatic ? BenchmarkTimeUnits(statistics.units()) : timeUnits self.scalingFactor = scalingFactor self.warmupIterations = warmupIterations self.thresholds = thresholds + self.tags = tags self.statistics = statistics } @@ -175,6 +177,7 @@ public struct BenchmarkResult: Codable, Comparable, Equatable { public var scalingFactor: BenchmarkScalingFactor public var warmupIterations: Int public var thresholds: BenchmarkThresholds? + public var tags: [String: String] public var statistics: Statistics public var scaledTimeUnits: BenchmarkTimeUnits { diff --git a/Tests/BenchmarkTests/BenchmarkTests.swift b/Tests/BenchmarkTests/BenchmarkTests.swift index fd9049b0..43e9769b 100644 --- a/Tests/BenchmarkTests/BenchmarkTests.swift +++ b/Tests/BenchmarkTests/BenchmarkTests.swift @@ -73,4 +73,17 @@ final class BenchmarkTests: XCTestCase { XCTAssertNotNil(benchmark) benchmark?.run() } + + func testBenchmarkParameterizedDescription() throws { + let benchmark = Benchmark("testBenchmarkParameterizedDescription benchmark", + configuration: .init( + tags: [ + "foo": "bar", + "bin": String(42), + "pi": String(3.14) + ] + )) { _ in } + XCTAssertNotNil(benchmark) + XCTAssertEqual(benchmark?.name, "testBenchmarkParameterizedDescription benchmark (bin: 42, foo: bar, pi: 3.14)") + } }