Add performance benchmarks to alert about performance issues (#215)

Co-authored-by: Adam <[email protected]>
Co-authored-by: Piotr Krzemiński <[email protected]>

.github/workflows/benchmark.yml

@@ -0,0 +1,94 @@
+name: Run Benchmarks
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  BENCHMARK_RESULTS: snake-kmp-benchmarks/build/reports/benchmarks
+
+concurrency:
+  group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-benchmark:
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+            additional-args: '-x jvmBenchmark -x jsBenchmark'
+          - os: macos-13 # for macosX64
+            additional-args: '-x jvmBenchmark -x jsBenchmark'
+          - os: windows-latest
+            additional-args: '-x jvmBenchmark -x jsBenchmark'
+    name: Performance regression check on ${{ matrix.os }} runner
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: 'Set up JDK'
+      uses: 'actions/setup-java@v4'
+      with:
+        java-version: '11'
+        distribution: 'zulu'
+        cache: 'gradle'
+    - name: Validate Gradle Wrapper
+      uses: gradle/actions/wrapper-validation@v3
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@v3
+      with:
+        gradle-version: wrapper
+    - name: Run benchmarks
+      run: ./gradlew -p snake-kmp-benchmarks benchmark --no-parallel ${{ matrix.additional-args }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: bench-results-${{ matrix.os }}
+        path: ${{ env.BENCHMARK_RESULTS }}/main/**/*.json
+  collect-benchmarks-results:
+    runs-on: ubuntu-latest
+    needs:
+      - run-benchmark
+    env:
+      RESULTS_DIR: bench-results
+    steps:
+    # without checkout step 'benchmark-action/github-action-benchmark' action won't work
+    - uses: actions/checkout@v4
+    - name: Download benchmark results
+      uses: actions/download-artifact@v4
+      with:
+        pattern: bench-results-*
+        path: ${{ env.RESULTS_DIR }}
+        merge-multiple: true
+    - name: Prepare and join benchmark reports
+      id: prep
+      run: |
+        for report in $(find ./${{ env.RESULTS_DIR }} -type f -name "*.json")
+        do
+          file_name=$(basename "$report")
+          platform="${file_name%.*}"
+          # Trim 'it.krzeminski.snakeyaml.engine.kmp.benchmark.' to make benchmark name more readable
+          jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"it.krzeminski.snakeyaml.engine.kmp.benchmark.\") | .params |= map_values(. |= match(\"data.+\"; \"g\").string) ]" $report > ${{ env.RESULTS_DIR }}/$platform.json
+        done
+        AGGREGATED_REPORT=aggregated.json
+        # Joined reports looks like this: [[{},{}], [{},{}]]
+        # We need to transform them into this: [{},{}]
+        ls ${{ env.RESULTS_DIR }}/*.json
+        jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT
+        echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT
+    - name: Store benchmark result
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: SnakeKMP benchmarks
+        tool: 'jmh'
+        output-file-path: ${{ steps.prep.outputs.report }}
+        comment-on-alert: true
+        summary-always: true
+        alert-threshold: '150%'
+        fail-threshold: '200%'
+        gh-repository: github.com/krzema12/snakeyaml-engine-kmp-benchmarks
+        github-token: ${{ secrets.PUBLISH_BENCHMARK_RESULTS }}
+        # Push and deploy GitHub pages branch automatically only if run in main repo and not in PR
+        auto-push: ${{ github.repository == 'krzema12/snakeyaml-engine-kmp' && github.event_name != 'pull_request' }}

build.gradle.kts

@@ -11,6 +11,10 @@ group = "it.krzeminski"
 version = "3.0.2-SNAPSHOT"
 description = "SnakeYAML Engine KMP"
 
+apiValidation {
+  ignoredProjects += listOf("snake-kmp-benchmarks")
+}
+
 kotlin {
     sourceSets {
         commonMain {

gradle/kotlin-js-store/yarn.lock

@@ -321,6 +321,14 @@ [email protected], base64id@~2.0.0:
   resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6"
   integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==
 
+benchmark@*:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
+  integrity sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==
+  dependencies:
+    lodash "^4.17.4"
+    platform "^1.3.3"
+
 binary-extensions@^2.0.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522"
@@ -1178,7 +1186,7 @@ locate-path@^6.0.0:
   dependencies:
     p-locate "^5.0.0"
 
-lodash@^4.17.15, lodash@^4.17.21:
+lodash@^4.17.15, lodash@^4.17.21, lodash@^4.17.4:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -1436,6 +1444,11 @@ pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
+platform@^1.3.3:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7"
+  integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==
+
 punycode@^2.1.0:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
@@ -1653,7 +1666,7 @@ [email protected]:
     iconv-lite "^0.6.3"
     source-map-js "^1.0.2"
 
-[email protected], source-map-support@~0.5.20:
+source-map-support@*, source-map-support@0.5.21, source-map-support@~0.5.20:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
   integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==

settings.gradle.kts

@@ -55,3 +55,6 @@ dependencyResolutionManagement {
 
 enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
 enableFeaturePreview("STABLE_CONFIGURATION_CACHE")
+
+
+include("snake-kmp-benchmarks")

snake-kmp-benchmarks/build.gradle.kts

@@ -0,0 +1,97 @@
+plugins {
+    kotlin("multiplatform")
+    kotlin("plugin.allopen") version "2.0.0"
+    id("org.jetbrains.kotlinx.benchmark") version "0.4.11"
+}
+
+allOpen {
+    // JMH requires all benchmark classes to be open
+    annotation("org.openjdk.jmh.annotations.State")
+}
+
+kotlin {
+    jvmToolchain(11)
+
+    //region JVM Targets
+    jvm()
+    //endregion
+
+    //region JS target
+    js(IR) {
+        nodejs()
+    }
+    //endregion
+
+    //region Native Targets
+    // According to https://kotlinlang.org/docs/native-target-support.html
+    // Tier 1
+    macosX64()
+    macosArm64()
+    iosSimulatorArm64()
+    iosX64()
+
+    // Tier 2
+    linuxX64()
+    linuxArm64()
+    watchosSimulatorArm64()
+    watchosX64()
+    watchosArm32()
+    watchosArm64()
+    tvosSimulatorArm64()
+    tvosX64()
+    tvosArm64()
+    iosArm64()
+
+    // Tier 3
+    mingwX64()
+    //endregion
+
+    sourceSets {
+        commonMain {
+            dependencies {
+                implementation(project.dependencies.platform("com.squareup.okio:okio-bom:3.9.0"))
+                implementation(projects.snakeyamlEngineKmp)
+                implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.11")
+                implementation("com.squareup.okio:okio")
+            }
+        }
+
+        jsMain {
+            dependencies {
+                implementation("net.thauvin.erik.urlencoder:urlencoder-lib:1.5.0") {
+                    because("https://github.com/Kotlin/kotlinx-benchmark/issues/185 - only compile dependnecies (declared as api) are using during benchmark compilation")
+                }
+                implementation("com.squareup.okio:okio-nodefilesystem")
+            }
+        }
+    }
+}
+
+benchmark {
+    configurations {
+        getByName("main") {
+            iterations = 10
+            iterationTime = 5
+            iterationTimeUnit = "s"
+            param(
+                "openAiYamlPath",
+                // Absolute path is required by JS target. Otherwise, file cannot be found
+                layout.projectDirectory
+                    .file("data/issues/kmp-issue-204-OpenAI-API.yaml")
+                    .asFile
+                    .absolutePath,
+            )
+        }
+    }
+    targets {
+        register("jvm")
+        register("js")
+        register("macosX64")
+        register("macosArm64")
+        register("iosX64")
+        register("iosArm64")
+        register("iosSimulatorArm64")
+        register("linuxX64")
+        register("mingwX64")
+    }
+}

snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.kt

@@ -0,0 +1,9 @@
+package it.krzeminski.snakeyaml.engine.kmp.benchmark
+
+import okio.FileSystem
+
+/**
+ * Because of JS (and Wasm) target it is required to have this method
+ * to access the file system in the common code
+ */
+expect fun fileSystem(): FileSystem

snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/LoadingTimeBenchmark.kt

@@ -0,0 +1,60 @@
+package it.krzeminski.snakeyaml.engine.kmp.benchmark
+
+import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings
+import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader
+import it.krzeminski.snakeyaml.engine.kmp.composer.Composer
+import it.krzeminski.snakeyaml.engine.kmp.constructor.BaseConstructor
+import it.krzeminski.snakeyaml.engine.kmp.constructor.StandardConstructor
+import it.krzeminski.snakeyaml.engine.kmp.parser.ParserImpl
+import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader
+import kotlinx.benchmark.*
+import okio.FileSystem
+import okio.Path.Companion.toPath
+import okio.buffer
+import okio.use
+
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS)
+class LoadingTimeBenchmark {
+    @Param("")
+    var openAiYamlPath: String = ""
+
+    private val loadSettings = LoadSettings.builder().build()
+
+    private lateinit var constructor: BaseConstructor
+
+    @Setup
+    fun setUp() {
+        constructor = StandardConstructor(loadSettings)
+    }
+
+    @Benchmark
+    fun loadsOpenAiSchema(): Map<*, *> {
+        return with(FILE_SYSTEM) {
+            openReadOnly(openAiYamlPath.toPath(normalize = true)).use { handle ->
+                handle.source().buffer().use { source ->
+                    // TODO: there is a Load class in JVM sources that can handle all of it
+                    //       but it is not available for common code.
+                    //       Probably, it should be moved from JVM sources to common sources.
+                    val reader = StreamReader(
+                        loadSettings = loadSettings,
+                        stream = YamlUnicodeReader(source),
+                    )
+                    val composer = Composer(
+                        settings = loadSettings,
+                        parser = ParserImpl(
+                            settings = loadSettings,
+                            reader = reader,
+                        )
+                    )
+                    constructor.constructSingleDocument(composer.getSingleNode()) as Map<*, *>
+                }
+            }
+        }
+    }
+
+    private companion object {
+        private val FILE_SYSTEM: FileSystem = fileSystem()
+    }
+}

snake-kmp-benchmarks/src/jsMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.js.kt

@@ -0,0 +1,10 @@
+package it.krzeminski.snakeyaml.engine.kmp.benchmark
+
+import okio.FileSystem
+import okio.NodeJsFileSystem
+
+/**
+ * Without JS and Wasm targets there is no need in this function
+ * but we can keep it so minimize change when those targets are added
+ */
+actual fun fileSystem(): FileSystem = NodeJsFileSystem

snake-kmp-benchmarks/src/jvmMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.jvm.kt

@@ -0,0 +1,5 @@
+package it.krzeminski.snakeyaml.engine.kmp.benchmark
+
+import okio.FileSystem
+
+actual fun fileSystem(): FileSystem = FileSystem.SYSTEM

snake-kmp-benchmarks/src/nativeMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.native.kt

@@ -0,0 +1,5 @@
+package it.krzeminski.snakeyaml.engine.kmp.benchmark
+
+import okio.FileSystem
+
+actual fun fileSystem(): FileSystem = FileSystem.SYSTEM