Add benchmarks (#218)

* Merge and add updated benchmarks * Update CI to use RunsOn * Format * Use ARM runner * Use default thresholds for now
vapor · Nov 18, 2024 · 9226a2f · 9226a2f
1 parent 6f745e9
commit 9226a2f
Show file tree

Hide file tree

Showing 5 changed files with 350 additions and 0 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,43 @@
+name: Benchmark PR vs main
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  benchmark-delta-linux:
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=2cpu-linux-arm64
+    container: swift:jammy
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: jemalloc dependency
+        run: apt-get update && apt-get install -y libjemalloc-dev
+      - name: Fix Git config
+        run: |
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+      - name: Run benchmarks for PR branch
+        continue-on-error: true
+        run: |
+          swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update pull_request --no-progress --quiet
+      - name: Run benchmarks for 'main' branch
+        run: |
+          git stash
+          git checkout main
+          swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update main --no-progress --quiet
+      - name: Compare benchmarks
+        continue-on-error: true
+        run: |
+          date >> "${GITHUB_STEP_SUMMARY}"
+          swift package -c release --package-path Benchmarks benchmark baseline check main pull_request --format markdown >> "${GITHUB_STEP_SUMMARY}"
+      - name: Get formatted date
+        id: get-date
+        run: echo "date=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_OUTPUT
+      - uses: thollander/actions-comment-pull-request@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          message: ${{ format('[PR benchmark comparison with main on ubuntu-latest run at {0}]({1}/{2}/actions/runs/{3})', steps.get-date.outputs.date, github.server_url, github.repository, github.run_id) }}
+          comment_tag: 'PR benchmark comparison Linux'
diff --git a/Benchmarks/Benchmarks/Signing/Signing.swift b/Benchmarks/Benchmarks/Signing/Signing.swift
@@ -0,0 +1,86 @@
+import Benchmark
+import Foundation
+import JWTKit
+
+let benchmarks = {
+    Benchmark("ES256") { benchmark in
+        let key = ES256PrivateKey()
+        let keyCollection = JWTKeyCollection()
+        keyCollection.add(ecdsa: key)
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.sign(payload)
+        }
+    }
+
+    Benchmark("RSA") { benchmark in
+        let key = try Insecure.RSA.PrivateKey(pem: rsaPrivateKey)
+        let keyCollection = JWTKeyCollection()
+        keyCollection.add(rsa: key, digestAlgorithm: .sha256)
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.sign(payload)
+        }
+    }
+
+    Benchmark("EdDSA") { benchmark in
+        let key = try EdDSA.PrivateKey()
+        let keyCollection = JWTKeyCollection()
+        keyCollection.add(eddsa: key)
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.sign(payload)
+        }
+    }
+}
+
+struct Payload: JWTPayload {
+    let name: String
+    let admin: Bool
+
+    func verify(using signer: some JWTAlgorithm) async throws {
+        // nothing to verify
+    }
+}
+
+let payload = Payload(name: "Kyle", admin: true)
+
+let ecdsaPrivateKey = """
+    -----BEGIN PRIVATE KEY-----
+    MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQg2sD+kukkA8GZUpmm
+    jRa4fJ9Xa/JnIG4Hpi7tNO66+OGgCgYIKoZIzj0DAQehRANCAATZp0yt0btpR9kf
+    ntp4oUUzTV0+eTELXxJxFvhnqmgwGAm1iVW132XLrdRG/ntlbQ1yzUuJkHtYBNve
+    y+77Vzsd
+    -----END PRIVATE KEY-----
+    """
+
+let rsaPrivateKey = """
+    -----BEGIN PRIVATE KEY-----
+    MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDL8W1D9w5zHpmD
+    JqpTngIRJ+Sm21e42cRnTudhdejzKUiJQWkHSvQV5yC/+0iEXUsUJYEdSyrKhJFD
+    PT+IFGdjIiwb7IX+rUreWXlD/YYBL3/byMG4kYoO4oiPp2A+WvfeyLpuN549OXhk
+    7o5kXEZjKfjHTfmnAbCMoYW5BEpiHQC3HAeJZ5EiwAn8HZn5UY6lxJcf7H9hR83x
+    D0W7IZTNyxUu4aLNuihFIxJKgP/L/y95Y6ddZsyyHQopM43/7JOYBwufa07MWaxi
+    AdBdq1bR/ZeOt2aZaXhV+J6QUoUO8Z6fG6b2cQmvMgk4ybqoeciLII0DfFsyqavu
+    ip4hRr59AgMBAAECggEAUIw994XwMw922hG/W98gOd5jtHMVJnD73UGQqTGEm+VG
+    PM+Ux8iWtr/ec3Svo3elW4OkhwlVET9ikAf0u64zVzf769ty4K9YzpDQEEZlUrqL
+    6SZVPKxetppKDVKx9G7BT0BAQZ+947h7EIIXwxOeyTOeijkFzSwhqqlwwy4qoqzV
+    FTQS20QHE62hxzwuS5HBqw8ds183qAg9NbzR0Cp4za9qTiBB6C8KEcLqeatO+q+d
+    VCDsJcAMZOvW14N6BozKgbQ/WXZQ/3kNUPBndZLzzqaILFNmB1Zf2DVVJ9gU7+EK
+    xOac60StIfG81NllCTBrmRVq8yitNqwmutHMlxrIkQKBgQDvp39MkEHtNunFGkI5
+    R8IB5BZjtx5OdRBKkmPasmNU8U0XoQAJUKY/9piIpCtRi87tMXv8WWmlbULi66pu
+    4BnMIisw78xlIWRZTSizFrkFcEoVgEnbZBtSrOg/J5PAcjLEGCQoAdmMXAekR2/m
+    htv7FPijHPNUjyIFLaxwjl9izwKBgQDZ2mQeKNRHjIb5ZBzB0ZCvUy2y4+kaLrhZ
+    +CWMN1flL4dd1KuZKvCEfHY9kWOjqw6XneN4yT0aPmbBft4fihiiNW0Sm8i+fSpy
+    g0klw2HJl49wnwctBpRgTdMKGo9n14OGeu0xKOAy7I4j1tKrUXiRWnP9R583Ti7c
+    w7YHgdHM8wKBgEV147SaPzF08A6bzMPzY2zO4hpmsdcFoQIsKdryR04QXkrR9EO+
+    52C0pYM9Kf0Jq6Ed7ZS3iaJT58YDjjNyqqd648/cQP6yzfYAIiK+HERSRnay5zU6
+    b5zn1qyvWOi3cLVbVedumdJPvjtEJU/ImKvOaT5FntVMYwzjLw60hTsLAoGAZJnt
+    UeAY51GFovUQMpDL96q5l7qXknewuhtVe4KzHCrun+3tsDWcDBJNp/DTymjbvDg1
+    KzoC9XOLkB8+A+KJrZ5uWAGImi7Cw07NIJsxNR7AJonJjolTS4Wkxy2su49SNW/e
+    yKzPm7SRjwtNDb/5pWXX2kaQx8Fa8qeOD7lrYPECgYAwQ6o0vYmr+L1tOZZgMVv9
+    Jusa8beVUH5hyduJjmxbYOtFTkggAozdx7rs4BgyRsmDlV48cEmcVf/7IH4gMJLb
+    O+bbERwCYUChe+piANhnwfwDHzbRd8mmQus54P06X7bWu6Rmi7gbQGVN/Z6VhbIm
+    D2cOo0w4bk/3yb01xz1MEw==
+    -----END PRIVATE KEY-----
+    """
+
+let eddsaPublicKeyBase64Url = "0ZcEvMCSYqSwR8XIkxOoaYjRQSAO8frTMSCpNbUl4lE"
+let eddsaPrivateKeyBase64Url = "d1H3_dcg0V3XyAuZW2TE5Z3rhY20M-4YAfYu_HUQd8w"
diff --git a/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift b/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift
@@ -0,0 +1,109 @@
+import Benchmark
+import Foundation
+import JWTKit
+
+let benchmarks = {
+    Benchmark("ES256 Generated") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let key = ES256PrivateKey()
+            let keyCollection = JWTKeyCollection()
+            keyCollection.add(ecdsa: key)
+            let token = try await keyCollection.sign(payload)
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("ES256 PEM") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let key = try ES256PrivateKey(pem: ecdsaPrivateKey)
+            let keyCollection = JWTKeyCollection()
+            keyCollection.add(ecdsa: key)
+            let token = try await keyCollection.sign(payload)
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("RSA PEM") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let key = try Insecure.RSA.PrivateKey(pem: rsaPrivateKey)
+            let keyCollection = JWTKeyCollection()
+            keyCollection.add(rsa: key, digestAlgorithm: .sha256)
+            let token = try await keyCollection.sign(payload)
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("EdDSA Generated") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let key = try EdDSA.PrivateKey()
+            let keyCollection = JWTKeyCollection()
+            keyCollection.add(eddsa: key)
+            let token = try await keyCollection.sign(payload)
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("EdDSA Coordinates") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let key = try EdDSA.PrivateKey(d: eddsaPrivateKeyBase64Url, curve: .ed25519)
+            let keyCollection = JWTKeyCollection()
+            keyCollection.add(eddsa: key)
+            let token = try await keyCollection.sign(payload)
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+}
+
+struct Payload: JWTPayload {
+    let name: String
+    let admin: Bool
+
+    func verify(using signer: some JWTAlgorithm) async throws {
+        // nothing to verify
+    }
+}
+
+let payload = Payload(name: "Kyle", admin: true)
+
+let ecdsaPrivateKey = """
+    -----BEGIN PRIVATE KEY-----
+    MIGTAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBHkwdwIBAQQg2sD+kukkA8GZUpmm
+    jRa4fJ9Xa/JnIG4Hpi7tNO66+OGgCgYIKoZIzj0DAQehRANCAATZp0yt0btpR9kf
+    ntp4oUUzTV0+eTELXxJxFvhnqmgwGAm1iVW132XLrdRG/ntlbQ1yzUuJkHtYBNve
+    y+77Vzsd
+    -----END PRIVATE KEY-----
+    """
+
+let rsaPrivateKey = """
+    -----BEGIN PRIVATE KEY-----
+    MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDL8W1D9w5zHpmD
+    JqpTngIRJ+Sm21e42cRnTudhdejzKUiJQWkHSvQV5yC/+0iEXUsUJYEdSyrKhJFD
+    PT+IFGdjIiwb7IX+rUreWXlD/YYBL3/byMG4kYoO4oiPp2A+WvfeyLpuN549OXhk
+    7o5kXEZjKfjHTfmnAbCMoYW5BEpiHQC3HAeJZ5EiwAn8HZn5UY6lxJcf7H9hR83x
+    D0W7IZTNyxUu4aLNuihFIxJKgP/L/y95Y6ddZsyyHQopM43/7JOYBwufa07MWaxi
+    AdBdq1bR/ZeOt2aZaXhV+J6QUoUO8Z6fG6b2cQmvMgk4ybqoeciLII0DfFsyqavu
+    ip4hRr59AgMBAAECggEAUIw994XwMw922hG/W98gOd5jtHMVJnD73UGQqTGEm+VG
+    PM+Ux8iWtr/ec3Svo3elW4OkhwlVET9ikAf0u64zVzf769ty4K9YzpDQEEZlUrqL
+    6SZVPKxetppKDVKx9G7BT0BAQZ+947h7EIIXwxOeyTOeijkFzSwhqqlwwy4qoqzV
+    FTQS20QHE62hxzwuS5HBqw8ds183qAg9NbzR0Cp4za9qTiBB6C8KEcLqeatO+q+d
+    VCDsJcAMZOvW14N6BozKgbQ/WXZQ/3kNUPBndZLzzqaILFNmB1Zf2DVVJ9gU7+EK
+    xOac60StIfG81NllCTBrmRVq8yitNqwmutHMlxrIkQKBgQDvp39MkEHtNunFGkI5
+    R8IB5BZjtx5OdRBKkmPasmNU8U0XoQAJUKY/9piIpCtRi87tMXv8WWmlbULi66pu
+    4BnMIisw78xlIWRZTSizFrkFcEoVgEnbZBtSrOg/J5PAcjLEGCQoAdmMXAekR2/m
+    htv7FPijHPNUjyIFLaxwjl9izwKBgQDZ2mQeKNRHjIb5ZBzB0ZCvUy2y4+kaLrhZ
+    +CWMN1flL4dd1KuZKvCEfHY9kWOjqw6XneN4yT0aPmbBft4fihiiNW0Sm8i+fSpy
+    g0klw2HJl49wnwctBpRgTdMKGo9n14OGeu0xKOAy7I4j1tKrUXiRWnP9R583Ti7c
+    w7YHgdHM8wKBgEV147SaPzF08A6bzMPzY2zO4hpmsdcFoQIsKdryR04QXkrR9EO+
+    52C0pYM9Kf0Jq6Ed7ZS3iaJT58YDjjNyqqd648/cQP6yzfYAIiK+HERSRnay5zU6
+    b5zn1qyvWOi3cLVbVedumdJPvjtEJU/ImKvOaT5FntVMYwzjLw60hTsLAoGAZJnt
+    UeAY51GFovUQMpDL96q5l7qXknewuhtVe4KzHCrun+3tsDWcDBJNp/DTymjbvDg1
+    KzoC9XOLkB8+A+KJrZ5uWAGImi7Cw07NIJsxNR7AJonJjolTS4Wkxy2su49SNW/e
+    yKzPm7SRjwtNDb/5pWXX2kaQx8Fa8qeOD7lrYPECgYAwQ6o0vYmr+L1tOZZgMVv9
+    Jusa8beVUH5hyduJjmxbYOtFTkggAozdx7rs4BgyRsmDlV48cEmcVf/7IH4gMJLb
+    O+bbERwCYUChe+piANhnwfwDHzbRd8mmQus54P06X7bWu6Rmi7gbQGVN/Z6VhbIm
+    D2cOo0w4bk/3yb01xz1MEw==
+    -----END PRIVATE KEY-----
+    """
+
+let eddsaPublicKeyBase64Url = "0ZcEvMCSYqSwR8XIkxOoaYjRQSAO8frTMSCpNbUl4lE"
+let eddsaPrivateKeyBase64Url = "d1H3_dcg0V3XyAuZW2TE5Z3rhY20M-4YAfYu_HUQd8w"
diff --git a/Benchmarks/Benchmarks/Verifying/Verifying.swift b/Benchmarks/Benchmarks/Verifying/Verifying.swift
@@ -0,0 +1,63 @@
+import Benchmark
+import Foundation
+import JWTKit
+
+let benchmarks = {
+    Benchmark("ES256") { benchmark in
+        let pem = """
+            -----BEGIN PUBLIC KEY-----
+            MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEVs/o5+uQbTjL3chynL4wXgUg2R9
+            q9UU8I5mEovUf86QZ7kOBIjJwqnzD1omageEHWwHdBO6B+dFabmdT9POxg==
+            -----END PUBLIC KEY-----
+            """
+        let token =
+            "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiSm9obiBEb2UiLCJhZG1pbiI6dHJ1ZX0.vY4BbTLWcVbA4sS_EnaSV-exTZT3mRpH6JNc5C7XiUDA1PfbTO6LdObMFYPEcKZMydfHy6SJz1eJySq2uYBLAA"
+        let key = try ES256PublicKey(pem: pem)
+        let keyCollection = JWTKeyCollection().add(ecdsa: key)
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("RS256") { benchmark in
+        let pem = """
+            -----BEGIN PUBLIC KEY-----
+            MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo
+            4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u
+            +qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh
+            kd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ
+            0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg
+            cKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc
+            mwIDAQAB
+            -----END PUBLIC KEY-----
+            """
+        let token =
+            "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiSm9obiBEb2UiLCJhZG1pbiI6dHJ1ZX0.AJCCTYfWKXUPf6dztCbYoVdB4E7FjvmD9WogWtZv20mL6Urt-fFU2DntUIBmoFXGJ5424ubslBt6sk5yBxS_DMnXKfhj2R-J6xDkT0vlldfFzrrDSQEIsbiErfmfVK40Fr9MW4XFKBZdKEI6X35SCmLx9s5RsQCejIo9pdHyx6jGbfXqN_04RWprx6pcqqOn6_Gm4jkofAd1duZ_IUlojUBKX56OgEweR_2glQ8uumb-oklwYl699ZF9DmTKRHHE2RMMT2QVy0RWl1R7HIvUOY0EzxeuKDiiOQC1bFxIH_EZpqBp5FbfW0iemK6Tm5v7_8UzEOmIVrFUIpqxwrI3Sg"
+        let key = try Insecure.RSA.PublicKey(pem: pem)
+        let keyCollection = JWTKeyCollection().add(rsa: key, digestAlgorithm: .sha256)
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+
+    Benchmark("EdDSA") { benchmark in
+        let eddsaPublicKeyBase64Url = "0ZcEvMCSYqSwR8XIkxOoaYjRQSAO8frTMSCpNbUl4lE"
+        let eddsaPrivateKeyBase64Url = "d1H3_dcg0V3XyAuZW2TE5Z3rhY20M-4YAfYu_HUQd8w"
+        let keyCollection = try JWTKeyCollection()
+            .add(eddsa: EdDSA.PrivateKey(d: eddsaPrivateKeyBase64Url, curve: .ed25519))
+        let token =
+            "eyJ0eXAiOiJKV1QiLCJhbGciOiJFZERTQSJ9.eyJuYW1lIjoiSm9obiBEb2UiLCJhZG1pbiI6dHJ1ZX0.UzwX3znh-Ne180bcZqjbTk8Dx-BmHR_IL6b8wR2K2AG5f8ny-vThSL0b9IUvR8ybDkUiubpqlKKQXrRtbKQzAA"
+        for _ in benchmark.scaledIterations {
+            _ = try await keyCollection.verify(token, as: Payload.self)
+        }
+    }
+}
+
+struct Payload: JWTPayload {
+    let name: String
+    let admin: Bool
+
+    func verify(using signer: some JWTAlgorithm) async throws {
+        // nothing to verify
+    }
+}
diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift
@@ -0,0 +1,49 @@
+// swift-tools-version:5.9
+
+import PackageDescription
+
+let package = Package(
+    name: "benchmarks",
+    platforms: [
+        .macOS(.v13)
+    ],
+    dependencies: [
+        .package(path: "../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.22.0"),
+    ],
+    targets: [
+        .executableTarget(
+            name: "Signing",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "JWTKit", package: "jwt-kit"),
+            ],
+            path: "Benchmarks/Signing",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+            ]
+        ),
+        .executableTarget(
+            name: "Verifying",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "JWTKit", package: "jwt-kit"),
+            ],
+            path: "Benchmarks/Verifying",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+            ]
+        ),
+        .executableTarget(
+            name: "TokenLifecycle",
+            dependencies: [
+                .product(name: "Benchmark", package: "package-benchmark"),
+                .product(name: "JWTKit", package: "jwt-kit"),
+            ],
+            path: "Benchmarks/TokenLifecycle",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+            ]
+        ),
+    ]
+)