fix: Catch2 v.3.5.0 changed output format

.github/workflows/catch2.yml

@@ -10,7 +10,7 @@ permissions:
 
 jobs:
   benchmark:
-    name: Run C++ benchmark example
+    name: Run Catch2 C++ Benchmark Framework example (v3.x)
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
@@ -25,7 +25,7 @@ jobs:
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          name: Catch2 Benchmark
+          name: Catch2 Benchmark (v3)
           tool: 'catch2'
           output-file-path: examples/catch2/benchmark_result.txt
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -34,12 +34,12 @@ jobs:
           alert-threshold: '200%'
           comment-on-alert: true
           fail-on-alert: true
-          alert-comment-cc-users: '@bernedom,@ktrz'
+          alert-comment-cc-users: '@ktrz'
 
       - name: Store benchmark result - separate results repo
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          name: Catch2 Benchmark
+          name: Catch2 Benchmark (v3)
           tool: 'catch2'
           output-file-path: examples/catch2/benchmark_result.txt
           github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
@@ -48,5 +48,5 @@ jobs:
           alert-threshold: '200%'
           comment-on-alert: true
           fail-on-alert: true
-          alert-comment-cc-users: '@bernedom,@ktrz'
+          alert-comment-cc-users: '@ktrz'
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

.github/workflows/catch2_v2.yml

@@ -0,0 +1,52 @@
+name: Catch2 C++ Example
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run Catch2 C++ Benchmark Framework example (v2.x)
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and run benchmarks with Catch2
+        run: |
+          cd examples/catch2_v2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench | tee ../benchmark_result.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Catch2 Benchmark
+          tool: 'catch2'
+          output-file-path: examples/catch2_v2/benchmark_result.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@ktrz'
+
+      - name: Store benchmark result - separate results repo
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Catch2 Benchmark
+          tool: 'catch2'
+          output-file-path: examples/catch2_v2/benchmark_result.txt
+          github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@ktrz'
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

.github/workflows/ci-results-repo.yml

@@ -79,8 +79,8 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository'
 
-  catch2-framework:
-    name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
+  catch2-v2-framework:
+    name: Run Catch2 C++ Benchmark Framework example (v2.x) - github.com/benchmark-action/github-action-benchmark-results
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
@@ -92,7 +92,7 @@ jobs:
       - run: npm run build
       - name: Run benchmark
         run: |
-          cd examples/catch2
+          cd examples/catch2_v2
           mkdir build && cd build
           cmake -DCMAKE_BUILD_TYPE=Release ..
           cmake --build . --config Release
@@ -112,12 +112,51 @@ jobs:
         with:
           name: Catch2 Benchmark
           tool: 'catch2'
-          output-file-path: examples/catch2/benchmark_result.txt
+          output-file-path: examples/catch2_v2/benchmark_result.txt
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository'
 
+  catch2-v3-framework:
+    name: Run Catch2 C++ Benchmark Framework example (v3.x) - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run build
+      - name: Run benchmark
+        run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
+
+      - uses: actions/checkout@v4
+        with:
+          repository: benchmark-action/github-action-benchmark-results
+          ref: 'gh-pages'
+          path: 'dist/other-repo'
+      - name: Save previous data.js
+        run: |
+          cp ./dist/other-repo/dev/bench/data.js before_data.js
+
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Catch2 Benchmark (v3)
+          tool: 'catch2'
+          output-file-path: examples/catch2/benchmark_result.txt
+          fail-on-alert: true
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' './benchmark-data-repository'
+
   cpp-framework:
     name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
     runs-on: ubuntu-20.04

.github/workflows/ci.yml

@@ -75,8 +75,8 @@ jobs:
           comment-on-alert: true
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark'
 
-  catch2-framework:
-    name: Run Catch2 C++ Benchmark Framework example
+  catch2-v2-framework:
+    name: Run Catch2 C++ Benchmark Framework example (v2.x)
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
       - run: npm run build
       - name: Run benchmark
         run: |
-          cd examples/catch2
+          cd examples/catch2_v2
           mkdir build && cd build
           cmake -DCMAKE_BUILD_TYPE=Release ..
           cmake --build . --config Release
@@ -104,14 +104,51 @@ jobs:
         with:
           name: Catch2 Benchmark
           tool: 'catch2'
-          output-file-path: examples/catch2/benchmark_result.txt
+          output-file-path: examples/catch2_v2/benchmark_result.txt
           skip-fetch-gh-pages: true
           fail-on-alert: true
           summary-always: true
           github-token: ${{ secrets.GITHUB_TOKEN }}
           comment-on-alert: true
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
 
+  catch2-v3-framework:
+    name: Run Catch2 C++ Benchmark Framework example (v3.x)
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run build
+      - name: Run benchmark
+        run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
+      - name: Save previous data.js
+        run: |
+          git fetch origin gh-pages
+          git checkout gh-pages
+          cp ./dev/bench/data.js before_data.js
+          git checkout -
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Catch2 Benchmark (v3)
+          tool: 'catch2'
+          output-file-path: examples/catch2/benchmark_result.txt
+          skip-fetch-gh-pages: true
+          fail-on-alert: true
+          summary-always: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)'
+
   cpp-framework:
     name: Run Google C++ Benchmark Framework example
     runs-on: ubuntu-20.04

CHANGELOG.md

@@ -1,4 +1,5 @@
 ## Unreleased
+- **fix** Catch2 v.3.5.0 changed output format [#241]
 
 <a name="v1.20.2"></a>
 # [v1.20.2](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.20.2) - 19 May 2024

examples/catch2/CMakeLists.txt

@@ -6,7 +6,7 @@ include(FetchContent)
 FetchContent_Declare(
   catch2
   GIT_REPOSITORY https://github.com/catchorg/Catch2.git
-  GIT_TAG v2.11.0)
+  GIT_TAG v3.6.0)
 
 FetchContent_GetProperties(catch2)
 if(NOT catch2_POPULATED)
@@ -16,7 +16,7 @@ endif()
 
 add_executable(${PROJECT_NAME})
 target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp)
-target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain)
 
 target_compile_options(
   ${PROJECT_NAME}

examples/catch2/catch2_bench.cpp

@@ -1,6 +1,6 @@
 #include "fib.hpp"
-#define CATCH_CONFIG_MAIN
-#include <catch2/catch.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/benchmark/catch_benchmark.hpp>
 
 TEST_CASE("Fibonacci") {
 

examples/catch2_v2/.gitignore

@@ -0,0 +1 @@
+build/

examples/catch2_v2/CMakeLists.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required(VERSION 3.12)
+project("Catch2_bench")
+
+include(FetchContent)
+
+FetchContent_Declare(
+  catch2
+  GIT_REPOSITORY https://github.com/catchorg/Catch2.git
+  GIT_TAG v2.13.10)
+
+FetchContent_GetProperties(catch2)
+if(NOT catch2_POPULATED)
+  FetchContent_Populate(catch2)
+  add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR})
+endif()
+
+add_executable(${PROJECT_NAME})
+target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+target_compile_options(
+  ${PROJECT_NAME}
+  PRIVATE
+    $<$<CXX_COMPILER_ID:MSVC>:/DCATCH_CONFIG_ENABLE_BENCHMARKING>
+    $<$<OR:$<CXX_COMPILER_ID:GNU>,$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>>:-DCATCH_CONFIG_ENABLE_BENCHMARKING>
+)

examples/catch2_v2/README.md

@@ -0,0 +1,68 @@
+C++ example for benchmarking with [Catch2 Framework][tool]
+====================================================================
+
+- [Workflow for this example](../../.github/workflows/catch2.yml)
+- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22)
+- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/)
+
+This directory shows how to use [`github-action-benchmark`][action] with [Catch2 Framework][tool].
+
+
+
+## Run benchmarks
+
+Official documentation for usage of Catch2 Framework can be found in its repository:
+
+https://github.com/catchorg/Catch2
+
+Since Catch2 is a header-only test framework, you don't need to build it in advance.
+Download and put the headers in your `include` directory and write your benchmarks.
+
+```cpp
+#define CATCH_CONFIG_MAIN
+#include <catch2/catch.hpp>
+
+TEST_CASE("Fibonacci") {
+  // now let's benchmark:
+  BENCHMARK("Some benchmark") {
+      // Your benchmark goes here
+  };
+}
+```
+
+Build the source with C++ compiler and run the built executable to get the benchmark output.
+Ensure to use `console` reporter for this. `xml` reporter may be supported in the future.
+
+
+
+## Process benchmark results
+
+Store the benchmark results with step using the action. Please set `catch2` to `tool` input.
+
+```yaml
+- name: Store benchmark result
+  uses: benchmark-action/github-action-benchmark@v1
+  with:
+    tool: 'catch2'
+    output-file-path: benchmark_result.json
+```
+
+Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.
+
+
+
+## Run this example
+
+To try this example, please use [cmake](./CMakeLists.txt) and `clang++`.
+
+```sh
+$ mkdir build
+$ cd build
+$ cmake -DCMAKE_BUILD_TYPE=Release ..
+$ cmake --build . --config Release
+```
+
+This will create `Catch2_bench` executable. The results are output to stdout.
+
+[tool]: https://github.com/catchorg/Catch2
+[action]: https://github.com/benchmark-action/github-action-benchmark

examples/catch2_v2/catch2_bench.cpp

@@ -0,0 +1,11 @@
+#include "fib.hpp"
+#define CATCH_CONFIG_MAIN
+#include <catch2/catch.hpp>
+
+TEST_CASE("Fibonacci") {
+
+  // now let's benchmark:
+  BENCHMARK("Fibonacci 10") { return fib(10); };
+
+  BENCHMARK("Fibonacci 20") { return fib(20); };
+}

examples/catch2_v2/fib.hpp

@@ -0,0 +1,11 @@
+#if !defined FIB_HPP_INCLUDED
+#define FIB_HPP_INCLUDED
+
+int fib(int const i) {
+  if (i <= 1) {
+    return 1;
+  }
+  return fib(i - 2) + fib(i - 1);
+}
+
+#endif // FIB_HPP_INCLUDED