diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
deleted file mode 100644
index eb78ee7..0000000
--- a/.github/workflows/benchmark.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: Run Benchmarks
-
-on:
-  workflow_run:
-    workflows:
-      - Verify Build
-    types:
-      - completed
-
-jobs:
-  run_benchmarks:
-    name: Run Benchmarks
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run Criterion
-        uses: boa-dev/criterion-compare-action@v3
-        with:
-          branchName: ${{ github.base_ref }}
-          token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e136412..e6dacfd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -97,3 +97,19 @@ jobs:
       - name: Run Java Tests
         run: gradle test
         working-directory: ./java
+
+  benchmarks:
+    name: Run Benchmarks
+    runs-on: ubuntu-latest
+    needs: java-bindings
+    # We only want to run benchmarks in case of a pull request
+    if: ${{ github.event_name == 'pull_request' }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Criterion
+        uses: boa-dev/criterion-compare-action@v3
+        with:
+          branchName: ${{ github.base_ref }}
+          token: ${{ secrets.GITHUB_TOKEN }}