run benchmarks on dedicated CI runner

.github/workflows/benchmark.yml

@@ -1,5 +1,5 @@
 
-name: Run benchmarks
+name: Tests
 
 on:
   push:
@@ -12,8 +12,8 @@ on:
       - '**'
 
 jobs:
-  max-cost-checks:
-    name: Cost checks
+  tests:
+    name: Unit Tests
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -70,18 +70,53 @@ jobs:
       run: |
         pytest tests
 
+  benchmarks:
+    name: Generator performance
+    runs-on: benchmark
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+
+    - uses: chia-network/actions/setup-python@main
+      name: Install Python ${{ matrix.python-version }}
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+
+    - name: setup venv
+      run: |
+        python -m venv venv
+
+    - uses: chia-network/actions/activate-venv@main
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install maturin colorama clvm_tools
+        rustup target add x86_64-unknown-linux-musl
+
+    - name: Build
+      env:
+        CC: gcc
+      run: |
+        maturin develop --release -m wheel/Cargo.toml
+
     - name: test generators
-      if: matrix.os == 'ubuntu-latest'
-      # TODO: This should happen everywhere but the benchmarking part is not passing
-      #       on all platforms so this can be revisited separately.
       run: |
         cd tests
         ./test-generators.py
 
     - name: Run cost checks
-      if: matrix.os == 'ubuntu-latest'
-      # TODO: This should happen everywhere but the benchmarking part is not passing
-      #       on all platforms so this can be revisited separately.
       run: |
         cd tests
         ./generate-programs.py

tests/test-generators.py

@@ -71,7 +71,7 @@ def parse_output(result, error_code) -> str:
             limit = 6
 
         if run_time > limit or run_time2 > limit:
-            print("run-time exceeds limit!")
+            print(f"run-time exceeds limit ({limit})!")
             failed = 1
 
 print(f"returning {failed}")