fix: attempt store and test benchmark logs

AztecProtocol · Aug 1, 2023 · aa2420f · aa2420f
1 parent b6b6eac
commit aa2420f
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 80 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -55,31 +55,6 @@ checkout: &checkout
       # Initialize submodules recursively (retry 10 times on failure)
       retry_10 git submodule update --init --recursive
 
-# This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format.
-# Initially it just fetches the latest version.
-benchmark_archive_setup: &benchmark_archive_setup
-  run:
-    name: "Add keys for getting the benchmark archive"
-    command: |
-      cd $HOME
-      mkdir -p .ssh
-      chmod 0700 .ssh
-      ssh-keyscan -t rsa github.com >> .ssh/known_hosts
-
-      # A read-write key for updating the repository.
-      echo "$GITHUB_BENCMARK_REPOSITORY_SSH_KEY" | base64 -d > .ssh/id_ed25519
-
-      # This allows github to discern wich key to use.
-      echo "Host github.com
-        Hostname github.com
-        IdentityFile=/root/.ssh/id_rsa
-
-      Host github.com-logs
-        Hostname github.com
-        IdentityFile=/root/.ssh/id_ed25519" > .ssh/config
-
-      chmod 0600 .ssh/id_ed25519
-      ssh-add .ssh/id_ed25519
 # Called setup_env to setup a bunch of global variables used throughout the rest of the build process.
 # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of
 # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds.
@@ -256,10 +231,9 @@ jobs:
           at: /tmp/test-logs
       - *checkout
       - *setup_env
-      - *benchmark_archive_setup
       - run:
           name: "Test"
-          command: store_test_benchmark_logs barretenberg-x86_64-linux-clang-assert
+          command: ./scripts/ci/store_test_benchmark_logs $AZTEC_GITHUB_TOKEN
 
   bb-js:
     machine:

diff --git a/circuits/cpp/barretenberg/.circleci/config.yml b/circuits/cpp/barretenberg/.circleci/config.yml
@@ -47,32 +47,6 @@ checkout: &checkout
       # Initialize submodules recursively
       git submodule update --init --recursive
 
-# This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format.
-# Initially it just fetches the latest version.
-benchmark_add_keys: &benchmark_add_keys
-  run:
-    name: "Add keys for getting the benchmark archive"
-    command: |
-      cd $HOME
-      mkdir -p .ssh
-      chmod 0700 .ssh
-      ssh-keyscan -t rsa github.com >> .ssh/known_hosts
-
-      # A read-write key for updating the repository.
-      echo "$GITHUB_BENCMARK_REPOSITORY_SSH_KEY" | base64 -d > .ssh/id_ed25519
-
-      # This allows github to discern wich key to use.
-      echo "Host github.com
-        Hostname github.com
-        IdentityFile=/root/.ssh/id_rsa
-
-      Host github.com-logs
-        Hostname github.com
-        IdentityFile=/root/.ssh/id_ed25519" > .ssh/config
-
-      chmod 0600 .ssh/id_ed25519
-      ssh-add .ssh/id_ed25519
-
 # Called setup_env to setup a bunch of global variables used throughout the rest of the build process.
 # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of
 # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds.
@@ -119,6 +93,7 @@ setup_aztec_commit: &setup_aztec_commit
       source "$BASH_ENV"
 
 # This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them
+# NOTE(AD): This is now obsolete as benchmarking has been moved to aztec-packages repo. Another pass may simplify this.
 save_logs: &save_logs
   persist_to_workspace:
     root: /tmp/test-logs
@@ -277,20 +252,6 @@ jobs:
   #         name: "Test"
   #         command: cond_spot_run_test_script ./scripts/bin-test.sh barretenberg-x86_64-linux-clang-assert
 
-  benchmark-aggregator:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - attach_workspace:
-          at: /tmp/test-logs
-      - *checkout
-      - *setup_env
-      - *benchmark_add_keys
-      - run:
-          name: "Test"
-          command: store_test_benchmark_logs barretenberg-x86_64-linux-clang-assert
-
   bb-js:
     machine:
       image: ubuntu-2004:202010-01
@@ -440,18 +401,6 @@ workflows:
       - stdlib-recursion-ultra-tests: *bb_test
       - join-split-tests: *bb_test
       # - bb-bin-tests: *bb_test
-      - benchmark-aggregator:
-          requires:
-            - barretenberg-tests
-            - stdlib-tests
-            - stdlib-recursion-turbo-tests
-            - stdlib-recursion-ultra-tests
-            - join-split-tests
-          filters:
-            branches:
-              only:
-                - master
-          <<: *defaults
       - bb-js:
           requires:
             - wasm-linux-clang

diff --git a/...arretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/aggregation_state.hpp b/...arretenberg/cpp/src/barretenberg/stdlib/recursion/aggregation_state/aggregation_state.hpp
@@ -5,6 +5,7 @@ namespace proof_system::plonk {
 namespace stdlib {
 namespace recursion {
 
+
 /**
  * Aggregation state contains the following:
  *   (P0, P1): the aggregated elements storing the verification results of proofs in the past
@@ -109,4 +110,4 @@ template <typename NCT> std::ostream& operator<<(std::ostream& os, aggregation_s
 
 } // namespace recursion
 } // namespace stdlib
-} // namespace proof_system::plonk
+} // namespace proof_system::plonk
diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+AZTEC_GITHUB_TOKEN=$1
+
+mkdir -p /tmp/csv
+export SAVED_TIMESTAMP=$(date +%s)
+export HUMAN_READABLE_TIME=$(date -u -d @${SAVED_TIMESTAMP})
+
+# Pick logs from test-logs and convert all information into CSV format including the current timestamp, branch, commit and tag information
+for file in $(ls /tmp/test-logs); do
+    echo $file
+    cat /tmp/test-logs/$file \
+        | grep "##BENCHMARK_INFO_PREFIX##" \
+        | sed "s/.*##BENCHMARK_INFO_PREFIX##\(.*\)##BENCHMARK_INFO_SUFFIX##.*/\1/" \
+        | sed "s/#/,/g" \
+        | sed "s_\(.*\)_$SAVED_TIMESTAMP,$HUMAN_READABLE_TIME,$BRANCH,$COMMIT_HASH,$COMMIT_TAG,\1_" \
+        >> /tmp/csv/new.csv
+done
+echo "Parsed from logs:"
+cat /tmp/csv/new.csv
+
+# We have lots of repeated entries, no need to put them into repository. Unfortunately build times differ a bit and uniq only works with space as separator
+cat /tmp/csv/new.csv \
+    | sort \
+    | sed "s_ _%_g" \
+    | sed "s_^\(.*\),\(.*\)\$_\2 \1_" \
+    | uniq -f 1 \
+    | sed "s_^\(.*\) \(.*\)\$_\2,\1_" \
+    | sed "s_%_ _g" >/tmp/csv/trimmed.csv
+
+# If there actually were any logs, update the information in the benchmark repository
+if [ -s /tmp/csv/trimmed.csv ]; then
+    cd /tmp
+
+    git clone --depth 1 https://$AZTEC_GITHUB_TOKEN:@github.com/AztecProtocol/benchmark-archive
+
+    cd benchmark-archive
+    git config --global user.name AztecBot
+    git config --global user.email [email protected]
+    cat /tmp/csv/trimmed.csv >>benchmarks.csv
+    git add benchmarks.csv
+    git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH"
+    git push
+fi