From 797d0ca4abb396b6325a8159ca3be248c16c6b97 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 2 Oct 2024 12:28:28 -0400 Subject: [PATCH 1/2] refactor(deploy): use docker run instead of helm test, metrics dashboard scripts (#8926) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - helm test was very opaque, everyone agreed more straightforward testing was a better fit in this case (I guess helm test is good if you want to run a lot of temporary k8s config) Note: This currently hangs with: ``` ... aztec:js:contract_interaction [DEBUG] Sent deployment tx of SchnorrAccount contract with deployment address 0x08822d7355bc089eaa45c394b987b17e0c15d309fa48dc469d081b1987137f08 +0ms FAIL src/spartan/transfer.test.ts (316.831 s) token transfer test ✕ can get info (4 ms) ✕ can transfer 1 token privately and publicly ● token transfer test › can get info (JSON-RPC PROPAGATED) fetch failed 49 | return resp.data; 50 | } else if (resp.status >= 400 && resp.status < 500) { > 51 | throw new NoRetryError('(JSON-RPC PROPAGATED) ' + resp.data.error.message); | ^ 52 | } else { 53 | throw new Error('(JSON-RPC PROPAGATED) ' + resp.data.error.message); 54 | } at axiosFetch (../../aztec.js/src/rpc_clients/node/index.ts:51:11) at retry (../../foundation/src/retry/index.ts:56:14) at fetch (../../aztec.js/src/rpc_clients/node/index.ts:67:12) at request (../../foundation/src/json-rpc/client/json_rpc_client.ts:116:17) at ../../aztec.js/src/contract/sent_tx.ts:130:27 at retryUntil (../../foundation/src/retry/index.ts:88:20) at DeployAccountSentTx.waitForReceipt (../../aztec.js/src/contract/sent_tx.ts:128:12) at DeployAccountSentTx.wait (../../aztec.js/src/contract/sent_tx.ts:80:21) at DeployAccountSentTx.wait (../../aztec.js/src/account_manager/deploy_account_sent_tx.ts:38:21) at async Promise.all (index 2) at fixtures/snapshot_manager.ts:507:5 at Object. (spartan/transfer.test.ts:66:29) ● token transfer test › can transfer 1 token privately and publicly (JSON-RPC PROPAGATED) fetch failed 49 | return resp.data; 50 | } else if (resp.status >= 400 && resp.status < 500) { > 51 | throw new NoRetryError('(JSON-RPC PROPAGATED) ' + resp.data.error.message); | ^ 52 | } else { 53 | throw new Error('(JSON-RPC PROPAGATED) ' + resp.data.error.message); 54 | } at axiosFetch (../../aztec.js/src/rpc_clients/node/index.ts:51:11) at retry (../../foundation/src/retry/index.ts:56:14) at fetch (../../aztec.js/src/rpc_clients/node/index.ts:67:12) at request (../../foundation/src/json-rpc/client/json_rpc_client.ts:116:17) at ../../aztec.js/src/contract/sent_tx.ts:130:27 at retryUntil (../../foundation/src/retry/index.ts:88:20) at DeployAccountSentTx.waitForReceipt (../../aztec.js/src/contract/sent_tx.ts:128:12) at DeployAccountSentTx.wait (../../aztec.js/src/contract/sent_tx.ts:80:21) at DeployAccountSentTx.wait (../../aztec.js/src/account_manager/deploy_account_sent_tx.ts:38:21) at async Promise.all (index 2) at fixtures/snapshot_manager.ts:507:5 at Object. (spartan/transfer.test.ts:66:29) ``` Bundled: - Helper scripts for getting a metrics dashboard with elasticsearch for nicer log searching (need to log in with user/pass: elastic/admin and port forward to local machine if on mainframe) --- scripts/run_on_tester | 1 + .../templates/tests/run-tests.yaml | 25 ------------- spartan/aztec-network/values.yaml | 8 +---- spartan/scripts/forward_metrics_dashboard.sh | 4 +++ spartan/scripts/setup_local_k8s.sh | 6 +++- spartan/scripts/setup_metrics_dashboard.sh | 10 ++++++ yarn-project/end-to-end/Earthfile | 4 +-- .../end-to-end/scripts/network_test.sh | 35 ++++++++++++++++--- 8 files changed, 54 insertions(+), 39 deletions(-) delete mode 100644 spartan/aztec-network/templates/tests/run-tests.yaml create mode 100755 spartan/scripts/forward_metrics_dashboard.sh create mode 100755 spartan/scripts/setup_metrics_dashboard.sh diff --git a/scripts/run_on_tester b/scripts/run_on_tester index 1443ebc67c0..fda1b756998 100755 --- a/scripts/run_on_tester +++ b/scripts/run_on_tester @@ -22,4 +22,5 @@ ENV_VARS=" # Format the environment variables for the SSH command ENV_EXPORTS=$(printf 'export %s; ' $ENV_VARS) +echo "Running on tester at IP: $SPOT_IP" ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" ubuntu@"$SPOT_IP" "$ENV_EXPORTS $@" diff --git a/spartan/aztec-network/templates/tests/run-tests.yaml b/spartan/aztec-network/templates/tests/run-tests.yaml deleted file mode 100644 index bc32cfd7b00..00000000000 --- a/spartan/aztec-network/templates/tests/run-tests.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ .Release.Name }}-test" - labels: {{- include "aztec-network.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - restartPolicy: Never - containers: - - name: test - image: {{ .Values.images.test.image }} - args: - - {{ .Values.test }} - env: - - name: SCENARIO - value: {{ .Values.scenario }} - - name: PXE_URL - value: {{ include "aztec-network.pxeUrl" . | quote }} - - name: DEBUG - value: "aztec:*" - - name: LOG_LEVEL - value: "debug" - - name: LOG_JSON - value: "1" diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index 1f1c8146c76..228a3750b99 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -1,14 +1,8 @@ -test: "sample" -scenario: "default" - network: public: false enableBots: true images: - test: - image: aztecprotocol/end-to-end - pullPolicy: IfNotPresent aztec: image: aztecprotocol/aztec pullPolicy: IfNotPresent @@ -182,4 +176,4 @@ metrics: periodSeconds: 10 timeoutSeconds: 5 successThreshold: 1 - failureThreshold: 3 + failureThreshold: 3 \ No newline at end of file diff --git a/spartan/scripts/forward_metrics_dashboard.sh b/spartan/scripts/forward_metrics_dashboard.sh new file mode 100755 index 00000000000..8619cbe5cc8 --- /dev/null +++ b/spartan/scripts/forward_metrics_dashboard.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "Run this script after the metrics pods all come up. After running this script, port forward 5601 locally" +echo "Then navigate to https://localhost:5601 and login with user elastic password admin" +kubectl port-forward -n metrics service/metrics-eck-kibana-kb-http 5601:5601 \ No newline at end of file diff --git a/spartan/scripts/setup_local_k8s.sh b/spartan/scripts/setup_local_k8s.sh index 68d59939613..37eaadf85e5 100755 --- a/spartan/scripts/setup_local_k8s.sh +++ b/spartan/scripts/setup_local_k8s.sh @@ -30,4 +30,8 @@ if ! command -v helm &> /dev/null; then rm get_helm.sh fi -kind create cluster +if kind get clusters | grep -q "^kind$"; then + echo "Cluster 'kind' already exists. Skipping creation." +else + kind create cluster +fi \ No newline at end of file diff --git a/spartan/scripts/setup_metrics_dashboard.sh b/spartan/scripts/setup_metrics_dashboard.sh new file mode 100755 index 00000000000..3b618116883 --- /dev/null +++ b/spartan/scripts/setup_metrics_dashboard.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +helm repo add elastic https://helm.elastic.co +helm repo update +helm dependency build "$(git rev-parse --show-toplevel)/spartan/metrics/" +kubectl create -f https://download.elastic.co/downloads/eck/2.14.0/crds.yaml 2>/dev/null +kubectl create secret generic elasticsearch-es-elastic-user --from-literal=elastic=admin -n metrics +kubectl apply -f https://download.elastic.co/downloads/eck/2.14.0/operator.yaml +helm upgrade --install metrics "$(git rev-parse --show-toplevel)/spartan/metrics/" --wait --namespace metrics --create-namespace --wait-for-jobs=true --timeout=30m \ No newline at end of file diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index e58867f50e8..de86cd9f3eb 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -295,9 +295,9 @@ e2e-cli-wallet: network-smoke: ARG values_file LOCALLY - RUN NAMESPACE=smoke FRESH_INSTALL=true FORCE_BUILD=$force_build VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/smoke.test.ts + RUN NAMESPACE=smoke FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/smoke.test.ts network-transfer: ARG values_file LOCALLY - RUN NAMESPACE=transfer FRESH_INSTALL=true FORCE_BUILD=$force_build VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/transfer.test.ts + RUN NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/transfer.test.ts diff --git a/yarn-project/end-to-end/scripts/network_test.sh b/yarn-project/end-to-end/scripts/network_test.sh index d757c45af80..00e58e54f4a 100755 --- a/yarn-project/end-to-end/scripts/network_test.sh +++ b/yarn-project/end-to-end/scripts/network_test.sh @@ -43,18 +43,45 @@ if [ "$FRESH_INSTALL" = "true" ]; then kubectl delete namespace "$NAMESPACE" --ignore-not-found=true --wait=true --now --timeout=10m fi +function show_status_until_pxe_ready() { + set +x # don't spam with our commands + sleep 15 # let helm upgrade start + for i in {1..100} ; do + if kubectl wait pod -l app==pxe --for=condition=Ready -n "transfer" --timeout=20s >/dev/null 2>/dev/null ; then + break # we are up, stop showing status + fi + # show startup status + kubectl get pods -n "$NAMESPACE" + done +} + +show_status_until_pxe_ready & + # Install the Helm chart -helm install spartan "$(git rev-parse --show-toplevel)/spartan/aztec-network/" \ +helm upgrade --install spartan "$(git rev-parse --show-toplevel)/spartan/aztec-network/" \ --namespace "$NAMESPACE" \ --create-namespace \ --values "$(git rev-parse --show-toplevel)/spartan/aztec-network/values/$VALUES_FILE" \ - --set images.test.image="aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG" \ --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \ - --set test="$TEST" \ + --set ingress.enabled=true \ --wait \ --wait-for-jobs=true \ --timeout=30m kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m -helm test spartan --namespace "$NAMESPACE" --timeout 30m +function forward_pxe_k8s_port() { + # NOTE we fail silently, and work in the background + kubectl port-forward --namespace transfer svc/spartan-aztec-network-pxe 9082:8080 2>/dev/null >/dev/null || true +} +# tunnel in to get access directly to our PXE service in k8s +(kubectl port-forward --namespace transfer svc/spartan-aztec-network-pxe 9082:8080 2>/dev/null >/dev/null || true) & + +# run our test in the host network namespace (so we can access the above with localhost) +docker run --rm --network=host \ + -e SCENARIO=default \ + -e PXE_URL=http://localhost:9082 \ + -e DEBUG="aztec:*" \ + -e LOG_LEVEL=debug \ + -e LOG_JSON=1 \ + aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG $TEST From 8031ef45fc02f8897336729e7c41925ecae7c2e2 Mon Sep 17 00:00:00 2001 From: Facundo Date: Wed, 2 Oct 2024 19:30:04 +0100 Subject: [PATCH 2/2] chore: small cleanup (#8965) Please read [contributing guidelines](CONTRIBUTING.md) and remove this line. --- .../state_vars/shared_mutable/shared_mutable.nr | 15 ++------------- .../crates/types/src/address/mod.nr | 12 ------------ .../crates/types/src/utils/arrays.nr | 12 ++---------- 3 files changed, 4 insertions(+), 35 deletions(-) diff --git a/noir-projects/aztec-nr/aztec/src/state_vars/shared_mutable/shared_mutable.nr b/noir-projects/aztec-nr/aztec/src/state_vars/shared_mutable/shared_mutable.nr index 31171f75b1c..2d6bef36da2 100644 --- a/noir-projects/aztec-nr/aztec/src/state_vars/shared_mutable/shared_mutable.nr +++ b/noir-projects/aztec-nr/aztec/src/state_vars/shared_mutable/shared_mutable.nr @@ -1,6 +1,6 @@ use dep::protocol_types::{ hash::{poseidon2_hash, poseidon2_hash_with_separator}, address::AztecAddress, - traits::{FromField, ToField} + traits::{FromField, ToField}, utils::arrays::array_concat }; use crate::context::{PrivateContext, PublicContext, UnconstrainedContext}; @@ -32,17 +32,6 @@ global HASH_SEPARATOR: u32 = 2; // can actually use it here impl Storage for SharedMutable {} -fn concat_arrays(arr_n: [Field; N], arr_m: [Field; M]) -> [Field; N + M] { - let mut out: [Field; N + M] = [0; N + M]; - for i in 0..N { - out[i] = arr_n[i]; - } - for i in 0..M { - out[N+i] = arr_m[i]; - } - out -} - // SharedMutable stores a value of type T that is: // - publicly known (i.e. unencrypted) // - mutable in public @@ -220,7 +209,7 @@ impl SharedMutable, delay_change: ScheduledDelayChange ) -> Field { - let concatenated: [Field; 4] = concat_arrays(value_change.serialize(), delay_change.serialize()); + let concatenated: [Field; 4] = array_concat(value_change.serialize(), delay_change.serialize()); poseidon2_hash(concatenated) } } diff --git a/noir-projects/noir-protocol-circuits/crates/types/src/address/mod.nr b/noir-projects/noir-protocol-circuits/crates/types/src/address/mod.nr index 9c7a400ea9c..88b0236e421 100644 --- a/noir-projects/noir-protocol-circuits/crates/types/src/address/mod.nr +++ b/noir-projects/noir-protocol-circuits/crates/types/src/address/mod.nr @@ -9,15 +9,3 @@ pub use eth_address::EthAddress; pub use partial_address::PartialAddress; pub use public_keys_hash::PublicKeysHash; pub use salted_initialization_hash::SaltedInitializationHash; - -use crate::{constants::GENERATOR_INDEX__CONSTRUCTOR, hash::poseidon2_hash_with_separator}; - -pub fn compute_initialization_hash(selector: Field, args_hash: Field) -> Field { - poseidon2_hash_with_separator( - [ - selector, - args_hash - ], - GENERATOR_INDEX__CONSTRUCTOR - ) -} diff --git a/noir-projects/noir-protocol-circuits/crates/types/src/utils/arrays.nr b/noir-projects/noir-protocol-circuits/crates/types/src/utils/arrays.nr index c69e6234ed1..718f83ef761 100644 --- a/noir-projects/noir-protocol-circuits/crates/types/src/utils/arrays.nr +++ b/noir-projects/noir-protocol-circuits/crates/types/src/utils/arrays.nr @@ -93,9 +93,8 @@ pub fn array_length(array: [T; N]) -> u32 where T: Empty + Eq { length } -pub fn array_concat(array1: [T; N], array2: [T; M]) -> [T; S] { - assert_eq(N + M, S, "combined array length does not match return array length"); - let mut result = [array1[0]; S]; +pub fn array_concat(array1: [T; N], array2: [T; M]) -> [T; N + M] { + let mut result = [array1[0]; N + M]; for i in 1..N { result[i] = array1[i]; } @@ -228,13 +227,6 @@ fn test_array_concat() { assert_eq(concated, [1, 2, 3, 4, 5]); } -#[test(should_fail_with="combined array length does not match return array length")] -fn array_concat_fails_inconsistent_lengths() { - let array0 = [1, 2, 3]; - let array1 = [4, 5]; - let _concated: [Field; 4] = array_concat(array0, array1); -} - #[test] fn check_permutation_basic_test() { let original_array = [1, 2, 3];