From 048bb58bad8022a737ea444d1f1c23aca13dbf56 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 13 Sep 2024 16:08:15 +0200 Subject: [PATCH 1/4] add playbook to run execution spec tests --- Dockerfile | 10 +- Dockerfile-local | 12 +- .../dev/execution-spec-tests-execute.yaml | 140 ++++++++++++++++++ 3 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 playbooks/dev/execution-spec-tests-execute.yaml diff --git a/Dockerfile b/Dockerfile index 5f0a972..52c0b9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN go mod download COPY . . RUN make build -FROM ubuntu:latest +FROM ubuntu:latest RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ libssl-dev \ ca-certificates \ @@ -14,7 +14,11 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco git \ curl \ make \ + build-essential \ + python3 \ + python3-pip \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm /usr/lib/python3*/EXTERNALLY-MANAGED COPY --from=builder /src/bin/assertoor /assertoor -ENTRYPOINT ["/assertoor"] \ No newline at end of file +ENTRYPOINT ["/assertoor"] diff --git a/Dockerfile-local b/Dockerfile-local index 065856d..62fb0cc 100644 --- a/Dockerfile-local +++ b/Dockerfile-local @@ -10,16 +10,22 @@ ADD pkg ./pkg RUN make build FROM debian:stable-slim -RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ +RUN dpkg --add-architecture amd64 \ + && apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ libssl-dev \ ca-certificates \ jq \ git \ curl \ make \ + build-essential \ + python3 \ + python3-pip \ + libc6:amd64 \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm /usr/lib/python3*/EXTERNALLY-MANAGED COPY --from=builder /src/bin/assertoor /assertoor RUN mkdir /workspace WORKDIR /workspace -ENTRYPOINT ["/assertoor"] \ No newline at end of file +ENTRYPOINT ["/assertoor"] diff --git a/playbooks/dev/execution-spec-tests-execute.yaml b/playbooks/dev/execution-spec-tests-execute.yaml new file mode 100644 index 0000000..522e7e6 --- /dev/null +++ b/playbooks/dev/execution-spec-tests-execute.yaml @@ -0,0 +1,140 @@ +id: execution-spec-tests-execute +name: "Run 'execute' on execution spec tests" +timeout: 1h +config: + gitRepo: https://github.com/ethereum/execution-spec-tests.git + gitBranch: main + testPath: "" + chainID: "0" + rpcEndpoint: http://127.0.0.1:8545 + privateKey: "" + seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input + extraFlags: "" + solcVersion: "0.8.24" +tasks: + - name: run_shell + title: "Execute tests: ${gitRepo}@${gitBranch} [${testPath}]" + id: execute + config: + shell: bash + shellArgs: [--login] + envVars: + GIT_REPO: gitRepo + GIT_BRANCH: gitBranch + TEST_PATH: testPath + CHAIN_ID: chainID + RPC_ENDPOINT: rpcEndpoint + PRIVATE_KEY: privateKey + SEED_AMOUNT: seedAmount + EXTRA_FLAGS: extraFlags + SOLC_VERSION: "solcVersion" + command: | + set -e + + # Convert env vars. They are passed as RAW JSON values + + GIT_REPO=$(echo $GIT_REPO | jq -r) + GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) + TEST_PATH=$(echo $TEST_PATH | jq -r) + CHAIN_ID=$(echo $CHAIN_ID | jq -r) + RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r) + PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r) + SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r) + EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r) + SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) + + echo "RPC_ENDPOINT: ${RPC_ENDPOINT}" + echo "CHAIN_ID: ${CHAIN_ID}" + + # Validate some inputs + if [ -z "$TEST_PATH" ]; then + echo + exit "You need to provide a test path" + fi + if [ -z "$PRIVATE_KEY" ]; then + echo + exit "You need to provide a private key to fund the tests" + fi + + # Check if pip (python package manager) is installed + if ! command -v pip &> /dev/null + then + echo "pip could not be found. Please install python3-pip" + exit 1 + fi + + # Create dir for temp files + tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) + cd $tmp_dir + export HOME=$tmp_dir + echo "============================" + echo "Temp dir created: ${tmp_dir}" + echo "============================" + + function cleanup { + rv=$? + rm -rf "$tmp_dir" + echo "tmpdir removed" + exit $rv + } + + trap cleanup EXIT # always remove tempdir on exit + + echo "============================" + echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}" + echo "============================" + git clone ${GIT_REPO} --branch ${GIT_BRANCH} --single-branch + cd execution-spec-tests + + echo "============================" + echo "Installing dependencies" + echo "============================" + pip install uv + uv sync --all-extras + uv run solc-select use "${SOLC_VERSION}" --always-install + source .venv/bin/activate + + echo "============================" + echo "Running test: ${TEST_PATH}" + echo "============================" + uv run execute remote "${TEST_PATH}" \ + --rpc-chain-id=${CHAIN_ID} \ + --rpc-endpoint=${RPC_ENDPOINT} \ + --rpc-seed-key=${PRIVATE_KEY} \ + --seed-account-sweep-amount=${SEED_AMOUNT} \ + --json-report \ + --json-report-file=report.json \ + --html=report.html \ + ${EXTRA_FLAGS[@]} + + + # Export reports + REPORT_JSON=$(cat report.json) + REPORT_HTML=$(jq -Rs '.' report.html) + echo "::set-output reportHTML ${REPORT_HTML}" + echo "::set-output-json reportJSON ${REPORT_JSON}" + + - name: run_task_matrix + title: "Show test results" + configVars: + matrixValues: "tasks.execute.outputs.reportJSON.tests" + config: + runConcurrent: true + matrixVar: "testResult" + task: + name: run_shell + title: "${{testResult.nodeid}}" + config: + shell: bash + envVars: + TEST_RESULT: testResult + command: | + DURATION_SECONDS=$(echo $TEST_RESULT | jq -r '.setup.duration + .call.duration + .teardown.duration') + echo "::set-output-json runeTimeOverrideSeconds ${DURATION_SECONDS}" + echo "::set-output-json execTestResult ${TEST_RESULT}" + if $(echo $TEST_RESULT | jq -e '.outcome == "passed"') ; then + echo "Test passed" + else + echo "Test failed" + exit 1 + fi From 0f7441a5e81689a8d2c7629c01e4b71f4cd974ff Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 13 Sep 2024 17:03:33 +0200 Subject: [PATCH 2/4] remove deps from dockerfiles --- Dockerfile | 6 +----- Dockerfile-local | 10 ++-------- playbooks/dev/execution-spec-tests-execute.yaml | 11 ++++++----- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52c0b9a..e469787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco git \ curl \ make \ - build-essential \ - python3 \ - python3-pip \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm /usr/lib/python3*/EXTERNALLY-MANAGED + && rm -rf /var/lib/apt/lists/* COPY --from=builder /src/bin/assertoor /assertoor ENTRYPOINT ["/assertoor"] diff --git a/Dockerfile-local b/Dockerfile-local index 62fb0cc..73b8706 100644 --- a/Dockerfile-local +++ b/Dockerfile-local @@ -10,21 +10,15 @@ ADD pkg ./pkg RUN make build FROM debian:stable-slim -RUN dpkg --add-architecture amd64 \ - && apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ libssl-dev \ ca-certificates \ jq \ git \ curl \ make \ - build-essential \ - python3 \ - python3-pip \ - libc6:amd64 \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm /usr/lib/python3*/EXTERNALLY-MANAGED + && rm -rf /var/lib/apt/lists/* COPY --from=builder /src/bin/assertoor /assertoor RUN mkdir /workspace WORKDIR /workspace diff --git a/playbooks/dev/execution-spec-tests-execute.yaml b/playbooks/dev/execution-spec-tests-execute.yaml index 522e7e6..aff28ba 100644 --- a/playbooks/dev/execution-spec-tests-execute.yaml +++ b/playbooks/dev/execution-spec-tests-execute.yaml @@ -7,7 +7,7 @@ config: testPath: "" chainID: "0" rpcEndpoint: http://127.0.0.1:8545 - privateKey: "" + seedPrivateKey: "" seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input extraFlags: "" solcVersion: "0.8.24" @@ -24,7 +24,7 @@ tasks: TEST_PATH: testPath CHAIN_ID: chainID RPC_ENDPOINT: rpcEndpoint - PRIVATE_KEY: privateKey + PRIVATE_KEY: seedPrivateKey SEED_AMOUNT: seedAmount EXTRA_FLAGS: extraFlags SOLC_VERSION: "solcVersion" @@ -107,8 +107,9 @@ tasks: --html=report.html \ ${EXTRA_FLAGS[@]} - - # Export reports + echo "============================" + echo "Exporting reports + echo "============================" REPORT_JSON=$(cat report.json) REPORT_HTML=$(jq -Rs '.' report.html) echo "::set-output reportHTML ${REPORT_HTML}" @@ -130,7 +131,7 @@ tasks: TEST_RESULT: testResult command: | DURATION_SECONDS=$(echo $TEST_RESULT | jq -r '.setup.duration + .call.duration + .teardown.duration') - echo "::set-output-json runeTimeOverrideSeconds ${DURATION_SECONDS}" + echo "::set-output-json customRunTimeSeconds ${DURATION_SECONDS}" echo "::set-output-json execTestResult ${TEST_RESULT}" if $(echo $TEST_RESULT | jq -e '.outcome == "passed"') ; then echo "Test passed" From a0b2e569a46109d5edaeeb4e63f94b77b15e7c98 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 13 Sep 2024 17:45:33 +0200 Subject: [PATCH 3/4] render custom timer when outputs.customRunTimeSeconds is set --- pkg/coordinator/web/handlers/test_run.go | 53 ++++++++++++------- .../web/templates/test_run/test_run.html | 33 ++++++++---- .../dev/execution-spec-tests-execute.yaml | 2 +- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/pkg/coordinator/web/handlers/test_run.go b/pkg/coordinator/web/handlers/test_run.go index 0371646..b12808f 100644 --- a/pkg/coordinator/web/handlers/test_run.go +++ b/pkg/coordinator/web/handlers/test_run.go @@ -28,25 +28,27 @@ type TestRunPage struct { } type TestRunTask struct { - Index uint64 `json:"index"` - ParentIndex uint64 `json:"parent_index"` - GraphLevels []uint64 `json:"graph_levels"` - Name string `json:"name"` - Title string `json:"title"` - IsStarted bool `json:"started"` - IsCompleted bool `json:"completed"` - StartTime time.Time `json:"start_time"` - StopTime time.Time `json:"stop_time"` - Timeout time.Duration `json:"timeout"` - HasTimeout bool `json:"has_timeout"` - RunTime time.Duration `json:"runtime"` - HasRunTime bool `json:"has_runtime"` - Status string `json:"status"` - Result string `json:"result"` - ResultError string `json:"result_error"` - Log []*TestRunTaskLog `json:"log"` - ConfigYaml string `json:"config_yaml"` - ResultYaml string `json:"result_yaml"` + Index uint64 `json:"index"` + ParentIndex uint64 `json:"parent_index"` + GraphLevels []uint64 `json:"graph_levels"` + Name string `json:"name"` + Title string `json:"title"` + IsStarted bool `json:"started"` + IsCompleted bool `json:"completed"` + StartTime time.Time `json:"start_time"` + StopTime time.Time `json:"stop_time"` + Timeout time.Duration `json:"timeout"` + HasTimeout bool `json:"has_timeout"` + RunTime time.Duration `json:"runtime"` + HasRunTime bool `json:"has_runtime"` + CustomRunTime time.Duration `json:"custom_runtime"` + HasCustomRunTime bool `json:"has_custom_runtime"` + Status string `json:"status"` + Result string `json:"result"` + ResultError string `json:"result_error"` + Log []*TestRunTaskLog `json:"log"` + ConfigYaml string `json:"config_yaml"` + ResultYaml string `json:"result_yaml"` } type TestRunTaskLog struct { @@ -267,7 +269,18 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error) taskData.ConfigYaml = fmt.Sprintf("\n%v\n", string(taskConfig)) } - taskResult, err := yaml.Marshal(taskState.GetTaskStatusVars().GetVarsMap(nil, false)) + taskStatusVars := taskState.GetTaskStatusVars().GetVarsMap(nil, false) + if taskOutput, ok := taskStatusVars["outputs"]; ok { + taskOutputMap := taskOutput.(map[string]interface{}) + if taskOutputMap != nil { + if customRunTimeSecondsRaw, ok := taskOutputMap["customRunTimeSeconds"]; ok { + taskData.CustomRunTime = time.Duration(customRunTimeSecondsRaw.(float64) * float64(time.Second)) + taskData.HasCustomRunTime = true + } + } + } + + taskResult, err := yaml.Marshal(taskStatusVars) if err != nil { taskData.ResultYaml = fmt.Sprintf("failed marshalling result: %v", err) } else { diff --git a/pkg/coordinator/web/templates/test_run/test_run.html b/pkg/coordinator/web/templates/test_run/test_run.html index 35ad7d0..ceb9e2f 100644 --- a/pkg/coordinator/web/templates/test_run/test_run.html +++ b/pkg/coordinator/web/templates/test_run/test_run.html @@ -1,7 +1,7 @@ {{ define "page" }}

Test Run {{ .RunID }}: {{ .Name }}

- + @@ -9,7 +9,7 @@

Test Run {{ .RunID }}: {{ .Name }}

Test ID: @@ -50,7 +50,7 @@

Test Run {{ .RunID }}: {{ .Name }}

Start Time: {{ end }} @@ -60,7 +60,7 @@

Test Run {{ .RunID }}: {{ .Name }}

Finish Time: {{ end }} @@ -69,11 +69,11 @@

Test Run {{ .RunID }}: {{ .Name }}

Timeout:
- {{ .TestID }} + {{ .TestID }}
- {{ formatDateTime .StartTime.UTC }} + {{ formatDateTime .StartTime.UTC }}
- {{ formatDateTime .StopTime.UTC }} + {{ formatDateTime .StopTime.UTC }}
- {{ .Timeout }} + {{ .Timeout }}
- +
Tasks
@@ -112,7 +112,17 @@
Tasks
{{ $task.Name }} {{ $task.Title }} - {{ if $task.HasRunTime }}{{ $task.RunTime }}{{ else }}?{{ end }}{{ if $task.HasTimeout }} / {{ $task.Timeout }}{{ end }} + + {{ if $task.HasRunTime }}{{ $task.RunTime }}{{ else }}?{{ end }} + {{ if $task.HasTimeout }} / {{ $task.Timeout }}{{ end }} + {{ if $task.HasCustomRunTime}} + + ({{ $task.CustomRunTime}}) + + {{ end }} + {{ if eq $task.Result "success" }} @@ -259,10 +269,13 @@
Tasks
- 
{{ $task.ConfigYaml }}
+ 
{{ $task.ConfigYaml }}
- 
{{ $task.ResultYaml }}
+ 
{{ $task.ResultYaml }}
+
+
+ 
{{ $task.ResultYaml }}
@@ -359,4 +372,4 @@
Tasks
} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/playbooks/dev/execution-spec-tests-execute.yaml b/playbooks/dev/execution-spec-tests-execute.yaml index aff28ba..88eca5d 100644 --- a/playbooks/dev/execution-spec-tests-execute.yaml +++ b/playbooks/dev/execution-spec-tests-execute.yaml @@ -108,7 +108,7 @@ tasks: ${EXTRA_FLAGS[@]} echo "============================" - echo "Exporting reports + echo "Exporting reports" echo "============================" REPORT_JSON=$(cat report.json) REPORT_HTML=$(jq -Rs '.' report.html) From 46fbfa90186ad5de4c4a5caf56bc985f1356e80a Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Fri, 13 Sep 2024 19:25:07 +0200 Subject: [PATCH 4/4] simplify --- pkg/coordinator/web/handlers/test_run.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/coordinator/web/handlers/test_run.go b/pkg/coordinator/web/handlers/test_run.go index b12808f..e810e63 100644 --- a/pkg/coordinator/web/handlers/test_run.go +++ b/pkg/coordinator/web/handlers/test_run.go @@ -271,12 +271,9 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error) taskStatusVars := taskState.GetTaskStatusVars().GetVarsMap(nil, false) if taskOutput, ok := taskStatusVars["outputs"]; ok { - taskOutputMap := taskOutput.(map[string]interface{}) - if taskOutputMap != nil { - if customRunTimeSecondsRaw, ok := taskOutputMap["customRunTimeSeconds"]; ok { - taskData.CustomRunTime = time.Duration(customRunTimeSecondsRaw.(float64) * float64(time.Second)) - taskData.HasCustomRunTime = true - } + if customRunTimeSecondsRaw, ok := taskOutput.(map[string]interface{})["customRunTimeSeconds"]; ok { + taskData.CustomRunTime = time.Duration(customRunTimeSecondsRaw.(float64) * float64(time.Second)) + taskData.HasCustomRunTime = true } }