add playbook to run execution spec tests

Dockerfile

@@ -6,7 +6,7 @@ RUN go mod download
 COPY . .
 RUN make build
 
-FROM ubuntu:latest  
+FROM ubuntu:latest
 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
   libssl-dev \
   ca-certificates \
@@ -17,4 +17,4 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /src/bin/assertoor /assertoor
-ENTRYPOINT ["/assertoor"]
+ENTRYPOINT ["/assertoor"]

Dockerfile-local

@@ -22,4 +22,4 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
 COPY --from=builder /src/bin/assertoor /assertoor
 RUN mkdir /workspace
 WORKDIR /workspace
-ENTRYPOINT ["/assertoor"]
+ENTRYPOINT ["/assertoor"]

pkg/coordinator/web/handlers/test_run.go

@@ -28,25 +28,27 @@ type TestRunPage struct {
 }
 
 type TestRunTask struct {
-	Index       uint64            `json:"index"`
-	ParentIndex uint64            `json:"parent_index"`
-	GraphLevels []uint64          `json:"graph_levels"`
-	Name        string            `json:"name"`
-	Title       string            `json:"title"`
-	IsStarted   bool              `json:"started"`
-	IsCompleted bool              `json:"completed"`
-	StartTime   time.Time         `json:"start_time"`
-	StopTime    time.Time         `json:"stop_time"`
-	Timeout     time.Duration     `json:"timeout"`
-	HasTimeout  bool              `json:"has_timeout"`
-	RunTime     time.Duration     `json:"runtime"`
-	HasRunTime  bool              `json:"has_runtime"`
-	Status      string            `json:"status"`
-	Result      string            `json:"result"`
-	ResultError string            `json:"result_error"`
-	Log         []*TestRunTaskLog `json:"log"`
-	ConfigYaml  string            `json:"config_yaml"`
-	ResultYaml  string            `json:"result_yaml"`
+	Index            uint64            `json:"index"`
+	ParentIndex      uint64            `json:"parent_index"`
+	GraphLevels      []uint64          `json:"graph_levels"`
+	Name             string            `json:"name"`
+	Title            string            `json:"title"`
+	IsStarted        bool              `json:"started"`
+	IsCompleted      bool              `json:"completed"`
+	StartTime        time.Time         `json:"start_time"`
+	StopTime         time.Time         `json:"stop_time"`
+	Timeout          time.Duration     `json:"timeout"`
+	HasTimeout       bool              `json:"has_timeout"`
+	RunTime          time.Duration     `json:"runtime"`
+	HasRunTime       bool              `json:"has_runtime"`
+	CustomRunTime    time.Duration     `json:"custom_runtime"`
+	HasCustomRunTime bool              `json:"has_custom_runtime"`
+	Status           string            `json:"status"`
+	Result           string            `json:"result"`
+	ResultError      string            `json:"result_error"`
+	Log              []*TestRunTaskLog `json:"log"`
+	ConfigYaml       string            `json:"config_yaml"`
+	ResultYaml       string            `json:"result_yaml"`
 }
 
 type TestRunTaskLog struct {
@@ -267,7 +269,15 @@ func (fh *FrontendHandler) getTestRunPageData(runID int64) (*TestRunPage, error)
 					taskData.ConfigYaml = fmt.Sprintf("\n%v\n", string(taskConfig))
 				}
 
-				taskResult, err := yaml.Marshal(taskState.GetTaskStatusVars().GetVarsMap(nil, false))
+				taskStatusVars := taskState.GetTaskStatusVars().GetVarsMap(nil, false)
+				if taskOutput, ok := taskStatusVars["outputs"]; ok {
+					if customRunTimeSecondsRaw, ok := taskOutput.(map[string]interface{})["customRunTimeSeconds"]; ok {
+						taskData.CustomRunTime = time.Duration(customRunTimeSecondsRaw.(float64) * float64(time.Second))
+						taskData.HasCustomRunTime = true
+					}
+				}
+
+				taskResult, err := yaml.Marshal(taskStatusVars)
 				if err != nil {
 					taskData.ResultYaml = fmt.Sprintf("failed marshalling result: %v", err)
 				} else {

pkg/coordinator/web/templates/test_run/test_run.html

@@ -1,15 +1,15 @@
 {{ define "page" }}
 <div class="d-flex flex-column flex-grow-1 mt-3 container-fluid">
     <h2 class="py-2">Test Run {{ .RunID }}: {{ .Name }}</h2>
-    
+
     <!-- client pool status -->
     <table class="test-header">
       <tr>
         <td style="width: 200px">
           Test ID:
         </td>
         <td>
-          {{ .TestID }}     
+          {{ .TestID }}
         </td>
       </tr>
       <tr>
@@ -50,7 +50,7 @@ <h2 class="py-2">Test Run {{ .RunID }}: {{ .Name }}</h2>
           Start Time:
         </td>
         <td>
-          {{ formatDateTime .StartTime.UTC }}     
+          {{ formatDateTime .StartTime.UTC }}
         </td>
       </tr>
       {{ end }}
@@ -60,7 +60,7 @@ <h2 class="py-2">Test Run {{ .RunID }}: {{ .Name }}</h2>
           Finish Time:
         </td>
         <td>
-          {{ formatDateTime .StopTime.UTC }}     
+          {{ formatDateTime .StopTime.UTC }}
         </td>
       </tr>
       {{ end }}
@@ -69,11 +69,11 @@ <h2 class="py-2">Test Run {{ .RunID }}: {{ .Name }}</h2>
           Timeout:
         </td>
         <td>
-          {{ .Timeout }}     
+          {{ .Timeout }}
         </td>
       </tr>
     </table>
-    
+
     <!-- task list -->
     <div class="task-list">
       <h5 class="mt-3 mb-0">Tasks</h5>
@@ -112,7 +112,17 @@ <h5 class="mt-3 mb-0">Tasks</h5>
             </td>
             <td>{{ $task.Name }}</td>
             <td>{{ $task.Title }}</td>
-            <td>{{ if $task.HasRunTime }}{{ $task.RunTime }}{{ else }}?{{ end }}{{ if $task.HasTimeout }} / {{ $task.Timeout }}{{ end }}</td>
+            <td>
+              {{ if $task.HasRunTime }}{{ $task.RunTime }}{{ else }}?{{ end }}
+              {{ if $task.HasTimeout }} / {{ $task.Timeout }}{{ end }}
+              {{ if $task.HasCustomRunTime}}
+                <span data-bs-toggle="tooltip"
+                      data-bs-placement="top"
+                      data-bs-title="Custom timer via outputs.customRunTimeSeconds" >
+                  ({{ $task.CustomRunTime}})
+                </span>
+              {{ end }}
+            </td>
             <td>
               {{ if eq $task.Result "success" }}
                 <span class="badge rounded-pill text-bg-success">
@@ -259,10 +269,13 @@ <h5 class="mt-3 mb-0">Tasks</h5>
                         </div>
                       </div>
                       <div class="tab-pane fade card-body" id="task{{ $task.Index }}-config" role="tabpanel" aria-labelledby="task{{ $task.Index }}-config-tab">
-                        <pre>{{ $task.ConfigYaml }}</pre>
+                        <pre style="text-wrap: pretty">{{ $task.ConfigYaml }}</pre>
                       </div>
                       <div class="tab-pane fade card-body" id="task{{ $task.Index }}-result" role="tabpanel" aria-labelledby="task{{ $task.Index }}-result-tab">
-                        <pre>{{ $task.ResultYaml }}</pre>
+                        <pre style="text-wrap: pretty">{{ $task.ResultYaml }}</pre>
+                      </div>
+                      <div class="tab-pane fade card-body" id="task{{ $task.Index }}-custom-html" role="tabpanel" aria-labelledby="task{{ $task.Index }}-custom-html-tab">
+                        <pre style="text-wrap: pretty">{{ $task.ResultYaml }}</pre>
                       </div>
                     </div>
                   </div>
@@ -359,4 +372,4 @@ <h5 class="mt-3 mb-0">Tasks</h5>
 }
 
 </style>
-{{ end }}
+{{ end }}

playbooks/dev/execution-spec-tests-execute.yaml

@@ -0,0 +1,141 @@
+id: execution-spec-tests-execute
+name: "Run 'execute' on execution spec tests"
+timeout: 1h
+config:
+  gitRepo: https://github.com/ethereum/execution-spec-tests.git
+  gitBranch: main
+  testPath: ""
+  chainID: "0"
+  rpcEndpoint: http://127.0.0.1:8545
+  seedPrivateKey: ""
+  seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input
+  extraFlags: ""
+  solcVersion: "0.8.24"
+tasks:
+  - name: run_shell
+    title: "Execute tests: ${gitRepo}@${gitBranch} [${testPath}]"
+    id: execute
+    config:
+      shell: bash
+      shellArgs: [--login]
+      envVars:
+        GIT_REPO: gitRepo
+        GIT_BRANCH: gitBranch
+        TEST_PATH: testPath
+        CHAIN_ID: chainID
+        RPC_ENDPOINT: rpcEndpoint
+        PRIVATE_KEY: seedPrivateKey
+        SEED_AMOUNT: seedAmount
+        EXTRA_FLAGS: extraFlags
+        SOLC_VERSION: "solcVersion"
+      command: |
+        set -e
+
+        # Convert env vars. They are passed as RAW JSON values
+
+        GIT_REPO=$(echo $GIT_REPO | jq -r)
+        GIT_BRANCH=$(echo $GIT_BRANCH | jq -r)
+        TEST_PATH=$(echo $TEST_PATH | jq -r)
+        CHAIN_ID=$(echo $CHAIN_ID | jq -r)
+        RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r)
+        PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r)
+        SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r)
+        EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r)
+        SOLC_VERSION=$(echo $SOLC_VERSION | jq -r)
+
+        echo "RPC_ENDPOINT: ${RPC_ENDPOINT}"
+        echo "CHAIN_ID: ${CHAIN_ID}"
+
+        # Validate some inputs
+        if [ -z "$TEST_PATH" ]; then
+          echo
+          exit "You need to provide a test path"
+        fi
+        if [ -z "$PRIVATE_KEY" ]; then
+          echo
+          exit "You need to provide a private key to fund the tests"
+        fi
+
+        # Check if pip (python package manager) is installed
+        if ! command -v pip &> /dev/null
+        then
+          echo "pip could not be found. Please install python3-pip"
+          exit 1
+        fi
+
+        # Create dir for temp files
+        tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX)
+        cd $tmp_dir
+        export HOME=$tmp_dir
+        echo "============================"
+        echo "Temp dir created: ${tmp_dir}"
+        echo "============================"
+
+        function cleanup {
+          rv=$?
+          rm -rf "$tmp_dir"
+          echo "tmpdir removed"
+          exit $rv
+        }
+
+        trap cleanup EXIT # always remove tempdir on exit
+
+        echo "============================"
+        echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}"
+        echo "============================"
+        git clone ${GIT_REPO} --branch ${GIT_BRANCH} --single-branch
+        cd execution-spec-tests
+
+        echo "============================"
+        echo "Installing dependencies"
+        echo "============================"
+        pip install uv
+        uv sync --all-extras
+        uv run solc-select use "${SOLC_VERSION}" --always-install
+        source .venv/bin/activate
+
+        echo "============================"
+        echo "Running test: ${TEST_PATH}"
+        echo "============================"
+        uv run execute remote "${TEST_PATH}" \
+          --rpc-chain-id=${CHAIN_ID} \
+          --rpc-endpoint=${RPC_ENDPOINT} \
+          --rpc-seed-key=${PRIVATE_KEY} \
+          --seed-account-sweep-amount=${SEED_AMOUNT} \
+          --json-report \
+          --json-report-file=report.json \
+          --html=report.html \
+          ${EXTRA_FLAGS[@]}
+
+        echo "============================"
+        echo "Exporting reports"
+        echo "============================"
+        REPORT_JSON=$(cat report.json)
+        REPORT_HTML=$(jq -Rs '.' report.html)
+        echo "::set-output reportHTML ${REPORT_HTML}"
+        echo "::set-output-json reportJSON ${REPORT_JSON}"
+
+  - name: run_task_matrix
+    title: "Show test results"
+    configVars:
+      matrixValues: "tasks.execute.outputs.reportJSON.tests"
+    config:
+      runConcurrent: true
+      matrixVar: "testResult"
+      task:
+        name: run_shell
+        title: "${{testResult.nodeid}}"
+        config:
+          shell: bash
+          envVars:
+            TEST_RESULT: testResult
+          command: |
+            DURATION_SECONDS=$(echo $TEST_RESULT | jq -r '.setup.duration + .call.duration + .teardown.duration')
+            echo "::set-output-json customRunTimeSeconds ${DURATION_SECONDS}"
+            echo "::set-output-json execTestResult ${TEST_RESULT}"
+            if $(echo $TEST_RESULT | jq -e '.outcome == "passed"') ; then
+              echo "Test passed"
+            else
+              echo "Test failed"
+              exit 1
+            fi