Merge pull request #3108 from oasisprotocol/tjanez/test-runner-consis…

…tency

go/oasis-test-runner: Refactor code to refer to scenario(s) consistently

.buildkite/benchmarks.pipeline.yml

@@ -29,7 +29,7 @@ docker_plugin_default_config: &docker_plugin_default_config
     - "BUILDKITE_PIPELINE_NAME"
     - "BUILDKITE_BUILD_NUMBER"
     - "BUILDKITE_BUILD_URL"
-    - "TESTS"
+    - "SCENARIOS"
     - "NUM_RUNS"
     - "SLACK_WEBHOOK_URL"
     - "METRICS_PUSH_ADDR"
@@ -123,7 +123,13 @@ steps:
     command:
       - .buildkite/scripts/download_e2e_test_artifacts.sh
       - rm -rf /var/tmp/benchmarks/*
-      - .buildkite/scripts/test_e2e.sh --metrics.address $METRICS_PUSH_ADDR --metrics.interval 5s --metrics.labels instance=\$BUILDKITE_PIPELINE_NAME-\$BUILDKITE_BUILD_NUMBER --num_runs $NUM_RUNS -t $TESTS
+      - >-
+          .buildkite/scripts/test_e2e.sh
+            --metrics.address $METRICS_PUSH_ADDR
+            --metrics.interval 5s
+            --metrics.labels instance=$BUILDKITE_PIPELINE_NAME-$BUILDKITE_BUILD_NUMBER
+            --num_runs $NUM_RUNS
+            --scenario $SCENARIOS
     env:
       TEST_BASE_DIR: /var/tmp/benchmarks
     agents:

.buildkite/code.pipeline.yml

@@ -244,8 +244,8 @@ steps:
     timeout_in_minutes: 21
     command:
       - .buildkite/scripts/download_e2e_test_artifacts.sh
-      # Only run runtime tests as others do not use SGX.
-      - .buildkite/scripts/test_e2e.sh -t e2e/runtime/.*
+      # Only run runtime scenarios as others do not use SGX.
+      - .buildkite/scripts/test_e2e.sh --scenario e2e/runtime/.*
     artifact_paths:
       - coverage-merged-e2e-*.txt
       - /tmp/e2e/**/*.log

.buildkite/scripts/daily_benchmark_analysis.sh

@@ -12,7 +12,7 @@
 # METRICS_SOURCE_GIT_BRANCH - name of feature branch on git (e.g. jsmith/feature/abc)
 # METRICS_TARGET_GIT_BRANCH - name of master branch on git (e.g. master)
 # METRICS_THRESHOLDS - max or min thresholds flags (e.g. --max_threshold.cpu.avg_ratio 1.05)
-# TESTS - names of test(s) to compare (e.g. e2e/runtime/runtime)
+# SCENARIOS - names of scenario(s) to compare (e.g. e2e/runtime/runtime)
 # SLACK_WEBHOOK_URL - slack webhook for reporting (e.g. https://hooks.slack.com/services/xxxxxx)
 
 set -ux
@@ -21,7 +21,7 @@ set -ux
   --metrics.address $METRICS_QUERY_ADDR \
   --metrics.source.git_branch $METRICS_SOURCE_GIT_BRANCH \
   --metrics.target.git_branch $METRICS_TARGET_GIT_BRANCH \
-  -t $TESTS \
+  --scenario $SCENARIOS \
   --log.level INFO \
   --log.format JSON \
   $METRICS_THRESHOLDS \

.buildkite/scripts/daily_txsource.sh

@@ -9,7 +9,7 @@ if [[ $BUILDKITE_RETRY_COUNT == 0 ]]; then
     ./.buildkite/scripts/test_e2e.sh \
         --metrics.address $METRICS_PUSH_ADDR \
         --metrics.labels instance=$BUILDKITE_PIPELINE_NAME-$BUILDKITE_BUILD_NUMBER \
-        --test e2e/runtime/txsource-multi
+        --scenario e2e/runtime/txsource-multi
 else
     curl -H "Content-Type: application/json" \
         -X POST \

.buildkite/scripts/sgx_ias_tests.sh

@@ -21,4 +21,4 @@ export OASIS_UNSAFE_KM_POLICY_KEYS=1
 export OASIS_UNSAFE_ALLOW_DEBUG_ENCLAVES=1
 
 make all
-.buildkite/scripts/test_e2e.sh -t e2e/runtime/runtime-encryption
+.buildkite/scripts/test_e2e.sh --scenario e2e/runtime/runtime-encryption

.changelog/3108.breaking.1.md

@@ -0,0 +1,3 @@
+go/oasis-test-runner/cmd: Rename `--test` flag to `--scenario` flag
+
+Rename the short version from `-t` to `-s`.

.changelog/3108.breaking.2.md

@@ -0,0 +1,7 @@
+go/oasis-test-runner/env: Rename types, fields, functions to refer to scenario
+
+Rename `TestInstanceInfo` type to `ScenarioInstanceInfo` and its `Test` field
+to `Scenario`.
+
+Rename `Env`'s `TestInfo()` and `WriteTestInfo()` methods to `ScenarioInfo()`
+and `WriteScenarioInfo()` for consistency.

.changelog/3108.breaking.3.md

@@ -0,0 +1 @@
+go/oasis-node/cmd/common/metrics: Rename "scenario" metrics label to "test"

.changelog/3108.internal.md

@@ -0,0 +1 @@
+go/oasis-test-runner: Refactor code to refer to scenario(s) consistently

docs/oasis-node/metrics.md

@@ -71,7 +71,7 @@ oasis_storage_failures | Counter | Number of storage failures. | call | [storage
 oasis_storage_latency | Summary | Storage call latency (seconds). | call | [storage](../../go/storage/metrics.go)
 oasis_storage_successes | Counter | Number of storage successes. | call | [storage](../../go/storage/metrics.go)
 oasis_storage_value_size | Summary | Storage call value size (bytes). | call | [storage](../../go/storage/metrics.go)
-oasis_up | Gauge | Is oasis-test-runner active for specific test. |  | [oasis-node/cmd/common/metrics](../../go/oasis-node/cmd/common/metrics/metrics.go)
+oasis_up | Gauge | Is oasis-test-runner active for specific scenario. |  | [oasis-node/cmd/common/metrics](../../go/oasis-node/cmd/common/metrics/metrics.go)
 oasis_worker_aborted_batch_count | Counter | Number of aborted batches. | runtime | [worker/compute/executor/committee](../../go/worker/compute/executor/committee/node.go)
 oasis_worker_aborted_merge_count | Counter | Number of aborted merges. | runtime | [worker/compute/merge/committee](../../go/worker/compute/merge/committee/node.go)
 oasis_worker_batch_processing_time | Summary | Time it takes for a batch to finalize (seconds). | runtime | [worker/compute/executor/committee](../../go/worker/compute/executor/committee/node.go)

go/oasis-node/cmd/common/metrics/metrics.go

@@ -37,7 +37,7 @@ const (
 	MetricsLabelInstance        = "instance"
 	MetricsLabelRun             = "run"
 	MetricsLabelSoftwareVersion = "software_version"
-	MetricsLabelTest            = "test"
+	MetricsLabelScenario        = "scenario"
 
 	MetricsModeNone = "none"
 	MetricsModePull = "pull"
@@ -51,7 +51,7 @@ var (
 	UpGauge = prometheus.NewGauge(
 		prometheus.GaugeOpts{
 			Name: MetricUp,
-			Help: "Is oasis-test-runner active for specific test.",
+			Help: "Is oasis-test-runner active for specific scenario.",
 		},
 	)
 )
@@ -271,11 +271,11 @@ func EscapeLabelCharacters(l string) string {
 }
 
 // GetDefaultPushLabels generates standard Prometheus push labels based on test current test instance info.
-func GetDefaultPushLabels(ti *env.TestInstanceInfo) map[string]string {
+func GetDefaultPushLabels(ti *env.ScenarioInstanceInfo) map[string]string {
 	labels := map[string]string{
 		MetricsLabelInstance:        ti.Instance,
 		MetricsLabelRun:             strconv.Itoa(ti.Run),
-		MetricsLabelTest:            ti.Test,
+		MetricsLabelScenario:        ti.Scenario,
 		MetricsLabelSoftwareVersion: version.SoftwareVersion,
 	}
 	if version.GitBranch != "" {

go/oasis-test-runner/README.md

@@ -1,79 +1,96 @@
 # Oasis Test Runner
 
-`oasis-test-runner` initializes and executes end-to-end and remote signer test
-suites.
+The Oasis Test Runner initializes and executes different types of tests (e.g.
+end-to-end, remote signer) also referred to as scenarios.
 
-To list all supported tests and corresponding parameters, type:
+To list all available scenarios and their corresponding parameters, use:
 
 ```bash
 oasis-test-runner list
 ```
 
-If no flags provided, all tests are executed. If you want to run specific test,
-pass `--test` parameter:
+If no flags are provided, all scenarios are executed.
+
+To run a specific scenario, e.g `e2e/runtime/runtime-dynamic`, pass the
+`--scenario` flag (or its short version, `-s`) followed by scenario's name:
 
 ```bash
-oasis-test-runner --test e2e/runtime/runtime-dynamic
+oasis-test-runner --scenario e2e/runtime/runtime-dynamic
 ```
 
 ## Benchmarking
 
-To benchmark tests, set the `--metrics.address` parameter to the address of the
-Prometheus push gateway along with `--metrics.interval`. Additionally, you
-can set test-specific parameters and the number of runs each test should be run
-with `--num_runs` parameter.
+To benchmark scenarios, set the `--metrics.address` flag to the address of the
+Prometheus push gateway along with the `--metrics.interval` flag.
+Additionally, you can set scenario-specific parameters and the number of runs of
+each scenario with the `--num_runs` flag.
 
 ## Benchmark analysis with `oasis-test-runner cmp` command
 
-`cmp` command connects to Prometheus server instance containing benchmark
-results generated by `oasis-test-runner` and corresponding `oasis-node`
-workers. It compares the benchmark results of the last (`source`) and
-pre-last (`target`) test batch (also called *instance*). You need to pass the
-address of Prometheus query server using `--metrics.address` flag:
+The `cmp` sub-command connects to the Prometheus server instance containing
+benchmark results generated by Oasis Test Runner and the corresponding Oasis
+Node workers.
+It compares the benchmark results of the last (`source`) and pre-last (`target`)
+scenario batch (also called *instance*).
+You need to pass the address of the Prometheus query server using
+`--metrics.address` flag, e.g.:
 
 ```bash
 oasis-test-runner cmp \
   --metrics.address http://prometheus.myorg.com:9090
 ```
 
-By default `cmp` command will fetch the results of all tests and metrics
-supported by `oasis-test-runner`. If you want to compare specific metric(s) and
-test(s), provide `--metrics` and `--test` flags respectively:
+By default, the `oasis-test-runner cmp` command will fetch the results of all
+scenarios and metrics supported by the Oasis Test Runner.
+If you want to compare specific metric(s) and scenario(s), set the `--metrics`
+and `--scenario` flags appropriately, e.g.:
 
 ```bash
 oasis-test-runner cmp \
   --metrics time \
-  --test e2e/runtime/multiple-runtimes
+  --scenario e2e/runtime/multiple-runtimes
 ```
 
-`cmp` takes *thresholds* for each metric into account. Currently, `avg_ratio`
-and `max_ratio` are supported which correspond to the average and maximum ratio
-of metric values of all test runs in the benchmark batch. For each ratio, you
-can set the `max_threshold` and `min_threshold`. The former requires that the
-ratio should not be exceeded and the latter that the provided threshold must be
-reached. If not, `oasis-test-runner cmp` will exit with error code. This is
-useful for integration into CI pipelines. Thresholds can be using the flag
-`--{min|max}_threshold.<metric name>.{avg|max}_ratio`.
+The `cmp` sub-command takes *thresholds* for each metric into account.
+Currently, `avg_ratio` and `max_ratio` are supported which correspond to the
+average and maximum ratio of metric values of all scenario runs in the benchmark
+batch.
+
+For each ratio, you can set the `max_threshold` and `min_threshold`.
+The former specifies which ratio should not be exceeded and the latter which
+ratio must be reached.
+If a value is not within these thresholds, the `oasis-test-runner cmp` command
+will exit with an error code.
+
+This is useful for integration into CI pipelines.
+Thresholds can be set using the following flag specification:
+
+```text
+--{min|max}_threshold.<metric name>.{avg|max}_ratio
+```
 
 For example:
 
 ```bash
 oasis-test-runner cmp \
   --metrics time \
-  --test e2e/runtime/multiple-runtimes \
+  --scenario e2e/runtime/multiple-runtimes \
   --max_threshold.time.avg_ratio 1.1
 ```
 
-will require that the average duration of the test in the last benchmark batch
-should be at most 10\% slower than from the pre-last benchmark batch.
+will require that the average duration of the scenario in the last benchmark
+batch should be at most 10\% slower than from the pre-last benchmark batch.
 
 If you are developing or improving a feature in a separate git branch, you will
 want to perform benchmarks and compare the results of your branch to the ones
-from the master branch. `oasis-test-runner` automatically sends information of
-the git branch it was compiled on to Prometheus, so all you need to do is
-perform benchmarks on both master and feature branches using the same Prometheus
-instance. Then, pass the name of source and target branch names to `cmp` and it
-will compare the last benchmark batch from each git branch respectively:
+from the `master` branch.
+The Oasis Test Runner automatically sends information of the git branch it was
+compiled on to Prometheus, so all you need to do is perform benchmarks on both,
+the `master` and a feature branch, using the same Prometheus instance.
+
+Afterwards, pass the name of source and target branch names to the `cmp`
+sub-command and it will compare the last benchmark batch from each git branch
+(respectively):
 
 ```bash
 oasis-test-runner cmp \