estimator: switch to Haswell-v4 architecture & podman work (#10645)

The previous CPU of choice does not support AVX, so the estimation does
not complete with that CPU of choice. However QEMU did not support these
instructions in its TCG in versions used before. I had to grab a more
recent version of it and build it in our reproducible environment. Now I
don't appreciate docker needing a root service so I rewrote some of the
text around the setup to use podman user-space container runtime
instead. Now the scripts run without any notable permission or service
requiremenets (besides the requirement to have podman tool available
instead of docker.)

References to docker still remain here and there – I will resolve them
once I'm done with the primary task at hand.

The estimations are not quite what they seem to have been recorded last
time continuous estimation ran successfully (protocol version was 50!?)
but this difference appears to be reasonably reproducible between the
two CPU choices (which makes sense -- we're counting instructions
executed) for what test cases work with the previous option.

Unfortunately it is difficult for me to provide specific numbers as the
full estimator run takes some 12 hours-or-so to run under qemu…

docs/practices/workflows/gas_estimations.md

@@ -16,11 +16,11 @@ cargo run -p runtime-params-estimator --features required -- \
 You should get an output like this.
 
 ```
-[elapsed 00:00:17 remaining 00:00:00] Writing into storage ████████████████████   20000/20000  
-ActionReceiptCreation         4_499_673_502_000 gas [  4.499674ms]    (computed in 7.22s) 
-ActionTransfer                  410_122_090_000 gas [   410.122µs]    (computed in 4.71s) 
-ActionCreateAccount             237_495_890_000 gas [   237.496µs]    (computed in 4.64s) 
-ActionFunctionCallBase          770_989_128_914 gas [   770.989µs]    (computed in 4.65s) 
+[elapsed 00:00:17 remaining 00:00:00] Writing into storage ████████████████████   20000/20000
+ActionReceiptCreation         4_499_673_502_000 gas [  4.499674ms]    (computed in 7.22s)
+ActionTransfer                  410_122_090_000 gas [   410.122µs]    (computed in 4.71s)
+ActionCreateAccount             237_495_890_000 gas [   237.496µs]    (computed in 4.64s)
+ActionFunctionCallBase          770_989_128_914 gas [   770.989µs]    (computed in 4.65s)
 
 
 Finished in 40.11s, output saved to:
@@ -63,11 +63,11 @@ instruction count, read IO bytes, and write IO bytes respectively. The IO byte
 count is known to be inaccurate.
 
 ```
-+ /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/qemu-x86_64 -plugin file=/host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/libcounter.so -cpu Westmere-v1 /host/nearcore/target/release/runtime-params-estimator --home /.near --accounts-num 20000 --iters 3 --warmup-iters 1 --metric icount --costs=ActionReceiptCreation,ActionTransfer,ActionCreateAccount,ActionFunctionCallBase --skip-build-test-contract --additional-accounts-num 0 --in-memory-db
-ActionReceiptCreation         214_581_685_500 gas [  1716653.48i 0.00r 0.00w]     (computed in 6.11s) 
-ActionTransfer                 21_528_212_916 gas [   172225.70i 0.00r 0.00w]     (computed in 4.71s) 
-ActionCreateAccount            26_608_336_250 gas [   212866.69i 0.00r 0.00w]     (computed in 4.67s) 
-ActionFunctionCallBase         12_193_364_898 gas [    97546.92i 0.00r 0.00w]     (computed in 2.39s) 
++ /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/qemu-x86_64 -plugin file=/host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/libcounter.so -cpu Haswell-v4 /host/nearcore/target/release/runtime-params-estimator --home /.near --accounts-num 20000 --iters 3 --warmup-iters 1 --metric icount --costs=ActionReceiptCreation,ActionTransfer,ActionCreateAccount,ActionFunctionCallBase --skip-build-test-contract --additional-accounts-num 0 --in-memory-db
+ActionReceiptCreation         214_581_685_500 gas [  1716653.48i 0.00r 0.00w]     (computed in 6.11s)
+ActionTransfer                 21_528_212_916 gas [   172225.70i 0.00r 0.00w]     (computed in 4.71s)
+ActionCreateAccount            26_608_336_250 gas [   212866.69i 0.00r 0.00w]     (computed in 4.67s)
+ActionFunctionCallBase         12_193_364_898 gas [    97546.92i 0.00r 0.00w]     (computed in 2.39s)
 
 
 Finished in 17.92s, output saved to:

runtime/runtime-params-estimator/README.md

@@ -17,18 +17,18 @@ Use this tool to measure the running time of elementary runtime operations that
 
     ```bash
     --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1 \
-      --docker --metric icount
+      --containerized --metric icount
     ```
 
-    This will run and build the estimator inside a docker container, using QEMU to precisely count
+    This will run and build the estimator inside a container, using QEMU to precisely count
     the number of executed instructions.
 
-    Alternatively you can skip docker during the development and iterate as such (actual production
-    estimations should still utilize docker to ensure reproducibility):
+    Alternatively you can skip containerization during the development and iterate as such (actual
+    production estimations should still utilize containers to ensure reproducibility):
 
     ```
     cargo build --profile=dev-release -p runtime-params-estimator --features required
-    qemu-x86 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/dev-release/runtime-params-estimator $ARGS
+    qemu-x86 -cpu Haswell-v4 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/dev-release/runtime-params-estimator $ARGS
     ```
 
     You can also significantly reduce the number of accounts in most of the cases.

runtime/runtime-params-estimator/compiler.sh

@@ -19,4 +19,4 @@ fi
 set -ex
 
 cargo build --release --package runtime-params-estimator --features $features
-./emu-cost/counter_plugin/qemu-x86_64 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/release/runtime-params-estimator --compile-only --vm-kind "$VMKIND"
+./emu-cost/counter_plugin/qemu-x86_64 -cpu Haswell-v4 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/release/runtime-params-estimator --compile-only --vm-kind "$VMKIND"

runtime/runtime-params-estimator/emu-cost/Dockerfile

@@ -1,10 +1,10 @@
 # our local base image
-FROM rust:1.76.0
+FROM docker.io/rust:1.76.0
 
 LABEL description="Container for builds"
 
-RUN rustup target add wasm32-unknown-unknown
+RUN rustup target add wasm32-unknown-unknown x86_64-unknown-linux-gnu && rustup component add rustfmt clippy
 
 # install build dependencies for QEMU
-RUN apt-get update && apt-get install -y  g++ rsync zip openssh-server \
-     make apt-utils git sudo pkg-config libglib2.0-dev curl clang gdb llvm-dev cmake
+RUN apt-get update && apt-get install -y  g++ rsync zip openssh-server make apt-utils git sudo \
+     pkg-config libglib2.0-dev curl clang gdb llvm-dev cmake python3-venv ninja-build

runtime/runtime-params-estimator/emu-cost/README.md

@@ -2,42 +2,45 @@
 
 ## Theory of operations
 
- Operation execution cost (aka gas cost) is computed basing on the number of userland x86 instructions required to perform the
+Operation execution cost (aka gas cost) is computed basing on the number of userland x86 instructions required to perform the
 particular operation in current NEAR runtime implementation. To compute this cost, we use instrumented QEMU binary
 translating engine to execute required operations in the userland Linux simulator.
 Thus, to measure the execution cost we have to compile NEAR runtime benchmark for Linux, execute the benchmark under
-instrumented QEMU running in Docker, and count how many x86 instructions are executed between start and end of execution.
+instrumented QEMU running in container, and count how many x86 instructions are executed between start and end of execution.
 
- Instrumentation of QEMU is implemented in the following manner. We install instrumentation callback which conditionally increments
+Instrumentation of QEMU is implemented in the following manner. We install instrumentation callback which conditionally increments
 the instruction counter on every instruction during translation by QEMU's JIT, TCG. We activate counting when specific Linux syscall
 (currently, 0 aka sys_read) is executed with the certain arguments (file descriptor argument == 0x0afebabe or 0x0afebabf).
 On start event we clear instruction counter, on stop event we stop counting and return counted instructions into the buffer provided
 to read syscall. As result, NEAR benchmark will know the exact instruction counter passed between two moments and this value
-is the pure function of Docker image used, Rust compiler version and the NEAR implementation and is fully reproducible.
+is the pure function of container image used, Rust compiler version and the NEAR implementation and is fully reproducible.
 
 ## Usage
 
-We build and run the cost estimator in the Docker container to make sure config is fully reproducible.
-Please make sure that Docker is given at least 4G of RAM, as running under emulator is rather resource consuming.
+**Note**: These instructions are written to work on a x86_64-linux-gnu host. The QEMU can be built
+to run on other architectures and operating systems, and it should produce reproducible results,
+but instructions to do so are out of scope for this document.
+
+We build and run the cost estimator in the container to make sure config is fully reproducible.
+Please make sure that the container is given at least 4G of RAM, as running under emulator is rather resource consuming.
 Note that for Mac the limit is configured in the desktop client, and default value most likely will be very low.
 
-First fetch appropriate base image, with `docker pull rust`.
-Then create a Docker image with `build.sh`, it will create a Docker image with additional build deps.
+First, create an image with `build.sh`.
 
 Set `HOST_DIR` environment variable to local folder where relevant sources are present.
-It will be mounted under `/host` in the Docker container.
+It will be mounted under `/host` in the container.
 
 Start container and build estimator with:
 
     host> ./run.sh
-    docker> cd /host/nearcore
-    docker> cd /host/nearcore/runtime/runtime-params-estimator
-    docker> pushd ./test-contract && ./build.sh && popd
-    docker> cargo build --release --package runtime-params-estimator --features required
+    container> cd /host/nearcore
+    container> cd /host/nearcore/runtime/runtime-params-estimator
+    container> pushd ./test-contract && ./build.sh && popd
+    container> cargo build --release --package runtime-params-estimator --features required
 
 Now start the estimator under QEMU with the counter plugin enabled (note, that Rust compiler produces SSE4, so specify recent CPU):
 
-    docker> ./emu-cost/counter_plugin/qemu-x86_64 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so \
+    container> ./emu-cost/counter_plugin/qemu-x86_64 -cpu Haswell-v4 -plugin file=./emu-cost/counter_plugin/libcounter.so \
          ../../target/release/runtime-params-estimator --accounts-num 20000 --additional-accounts-num 200000 --iters 1 --warmup-iters 1
 
 ### Notes
@@ -64,7 +67,7 @@ however, as we measure our gas in instructions, we have to compute abstract scal
 the number of bytes read/written in IO to instructions executed.
 We do that by computing following operation:
 
-    ./emu-cost/counter_plugin/qemu-x86_64  -d plugin -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so \
+    ./emu-cost/counter_plugin/qemu-x86_64  -d plugin -cpu Haswell-v4 -plugin file=./emu-cost/counter_plugin/libcounter.so \
         ../../target/release/genesis-populate --home /tmp/data --additional-accounts-num <NUM_ACCOUNTS>
 
 and checking how much data to be read/written depending on number of create accounts.
@@ -97,36 +100,31 @@ Thus, when measuring costs we set the operation cost to be:
 
 ## Optional: re-building QEMU and the instruction counter plugin
 
-We ship prebuilt QEMU and TCG instruction counter plugin, so in many cases one doesn't have to build it.
-However, in case you still want to build it - use the following steps.
+We ship prebuilt QEMU and TCG instruction counter plugin, so in many cases one doesn't have to
+build it. However, in case you still want to build it - use the following steps.
+
+First of all clone QEMU and fetch the submodules:
 
-Important: we build QEMU and the TCG plugin inside the container, so execute following commands inside Docker.
-Set environment variable HOST_DIR (on the host) to location where both QEMU and nearcore source code is checked
-out, it will be mounted as `/host` inside the Docker container.
-Start container with:
+    host> git clone https://github.com/qemu/qemu --branch v8.2.1
+    host> cd qemu
+    host> meson subprojects download
+
+**Important**: we build QEMU and the TCG plugin inside the container. Set environment variable
+HOST_DIR (on the host) to location where both QEMU and nearcore source code is checked out, it will
+be mounted as `/host` inside the container. Start the container with:
 
     ./run.sh
 
 To build QEMU use:
 
-    cd /host/qemu
-    ./configure --disable-system --enable-user --enable-plugins  --prefix=/host/qemu-linux --target-list=x86_64-linux-user
-    make && make install
+    container> cd /host/qemu
+    container> ./configure --disable-system --enable-user --enable-plugins  --prefix=/host/qemu-linux --target-list=x86_64-linux-user
+    container> make && make install
+    container> mv /host/qemu-linux/bin/qemu-x86_64 /nearcore
 
 Then build and test the QEMU's JIT plugin:
 
-    cd /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin
+    cd /nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin
     cp /host/qemu-linux/bin/qemu-x86_64 ./
-    make QEMU_DIR=/host/qemu
+    make QEMU=/host/qemu-linux/bin/qemu-x86_64
     make test
-
-To execute commands in already running container first find its id with:
-
-    > docker ps
-
-    CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES
-    e9dcb52cc91b        ubuntu-emu         "/usr/bin/env bash"   	2 hours ago         Up 2 hours          0.0.0.0:5000->22/tcp   reverent_carson
-
-and the use container ID for `docker exec` command, like:
-
-    docker exec -it e9dcb52cc91b /host/qemu-linux/bin/qemu-x86_64 -d plugin -plugin file=/host/qemu-linux/plugins/libcounter.so /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/test_binary

runtime/runtime-params-estimator/emu-cost/build.sh

@@ -1,2 +1,9 @@
-#!/bin/sh
-docker build -t rust-emu .
+#!/usr/bin/env bash
+cd "${0%/*}"
+# FIXME: does this need to use XDG? Well, whoever sets it to non-default value is on the hook for
+# making it work.
+if [[ ! -f "$HOME/.config/containers/policy.json" ]]; then
+    mkdir -p "$HOME/.config/containers"
+    cp policy.json "$HOME/.config/containers"
+fi
+exec podman --runtime=crun build -t rust-emu .

runtime/runtime-params-estimator/emu-cost/io_cost.sh

@@ -16,7 +16,7 @@ dir=/tmp/data$acc
 rm -rf $dir
 $base/../../target/release/neard  --home $dir init \
     --test-seed=alice.near --account-id=test.near --fast
-$base/emu-cost/counter_plugin/qemu-x86_64  -d plugin -cpu Westmere-v1 -R 8G \
+$base/emu-cost/counter_plugin/qemu-x86_64  -d plugin -cpu Haswell-v4 -R 8G \
       -plugin file=$base/emu-cost/counter_plugin/libcounter.so,arg="started",arg="on_every_close"  \
       $base/../../target/release/genesis-populate --home $dir --additional-accounts-num $acc 2>&1 | tee -a $log
 rm -rf $dir

runtime/runtime-params-estimator/emu-cost/policy.json

@@ -0,0 +1,7 @@
+{
+    "default": [
+        {
+            "type": "insecureAcceptAnything"
+        }
+    ]
+}

runtime/runtime-params-estimator/emu-cost/run.sh

@@ -1,6 +1,12 @@
-#!/bin/sh
-docker run \
-     --rm --mount type=bind,source=$HOST_DIR,target=/host \
+#!/usr/bin/env bash
+cd "${0%/*}"
+
+NEARCORE=$PWD/../../../
+
+exec podman --runtime=crun run --rm \
+     --mount type=bind,source=$HOST_DIR,target=/host \
+     --mount type=bind,source=$NEARCORE,target=/host/nearcore \
      --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
+     --network host \
      -i -t rust-emu \
      /usr/bin/env bash

runtime/runtime-params-estimator/estimate.sh

@@ -24,5 +24,5 @@ cd "${script_dir}"/../../genesis-tools/genesis-populate
 cargo run --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=10000000 --home /tmp/data
 cd "${script_dir}"
 cargo build --release --package runtime-params-estimator --features $features
-./emu-cost/counter_plugin/qemu-x86_64 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/release/runtime-params-estimator --home /tmp/data --accounts-num 1000000 --iters 1 --warmup-iters 0 --warmup-transactions 0 --vm-kind $vmkind
+./emu-cost/counter_plugin/qemu-x86_64 -cpu Haswell-v4 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/release/runtime-params-estimator --home /tmp/data --accounts-num 1000000 --iters 1 --warmup-iters 0 --warmup-transactions 0 --vm-kind $vmkind
 

runtime/runtime-params-estimator/estimator-warehouse/src/estimate.rs

@@ -97,7 +97,7 @@ pub(crate) fn run_estimation(db: &Db, config: &EstimateConfig) -> anyhow::Result
     if config.metrics.iter().any(|m| m == "icount") {
         let estimation_output =
             cmd!(sh,
-                "{estimator_binary} --iters {iters} --warmup-iters {warmup_iters} --json-output --home {estimator_home} --metric icount --docker"
+                "{estimator_binary} --iters {iters} --warmup-iters {warmup_iters} --json-output --home {estimator_home} --metric icount --containerize"
             ).read()?;
         db.import_json_lines(
             &ImportConfig { commit_hash: Some(commit_hash), protocol_version: None },