misc: add read-amount and read-count for benchmark

We should add the read-amount and read-count for nydus benchmark to
compare nydus and zran or different batchsize.

Signed-off-by: Desiki-high <[email protected]>

misc/benchmark/bench.py

@@ -15,6 +15,8 @@
 from datetime import datetime
 from io import TextIOWrapper
 
+import metrics
+
 
 def logging_setup(logging_stream=sys.stderr):
     root = logging.getLogger()
@@ -182,11 +184,13 @@ def run_cmd_url_wait(self, repo, runargs):
         run_elapsed = datetime.timestamp(end_run) - datetime.timestamp(start_run)
 
         print("Run time: %f s" % run_elapsed)
-
+        read_amount, read_count = "-", "-"
+        if self.snapshotter == "nydus":
+            read_amount, read_count = metrics.collect_backend()
         if self.cleanup:
             self.clean_up(image_ref, container_id)
 
-        return pull_elapsed, create_elapsed, run_elapsed
+        return pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count
 
     def pull_cmd(self, image_ref):
         insecure_flag = "--insecure-registry" if self.insecure_registry else ""
@@ -249,11 +253,11 @@ def bench_image(local_registry, insecure_local_registry, image, f: TextIOWrapper
         cleanup=True,
         insecure_registry=insecure_local_registry,
     )
-    pull_elapsed, create_elapsed, run_elapsed = runner.run(bench)
+    pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count = runner.run(bench)
     total_elapsed = f"{pull_elapsed + create_elapsed + run_elapsed: .6f}"
     pull_elapsed = f"{pull_elapsed: .6f}"
     create_elapsed = f"{create_elapsed: .6f}"
     run_elapsed = f"{run_elapsed: .6f}"
-    line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed}"
+    line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed},{read_amount},{read_count}"
     f.writelines(line + "\n")
     f.flush()

misc/benchmark/benchmark.py

@@ -43,17 +43,17 @@ def main():
 
 def collect_metrics(cfg: dict, image: str) -> str:
     """
-    collect metrics
+    collect container access metrics
     """
-    return metrics.collect(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image))
+    return metrics.collect_access(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image))
 
 
 def start_bench(cfg: dict, image: str, mode: str):
     """
     bench oci, nydus without prefetch, nydus with all prefetch, nydus witch prefetch file list
     """
     f = open(util.image_repo(image) + ".csv", "w")
-    csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s)"
+    csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s),read_amount(MB),read_count"
     f.writelines(csv_headers + "\n")
     f.flush()
     if mode == "oci":

misc/benchmark/benchmark_summary.sh

@@ -7,21 +7,26 @@ sudo install -m 755 benchmark-nydus-no-prefetch/wordpress.csv nydus-no-prefetch.
 sudo install -m 755 benchmark-nydus-all-prefetch/wordpress.csv nydus-all-prefetch.csv
 sudo install -m 755 benchmark-nydus-filelist-prefetch/wordpress.csv nydus-filelist-prefetch.csv
 
-echo "| benchmark-result | pull-elapsed(s) | create-elapsed(s) | run-elapsed(s) | total-elapsed(s) |"
-echo "|:-------|:-----------------:|:-------------------:|:----------------:|:------------------:|"
+echo "| bench-result | pull-elapsed(s) | create-elapsed(s) | run-elapsed(s) | total-elapsed(s) |read-amount(MB) |read-count |"
+echo "|:-------|:-----------------:|:-------------------:|:----------------:|:------------------:|:------------------:|:------------------:|"
 
 files=(oci.csv nydus-all-prefetch.csv zran-all-prefetch.csv nydus-no-prefetch.csv zran-no-prefetch.csv nydus-filelist-prefetch.csv)
 
 for file in "${files[@]}"; do
-if ! [ -f "$file" ]; then
-    continue
-fi
-filename=$(basename "$file" .csv)
-tail -n +2 "$file" | while read line; do
-    pull=$(echo "$line" | cut -d ',' -f 2)
-    create=$(echo "$line" | cut -d ',' -f 3)
-    run=$(echo "$line" | cut -d ',' -f 4)
-    total=$(echo "$line" | cut -d ',' -f 5)
-    printf "| %s | %s | %s | %s | %s |\n" "$filename" "$pull" "$create" "$run" "$total"
-done
+    if ! [ -f "$file" ]; then
+        continue
+    fi
+    filename=$(basename "$file" .csv)
+    tail -n +2 "$file" | while read line; do
+        if [ -z "$line" ]; then
+            continue
+        fi
+        pull=$(echo "$line" | cut -d ',' -f 2)
+        create=$(echo "$line" | cut -d ',' -f 3)
+        run=$(echo "$line" | cut -d ',' -f 4)
+        total=$(echo "$line" | cut -d ',' -f 5)
+        amount=$(echo "$line" | cut -d ',' -f 6)
+        count=$(echo "$line" | cut -d ',' -f 7)
+        printf "| %s | %s | %s | %s | %s | %s | %s |\n" "$filename" "$pull" "$create" "$run" "$total" "$amount" "$count"
+    done
 done

misc/benchmark/metrics.py

@@ -10,6 +10,7 @@
 import subprocess
 import time
 import urllib.request
+from typing import Tuple
 
 """
 define some file path, and sock api url
@@ -18,6 +19,7 @@
 TEMP_DIR = "tmp"
 URL_PREFIX = "http://localhost/api"
 ACCESS_PATTERN_METRICS = "/v1/metrics/pattern"
+BACKEND_METRICS = "/v1/metrics/backend"
 BOOTSTRAP_DIR = "/var/lib/containerd-nydus/snapshots"
 API_DIR = "/var/lib/containerd-nydus/socket"
 
@@ -189,6 +191,16 @@ def get_access_pattern(sock, bootstap_data):
     return access_pattern_list
 
 
+def get_backend_metrics(sock) -> dict:
+    """
+    get the backend metrics from the sock
+    """
+    with open(send_request(sock, BACKEND_METRICS), 'r') as file:
+        content = file.read()
+
+    return eval(content)
+
+
 def random_string():
     """
     generate a random string of fixed length
@@ -248,12 +260,28 @@ def init():
     os.mkdir(TEMP_DIR)
 
 
-def collect(local_registry, insecure_local_registry, image) -> str:
+def collect_access(local_registry, insecure_local_registry, image) -> str:
     """
-    serve for benchmark.py
+    collect access pattern for benchmark.py
     """
     init()
     cfg = {"registry": local_registry, "insecure_registry": insecure_local_registry, "image": image}
     file = MetricsCollector(cfg).start_collet()
     shutil.rmtree(TEMP_DIR)
     return file
+
+
+def collect_backend() -> Tuple[str, str]:
+    """
+    collect backend metrics for benchmark.py
+
+    return (Read Amount, Read Count)
+    """
+    init()
+    socket = search_file(API_DIR, "api.sock")
+    if socket == None:
+        print("can't find the api.sock")
+        exit(1)
+    backend = get_backend_metrics(socket)
+    shutil.rmtree(TEMP_DIR)
+    return round(backend["read_amount_total"] / 1024 / 1024, 2), backend["read_count"]