Add perf metrics for 2.35.0 (ray-project#47283)

``` REGRESSION 13.65%: client__get_calls (THROUGHPUT) regresses from 1119.7725751916082 to 966.9141307622872 in microbenchmark.json REGRESSION 9.23%: single_client_put_gigabytes (THROUGHPUT) regresses from 20.184014305625574 to 18.32083810818594 in microbenchmark.json REGRESSION 8.40%: multi_client_tasks_async (THROUGHPUT) regresses from 23311.858831941317 to 21353.682091539627 in microbenchmark.json REGRESSION 6.66%: 1_1_async_actor_calls_with_args_async (THROUGHPUT) regresses from 3038.941703794114 to 2836.601104413851 in microbenchmark.json REGRESSION 4.39%: 1_1_async_actor_calls_async (THROUGHPUT) regresses from 4456.606860484332 to 4261.050694056448 in microbenchmark.json REGRESSION 3.77%: actors_per_second (THROUGHPUT) regresses from 627.338335492887 to 603.6854672610009 in benchmarks/many_actors.json REGRESSION 3.47%: single_client_get_object_containing_10k_refs (THROUGHPUT) regresses from 13.679337230724197 to 13.204885454613315 in microbenchmark.json REGRESSION 3.38%: 1_1_actor_calls_sync (THROUGHPUT) regresses from 2055.7051275912527 to 1986.177233156469 in microbenchmark.json REGRESSION 2.44%: 1_1_actor_calls_concurrent (THROUGHPUT) regresses from 5167.9800954515 to 5041.760637338739 in microbenchmark.json REGRESSION 2.33%: placement_group_create/removal (THROUGHPUT) regresses from 824.4108502776797 to 805.1759941825478 in microbenchmark.json REGRESSION 1.64%: single_client_wait_1k_refs (THROUGHPUT) regresses from 5.485273551888224 to 5.39514490847805 in microbenchmark.json REGRESSION 1.28%: single_client_tasks_sync (THROUGHPUT) regresses from 986.5998779605792 to 973.959307673384 in microbenchmark.json REGRESSION 0.95%: pgs_per_second (THROUGHPUT) regresses from 22.249430148995714 to 22.037557767422825 in benchmarks/many_pgs.json REGRESSION 0.66%: n_n_actor_calls_async (THROUGHPUT) regresses from 26545.931713712664 to 26370.461840482538 in microbenchmark.json REGRESSION 0.53%: 1_1_actor_calls_async (THROUGHPUT) regresses from 9060.701663275304 to 9012.880467992636 in microbenchmark.json REGRESSION 0.28%: single_client_tasks_async (THROUGHPUT) regresses from 8011.455682416454 to 7988.9069673790045 in microbenchmark.json REGRESSION 0.19%: 1_1_async_actor_calls_sync (THROUGHPUT) regresses from 1486.2327104183764 to 1483.4703793760418 in microbenchmark.json REGRESSION 107.66%: dashboard_p95_latency_ms (LATENCY) regresses from 34.039 to 70.687 in benchmarks/many_nodes.json REGRESSION 30.19%: stage_0_time (LATENCY) regresses from 8.773437261581421 to 11.421970844268799 in stress_tests/stress_test_many_tasks.json REGRESSION 27.05%: dashboard_p50_latency_ms (LATENCY) regresses from 3.87 to 4.917 in benchmarks/many_nodes.json REGRESSION 9.72%: dashboard_p99_latency_ms (LATENCY) regresses from 119.573 to 131.198 in benchmarks/many_nodes.json REGRESSION 9.58%: stage_1_avg_iteration_time (LATENCY) regresses from 23.938837790489195 to 26.23279986381531 in stress_tests/stress_test_many_tasks.json REGRESSION 9.41%: stage_3_time (LATENCY) regresses from 3035.906775712967 to 3321.615835428238 in stress_tests/stress_test_many_tasks.json REGRESSION 6.37%: dashboard_p95_latency_ms (LATENCY) regresses from 3542.989 to 3768.817 in benchmarks/many_actors.json REGRESSION 4.93%: dashboard_p99_latency_ms (LATENCY) regresses from 358.789 to 376.468 in benchmarks/many_pgs.json REGRESSION 3.70%: time_to_broadcast_1073741824_bytes_to_50_nodes (LATENCY) regresses from 18.28579454300001 to 18.961532712000007 in scalability/object_store.json REGRESSION 3.56%: avg_pg_create_time_ms (LATENCY) regresses from 0.9371462897900398 to 0.9705077387385862 in stress_tests/stress_test_placement_group.json REGRESSION 3.24%: stage_2_avg_iteration_time (LATENCY) regresses from 61.69442081451416 to 63.694758081436156 in stress_tests/stress_test_many_tasks.json REGRESSION 2.07%: 10000_get_time (LATENCY) regresses from 23.411743029999997 to 23.896780481999997 in scalability/single_node.json REGRESSION 1.74%: dashboard_p50_latency_ms (LATENCY) regresses from 167.38 to 170.294 in benchmarks/many_tasks.json REGRESSION 1.51%: 1000000_queued_time (LATENCY) regresses from 186.319367591 to 189.12986922100004 in scalability/single_node.json REGRESSION 1.39%: avg_pg_remove_time_ms (LATENCY) regresses from 0.9081441951950084 to 0.9207600330309926 in stress_tests/stress_test_placement_group.json REGRESSION 0.59%: dashboard_p95_latency_ms (LATENCY) regresses from 12.055 to 12.126 in benchmarks/many_pgs.json ``` Signed-off-by: kevin <[email protected]> Signed-off-by: ujjawal-khare <[email protected]>
ujjawal-khare-27 · Oct 15, 2024 · ef33a5d · ef33a5d
1 parent eda80d5
commit ef33a5d
Show file tree

Hide file tree

Showing 11 changed files with 169 additions and 169 deletions.
diff --git a/release/perf_metrics/benchmarks/many_actors.json b/release/perf_metrics/benchmarks/many_actors.json
@@ -1,32 +1,32 @@
 {
-    "_dashboard_memory_usage_mb": 533.397504,
+    "_dashboard_memory_usage_mb": 485.167104,
     "_dashboard_test_success": true,
-    "_peak_memory": 3.81,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1138\t7.94GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3334\t1.82GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4274\t0.89GiB\tpython distributed/test_many_actors.py\n3452\t0.37GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n1900\t0.36GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3616\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n2136\t0.08GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3618\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n4065\t0.07GiB\tray::JobSupervisor\n3666\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-",
-    "actors_per_second": 576.4528544875533,
+    "_peak_memory": 3.74,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1215\t8.81GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n6550\t1.74GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n7868\t0.92GiB\tpython distributed/test_many_actors.py\n2138\t0.36GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n6668\t0.35GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n6842\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n2403\t0.07GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n6844\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n7668\t0.07GiB\tray::JobSupervisor\n6868\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-",
+    "actors_per_second": 603.6854672610009,
     "num_actors": 10000,
     "perf_metrics": [
         {
             "perf_metric_name": "actors_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 576.4528544875533
+            "perf_metric_value": 603.6854672610009
         },
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 133.9
+            "perf_metric_value": 104.211
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 3471.736
+            "perf_metric_value": 3768.817
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 4044.61
+            "perf_metric_value": 3786.709
         }
     ],
     "success": "1",
-    "time": 17.347472429275513
+    "time": 16.56491756439209
 }
diff --git a/release/perf_metrics/benchmarks/many_nodes.json b/release/perf_metrics/benchmarks/many_nodes.json
@@ -1,14 +1,14 @@
 {
-    "_dashboard_memory_usage_mb": 189.595648,
+    "_dashboard_memory_usage_mb": 201.936896,
     "_dashboard_test_success": true,
-    "_peak_memory": 1.7,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3362\t0.54GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n1990\t0.26GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1123\t0.25GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3478\t0.18GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n4546\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3645\t0.1GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n4824\t0.08GiB\tray::StateAPIGeneratorActor.start\n2158\t0.08GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3647\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n4339\t0.07GiB\tray::JobSupervisor",
+    "_peak_memory": 1.71,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3523\t0.54GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n1766\t0.24GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1244\t0.23GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3639\t0.19GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n5664\t0.16GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3806\t0.1GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n5894\t0.08GiB\tray::StateAPIGeneratorActor.start\n2742\t0.07GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3808\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n5469\t0.07GiB\tray::JobSupervisor",
     "num_tasks": 1000,
     "perf_metrics": [
         {
             "perf_metric_name": "tasks_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 355.14364663245095
+            "perf_metric_value": 348.36498762040105
         },
         {
             "perf_metric_name": "used_cpus_by_deadline",
@@ -18,21 +18,21 @@
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 3.983
+            "perf_metric_value": 4.917
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 39.613
+            "perf_metric_value": 70.687
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 102.131
+            "perf_metric_value": 131.198
         }
     ],
     "success": "1",
-    "tasks_per_second": 355.14364663245095,
-    "time": 302.81576204299927,
+    "tasks_per_second": 348.36498762040105,
+    "time": 302.87055253982544,
     "used_cpus": 250.0
 }
diff --git a/release/perf_metrics/benchmarks/many_pgs.json b/release/perf_metrics/benchmarks/many_pgs.json
@@ -1,32 +1,32 @@
 {
-    "_dashboard_memory_usage_mb": 137.936896,
+    "_dashboard_memory_usage_mb": 127.606784,
     "_dashboard_test_success": true,
-    "_peak_memory": 2.23,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1131\t6.89GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3279\t0.98GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4259\t0.42GiB\tpython distributed/test_many_pgs.py\n2056\t0.28GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3395\t0.11GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n3558\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n1960\t0.08GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3560\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n4044\t0.07GiB\tray::JobSupervisor\n3583\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-",
+    "_peak_memory": 2.17,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n1219\t7.63GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3423\t0.89GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4478\t0.41GiB\tpython distributed/test_many_pgs.py\n2007\t0.34GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3539\t0.11GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n3704\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n3706\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti\n2349\t0.07GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4272\t0.07GiB\tray::JobSupervisor\n3731\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-",
     "num_pgs": 1000,
     "perf_metrics": [
         {
             "perf_metric_name": "pgs_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 22.444531603492308
+            "perf_metric_value": 22.037557767422825
         },
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 3.614
+            "perf_metric_value": 3.347
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 23.021
+            "perf_metric_value": 12.126
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 256.665
+            "perf_metric_value": 376.468
         }
     ],
-    "pgs_per_second": 22.444531603492308,
+    "pgs_per_second": 22.037557767422825,
     "success": "1",
-    "time": 44.554282426834106
+    "time": 45.377079010009766
 }
diff --git a/release/perf_metrics/benchmarks/many_tasks.json b/release/perf_metrics/benchmarks/many_tasks.json
@@ -1,14 +1,14 @@
 {
-    "_dashboard_memory_usage_mb": 1280.004096,
+    "_dashboard_memory_usage_mb": 1249.652736,
     "_dashboard_test_success": true,
-    "_peak_memory": 5.0,
-    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3387\t2.1GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3503\t1.26GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n4331\t0.74GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n1885\t0.24GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1139\t0.22GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3668\t0.1GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n4535\t0.09GiB\tray::DashboardTester.run\n4609\t0.08GiB\tray::StateAPIGeneratorActor.start\n1914\t0.08GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3670\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti",
+    "_peak_memory": 5.1,
+    "_peak_process_memory": "PID\tMEM\tCOMMAND\n3389\t1.95GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3505\t1.58GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dashboa\n4343\t0.74GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n1881\t0.23GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1182\t0.23GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3669\t0.1GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/agen\n4547\t0.09GiB\tray::DashboardTester.run\n4621\t0.08GiB\tray::StateAPIGeneratorActor.start\n2239\t0.07GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3671\t0.07GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/runti",
     "num_tasks": 10000,
     "perf_metrics": [
         {
             "perf_metric_name": "tasks_per_second",
             "perf_metric_type": "THROUGHPUT",
-            "perf_metric_value": 568.6168662449907
+            "perf_metric_value": 565.2052803056663
         },
         {
             "perf_metric_name": "used_cpus_by_deadline",
@@ -18,21 +18,21 @@
         {
             "perf_metric_name": "dashboard_p50_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 69.512
+            "perf_metric_value": 170.294
         },
         {
             "perf_metric_name": "dashboard_p95_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 367.275
+            "perf_metric_value": 2826.475
         },
         {
             "perf_metric_name": "dashboard_p99_latency_ms",
             "perf_metric_type": "LATENCY",
-            "perf_metric_value": 553.345
+            "perf_metric_value": 4064.973
         }
     ],
     "success": "1",
-    "tasks_per_second": 568.6168662449907,
-    "time": 317.5865342617035,
+    "tasks_per_second": 565.2052803056663,
+    "time": 317.69268679618835,
     "used_cpus": 2500.0
 }
diff --git a/release/perf_metrics/metadata.json b/release/perf_metrics/metadata.json
@@ -1 +1 @@
-{"release_version": "2.37.0"}
+{"release_version": "2.35.0"}