Latency bench (#463)

* latency benchmarks * latency benchmarks * setup without workers start - wip * cold start bench - warmup * test worker executor - Local compiled component service * typo fix
golemcloud · Apr 29, 2024 · 1ad35f1 · 1ad35f1
1 parent df652cc
commit 1ad35f1
Show file tree

Hide file tree

Showing 11 changed files with 311 additions and 54 deletions.
diff --git a/golem-test-framework/src/components/worker_executor/mod.rs b/golem-test-framework/src/components/worker_executor/mod.rs
@@ -94,7 +94,8 @@ fn env_vars(
         ("GOLEM__COMPONENT_SERVICE__CONFIG__HOST"        , &component_service.private_host()),
         ("GOLEM__COMPONENT_SERVICE__CONFIG__PORT"        , &component_service.private_grpc_port().to_string()),
         ("GOLEM__COMPONENT_SERVICE__CONFIG__ACCESS_TOKEN", "2A354594-7A63-4091-A46B-CC58D379F677"),
-        ("GOLEM__COMPILED_COMPONENT_SERVICE__TYPE"       , "Disabled"),
+        ("GOLEM__COMPILED_COMPONENT_SERVICE__TYPE", "Local"),
+        ("GOLEM__COMPILED_COMPONENT_SERVICE__CONFIG__ROOT", "/tmp/ittest-local-object-store/golem"),
         ("GOLEM__BLOB_STORE_SERVICE__TYPE"               , "InMemory"),
         ("GOLEM__SHARD_MANAGER_SERVICE__TYPE"            , "Grpc"),
         ("GOLEM__SHARD_MANAGER_SERVICE__CONFIG__HOST"    , &shard_manager.private_host()),

diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml
@@ -53,3 +53,16 @@ path = "src/benchmarks/cold_start_medium.rs"
 [[bin]]
 name = "benchmark_cold_start_large"
 path = "src/benchmarks/cold_start_large.rs"
+
+[[bin]]
+name = "benchmark_latency_small"
+path = "src/benchmarks/latency_small.rs"
+
+[[bin]]
+name = "benchmark_latency_medium"
+path = "src/benchmarks/latency_medium.rs"
+
+[[bin]]
+name = "benchmark_latency_large"
+path = "src/benchmarks/latency_large.rs"
+
diff --git a/integration-tests/src/benchmarks/cold_start_large.rs b/integration-tests/src/benchmarks/cold_start_large.rs
@@ -13,10 +13,13 @@
 // limitations under the License.
 
 use async_trait::async_trait;
+use golem_common::model::WorkerId;
 
 use golem_test_framework::config::{CliParams, TestDependencies};
 use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
-use integration_tests::benchmarks::{run_benchmark, run_echo, setup, Context};
+use integration_tests::benchmarks::{
+    get_worker_ids, run_benchmark, run_echo, setup, start, Context,
+};
 
 struct ColdStartEchoLarge {
     config: CliParams,
@@ -35,13 +38,23 @@ impl Benchmark for ColdStartEchoLarge {
     }
 
     async fn setup_iteration(&self) -> Self::IterationContext {
-        setup(self.config.clone(), "py-echo").await
+        setup(self.config.clone(), "py-echo", false).await
     }
 
-    async fn warmup(&self, _: &Self::IterationContext) {}
+    async fn warmup(&self, context: &Self::IterationContext) {
+        // warmup with other workers
+        if let Some(WorkerId { component_id, .. }) = context.worker_ids.clone().first() {
+            start(
+                get_worker_ids(context.worker_ids.len(), component_id, "warmup-worker"),
+                context.deps.clone(),
+            )
+            .await
+        }
+    }
 
     async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
-        run_echo(self.config.benchmark_config.length, context, recorder).await
+        // config.benchmark_config.length is not used, we want to have only one invocation per worker in this benchmark
+        run_echo(1, context, recorder).await
     }
 
     async fn cleanup_iteration(&self, context: Self::IterationContext) {

diff --git a/integration-tests/src/benchmarks/cold_start_medium.rs b/integration-tests/src/benchmarks/cold_start_medium.rs
@@ -13,10 +13,13 @@
 // limitations under the License.
 
 use async_trait::async_trait;
+use golem_common::model::WorkerId;
 
 use golem_test_framework::config::{CliParams, TestDependencies};
 use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
-use integration_tests::benchmarks::{run_benchmark, run_echo, setup, Context};
+use integration_tests::benchmarks::{
+    get_worker_ids, run_benchmark, run_echo, setup, start, Context,
+};
 
 struct ColdStartEchoMedium {
     config: CliParams,
@@ -35,13 +38,23 @@ impl Benchmark for ColdStartEchoMedium {
     }
 
     async fn setup_iteration(&self) -> Self::IterationContext {
-        setup(self.config.clone(), "js-echo").await
+        setup(self.config.clone(), "js-echo", false).await
     }
 
-    async fn warmup(&self, _: &Self::IterationContext) {}
+    async fn warmup(&self, context: &Self::IterationContext) {
+        // warmup with other workers
+        if let Some(WorkerId { component_id, .. }) = context.worker_ids.clone().first() {
+            start(
+                get_worker_ids(context.worker_ids.len(), component_id, "warmup-worker"),
+                context.deps.clone(),
+            )
+            .await
+        }
+    }
 
     async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
-        run_echo(self.config.benchmark_config.length, context, recorder).await
+        // config.benchmark_config.length is not used, we want to have only one invocation per worker in this benchmark
+        run_echo(1, context, recorder).await
     }
 
     async fn cleanup_iteration(&self, context: Self::IterationContext) {

diff --git a/integration-tests/src/benchmarks/cold_start_small.rs b/integration-tests/src/benchmarks/cold_start_small.rs
@@ -13,10 +13,13 @@
 // limitations under the License.
 
 use async_trait::async_trait;
+use golem_common::model::WorkerId;
 
 use golem_test_framework::config::{CliParams, TestDependencies};
 use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
-use integration_tests::benchmarks::{run_benchmark, run_echo, setup, Context};
+use integration_tests::benchmarks::{
+    get_worker_ids, run_benchmark, run_echo, setup, start, Context,
+};
 
 struct ColdStartEchoSmall {
     config: CliParams,
@@ -35,13 +38,23 @@ impl Benchmark for ColdStartEchoSmall {
     }
 
     async fn setup_iteration(&self) -> Self::IterationContext {
-        setup(self.config.clone(), "rust-echo").await
+        setup(self.config.clone(), "rust-echo", false).await
     }
 
-    async fn warmup(&self, _: &Self::IterationContext) {}
+    async fn warmup(&self, context: &Self::IterationContext) {
+        // warmup with other workers
+        if let Some(WorkerId { component_id, .. }) = context.worker_ids.clone().first() {
+            start(
+                get_worker_ids(context.worker_ids.len(), component_id, "warmup-worker"),
+                context.deps.clone(),
+            )
+            .await
+        }
+    }
 
     async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
-        run_echo(self.config.benchmark_config.length, context, recorder).await
+        // config.benchmark_config.length is not used, we want to have only one invocation per worker in this benchmark
+        run_echo(1, context, recorder).await
     }
 
     async fn cleanup_iteration(&self, context: Self::IterationContext) {

diff --git a/integration-tests/src/benchmarks/latency_large.rs b/integration-tests/src/benchmarks/latency_large.rs
@@ -0,0 +1,57 @@
+// Copyright 2024 Golem Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+
+use golem_test_framework::config::{CliParams, TestDependencies};
+use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
+use integration_tests::benchmarks::{run_benchmark, run_echo, setup, warmup_echo, Context};
+
+struct WorkerLatencyLarge {
+    config: CliParams,
+}
+
+#[async_trait]
+impl Benchmark for WorkerLatencyLarge {
+    type IterationContext = Context;
+
+    fn name() -> &'static str {
+        "latency-large"
+    }
+
+    async fn create(config: CliParams) -> Self {
+        Self { config }
+    }
+
+    async fn setup_iteration(&self) -> Self::IterationContext {
+        setup(self.config.clone(), "py-echo", true).await
+    }
+
+    async fn warmup(&self, context: &Self::IterationContext) {
+        warmup_echo(context).await
+    }
+
+    async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
+        run_echo(self.config.benchmark_config.length, context, recorder).await
+    }
+
+    async fn cleanup_iteration(&self, context: Self::IterationContext) {
+        context.deps.kill_all();
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    run_benchmark::<WorkerLatencyLarge>().await;
+}
diff --git a/integration-tests/src/benchmarks/latency_medium.rs b/integration-tests/src/benchmarks/latency_medium.rs
@@ -0,0 +1,57 @@
+// Copyright 2024 Golem Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+
+use golem_test_framework::config::{CliParams, TestDependencies};
+use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
+use integration_tests::benchmarks::{run_benchmark, run_echo, setup, warmup_echo, Context};
+
+struct WorkerLatencyMedium {
+    config: CliParams,
+}
+
+#[async_trait]
+impl Benchmark for WorkerLatencyMedium {
+    type IterationContext = Context;
+
+    fn name() -> &'static str {
+        "latency-medium"
+    }
+
+    async fn create(config: CliParams) -> Self {
+        Self { config }
+    }
+
+    async fn setup_iteration(&self) -> Self::IterationContext {
+        setup(self.config.clone(), "js-echo", true).await
+    }
+
+    async fn warmup(&self, context: &Self::IterationContext) {
+        warmup_echo(context).await
+    }
+
+    async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
+        run_echo(self.config.benchmark_config.length, context, recorder).await
+    }
+
+    async fn cleanup_iteration(&self, context: Self::IterationContext) {
+        context.deps.kill_all();
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    run_benchmark::<WorkerLatencyMedium>().await;
+}
diff --git a/integration-tests/src/benchmarks/latency_small.rs b/integration-tests/src/benchmarks/latency_small.rs
@@ -0,0 +1,57 @@
+// Copyright 2024 Golem Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+
+use golem_test_framework::config::{CliParams, TestDependencies};
+use golem_test_framework::dsl::benchmark::{Benchmark, BenchmarkRecorder};
+use integration_tests::benchmarks::{run_benchmark, run_echo, setup, warmup_echo, Context};
+
+struct WorkerLatencySmall {
+    config: CliParams,
+}
+
+#[async_trait]
+impl Benchmark for WorkerLatencySmall {
+    type IterationContext = Context;
+
+    fn name() -> &'static str {
+        "latency-small"
+    }
+
+    async fn create(config: CliParams) -> Self {
+        Self { config }
+    }
+
+    async fn setup_iteration(&self) -> Self::IterationContext {
+        setup(self.config.clone(), "rust-echo", true).await
+    }
+
+    async fn warmup(&self, context: &Self::IterationContext) {
+        warmup_echo(context).await
+    }
+
+    async fn run(&self, context: &Self::IterationContext, recorder: BenchmarkRecorder) {
+        run_echo(self.config.benchmark_config.length, context, recorder).await
+    }
+
+    async fn cleanup_iteration(&self, context: Self::IterationContext) {
+        context.deps.kill_all();
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    run_benchmark::<WorkerLatencySmall>().await;
+}