Add std-libs benchmarks GH workflow (#7597)

Adds "Benchmark Standard Libraries" workflow that can be run locally with `sbt std-benchmarks/bench`.
enso-org · Aug 23, 2023 · c32bfad · c32bfad
1 parent d7e5c3f
commit c32bfad
Show file tree

Hide file tree

Showing 19 changed files with 172 additions and 38 deletions.
diff --git a/.github/workflows/benchmark.yml → .github/workflows/engine-benchmark.yml b/.github/workflows/benchmark.yml → .github/workflows/engine-benchmark.yml
@@ -69,7 +69,7 @@ jobs:
         run: ./run git-clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    timeout-minutes: 480
+    timeout-minutes: 240
 env:
   ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }}
   ENSO_BUILD_SKIP_VERSION_CHECK: "true"
diff --git a/.github/workflows/std-libs-benchmark.yml b/.github/workflows/std-libs-benchmark.yml
@@ -0,0 +1,75 @@
+# This file is auto-generated. Do not edit it manually!
+# Edit the enso_build::ci_gen module instead and run `cargo run --package enso-build-ci-gen`.
+
+name: Benchmark Standard Libraries
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+    inputs:
+      just-check:
+        description: If set, benchmarks will be only checked to run correctly, not to measure actual performance.
+        required: true
+        type: boolean
+        default: false
+jobs:
+  benchmark-standard-libraries:
+    name: Benchmark Standard Libraries
+    runs-on:
+      - benchmark
+    steps:
+      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
+        name: Setup conda (GH runners only)
+        uses: s-weigand/[email protected]
+        with:
+          update-conda: false
+          conda-channels: anaconda, conda-forge
+      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
+        name: Installing wasm-pack
+        uses: jetli/[email protected]
+        with:
+          version: v0.10.2
+      - name: Expose Artifact API and context information.
+        uses: actions/github-script@v6
+        with:
+          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
+      - if: runner.os == 'Windows'
+        name: Workaround for https://github.com/actions/checkout/issues/590 (Windows)
+        run: '"c:\Program Files\Git\bin\bash.exe" -c "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"'
+        shell: cmd
+      - if: runner.os != 'Windows'
+        name: Workaround for  https://github.com/actions/checkout/issues/590 (non-Windows)
+        run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
+        shell: bash
+      - name: Checking out the repository
+        uses: actions/checkout@v2
+        with:
+          clean: false
+          submodules: recursive
+      - name: Build Script Setup
+        run: ./run --help
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: "contains(github.event.pull_request.labels.*.name, 'CI: Clean build required')"
+        name: Clean before
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: ./run backend benchmark enso-jmh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: failure() && runner.os == 'Windows'
+        name: List files if failed (Windows)
+        run: Get-ChildItem -Force -Recurse
+      - if: failure() && runner.os != 'Windows'
+        name: List files if failed (non-Windows)
+        run: ls -lAR
+      - if: "always() && always() && contains(github.event.pull_request.labels.*.name, 'CI: Clean build required')"
+        name: Clean after
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    timeout-minutes: 240
+env:
+  ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }}
+  ENSO_BUILD_SKIP_VERSION_CHECK: "true"
diff --git a/build.sbt b/build.sbt
@@ -1886,7 +1886,14 @@ lazy val `std-benchmarks` = (project in file("std-bits/benchmarks"))
     }
   )
   .settings(
-    bench := (Benchmark / run).toTask("").tag(Exclusive).value,
+    bench := Def
+      .task {
+        (Benchmark / run).toTask("").tag(Exclusive).value
+      }
+      .dependsOn(
+        buildEngineDistribution
+      )
+      .value,
     benchOnly := Def.inputTaskDyn {
       import complete.Parsers.spaceDelimited
       val name = spaceDelimited("<name>").parsed match {
@@ -2387,6 +2394,12 @@ buildEngineDistribution := {
   log.info(s"Engine package created at $root")
 }
 
+// This makes the buildEngineDistribution task usable as a dependency
+// of other tasks.
+ThisBuild / buildEngineDistribution := {
+  buildEngineDistribution.result.value
+}
+
 lazy val buildEngineDistributionNoIndex =
   taskKey[Unit]("Builds the engine distribution without generating indexes")
 buildEngineDistributionNoIndex := {

diff --git a/build/build/paths.yaml b/build/build/paths.yaml
@@ -4,7 +4,8 @@
 <repo_root>/:
   .github/:
     workflows/:
-      benchmark.yml:
+      engine-benchmark.yml:
+      std-libs-benchmark.yml:
       changelog.yml:
       gui.yml:
       nightly.yml:
@@ -100,6 +101,9 @@
         docker-entrypoint.sh:
         Dockerfile:
     simple-library-server/:
+  std-bits/:
+    benchmarks/:
+      bench-report.xml:
   build.sbt:
   run:
   runner: # The runner native image (Linux only).

diff --git a/build/build/src/ci_gen.rs b/build/build/src/ci_gen.rs
@@ -470,7 +470,15 @@ pub fn backend() -> Result<Workflow> {
     Ok(workflow)
 }
 
-pub fn benchmark() -> Result<Workflow> {
+pub fn engine_benchmark() -> Result<Workflow> {
+    benchmark("Benchmark Engine", "backend benchmark runtime", Some(4 * 60))
+}
+
+pub fn std_libs_benchmark() -> Result<Workflow> {
+    benchmark("Benchmark Standard Libraries", "backend benchmark enso-jmh", Some(4 * 60))
+}
+
+fn benchmark(name: &str, cmd_line: &str, timeout: Option<u32>) -> Result<Workflow> {
     let just_check_input_name = "just-check";
     let just_check_input = WorkflowDispatchInput {
         r#type: WorkflowDispatchInputType::Boolean{default: Some(false)},
@@ -483,21 +491,21 @@ pub fn benchmark() -> Result<Workflow> {
         schedule: vec![Schedule::new("0 0 * * *")?],
         ..default()
     };
-    let mut workflow = Workflow { name: "Benchmark Engine".into(), on, ..default() };
+    let mut workflow = Workflow { name: name.into(), on, ..default() };
     // Note that we need to use `true == input` instead of `input` because that interprets input as
     // `false` rather than empty string. Empty string is not falsy enough.
     workflow.env(
         "ENSO_BUILD_MINIMAL_RUN",
         wrap_expression(format!("true == inputs.{just_check_input_name}")),
     );
 
-    let mut benchmark_job =
-        plain_job(&BenchmarkRunner, "Benchmark Engine", "backend benchmark runtime");
-    benchmark_job.timeout_minutes = Some(60 * 8);
+    let mut benchmark_job = plain_job(&BenchmarkRunner, name, cmd_line);
+    benchmark_job.timeout_minutes = timeout;
     workflow.add_job(benchmark_job);
     Ok(workflow)
 }
 
+
 /// Generate workflows for the CI.
 pub fn generate(
     repo_root: &crate::paths::generated::RepoRootGithubWorkflows,
@@ -507,7 +515,8 @@ pub fn generate(
         (repo_root.nightly_yml.to_path_buf(), nightly()?),
         (repo_root.scala_new_yml.to_path_buf(), backend()?),
         (repo_root.gui_yml.to_path_buf(), gui()?),
-        (repo_root.benchmark_yml.to_path_buf(), benchmark()?),
+        (repo_root.engine_benchmark_yml.to_path_buf(), engine_benchmark()?),
+        (repo_root.std_libs_benchmark_yml.to_path_buf(), std_libs_benchmark()?),
         (repo_root.release_yml.to_path_buf(), release()?),
         (repo_root.promote_yml.to_path_buf(), promote()?),
     ];

diff --git a/build/build/src/engine.rs b/build/build/src/engine.rs
@@ -88,6 +88,8 @@ pub enum Benchmarks {
     Runtime,
     /// Run benchmarks written in pure Enso.
     Enso,
+    /// Run Enso benchmarks via JMH
+    EnsoJMH,
 }
 
 #[derive(Clone, Copy, Debug, Display, PartialEq, Eq, PartialOrd, Ord, clap::ArgEnum)]
@@ -103,6 +105,7 @@ impl Benchmarks {
             Benchmarks::All => Some("bench"),
             Benchmarks::Runtime => Some("runtime/bench"),
             Benchmarks::Enso => None,
+            Benchmarks::EnsoJMH => Some("std-benchmarks/bench"),
         }
     }
 }

diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs
@@ -323,6 +323,7 @@ impl RunContext {
                     "runtime/Benchmark/compile",
                     "language-server/Benchmark/compile",
                     "searcher/Benchmark/compile",
+                    "std-benchmarks/Benchmark/compile",
                 ]);
             }
 
@@ -372,6 +373,9 @@ impl RunContext {
 
                 // Check Searcher Benchmark Compilation
                 sbt.call_arg("searcher/Benchmark/compile").await?;
+
+                // Check Enso JMH benchmark compilation
+                sbt.call_arg("std-benchmarks/Benchmark/compile").await?;
             }
 
             for benchmark in &self.config.execute_benchmarks {
@@ -419,15 +423,42 @@ impl RunContext {
 
         // If we were running any benchmarks, they are complete by now. Upload the report.
         if is_in_env() {
-            let path = &self.paths.repo_root.engine.runtime.bench_report_xml;
-            if path.exists() {
-                ide_ci::actions::artifacts::upload_single_file(
-                    &self.paths.repo_root.engine.runtime.bench_report_xml,
-                    "Runtime Benchmark Report",
-                )
-                .await?;
-            } else {
-                info!("No benchmark file found at {}, nothing to upload.", path.display());
+            for bench in &self.config.execute_benchmarks {
+                match bench {
+                    Benchmarks::Runtime => {
+                        let runtime_bench_report =
+                            &self.paths.repo_root.engine.runtime.bench_report_xml;
+                        if runtime_bench_report.exists() {
+                            ide_ci::actions::artifacts::upload_single_file(
+                                runtime_bench_report,
+                                "Runtime Benchmark Report",
+                            )
+                            .await?;
+                        } else {
+                            warn!(
+                                "No Runtime Benchmark Report file found at {}, nothing to upload.",
+                                runtime_bench_report.display()
+                            );
+                        }
+                    }
+                    Benchmarks::EnsoJMH => {
+                        let enso_jmh_report =
+                            &self.paths.repo_root.std_bits.benchmarks.bench_report_xml;
+                        if enso_jmh_report.exists() {
+                            ide_ci::actions::artifacts::upload_single_file(
+                                enso_jmh_report,
+                                "Enso JMH Benchmark Report",
+                            )
+                            .await?;
+                        } else {
+                            warn!(
+                                "No Enso JMH Benchmark Report file found at {}, nothing to upload.",
+                                enso_jmh_report.display()
+                            );
+                        }
+                    }
+                    _ => {}
+                }
             }
         }
 

diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java
@@ -226,9 +226,11 @@ private void generateClassForGroup(
     out.println("  ");
     out.println("  @Setup");
     out.println("  public void setup(BenchmarkParams params) throws Exception {");
+    // Workaround for compilation failures on Windows.
+    String projectRootDirPath = projectRootDir.getPath().contains("\\") ? projectRootDir.getPath().replace("\\", "\\\\") : projectRootDir.getPath();
     out
         .append("    File projectRootDir = Utils.findRepoRootDir().toPath().resolve(\"")
-        .append(projectRootDir.toString())
+        .append(projectRootDirPath)
         .append("\").toFile();\n");
     out.println(
         "    if (projectRootDir == null || !projectRootDir.exists() || !projectRootDir.canRead()) {");

diff --git a/test/Benchmarks/src/Collections.enso b/test/Benchmarks/src/Collections.enso
@@ -22,8 +22,6 @@ build_map size =
     rand = Java_Random.new
     0.up_to size . fold Map.empty (m -> i -> m.insert (rand.nextInt 10000) i)
 
-options = Bench.options
-
 type Data
     Value ~list ~vec ~vec_decimal
 
@@ -34,7 +32,7 @@ type Data
 collect_benches = Bench.build builder->
     data = Data.create
 
-    builder.group "Collections" options group_builder->
+    builder.group "Collections" (Bench.options.set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 2 3)) group_builder->
         group_builder.specify "list_meta_fold" <|
             sum_list_meta data.list
 
@@ -47,8 +45,9 @@ collect_benches = Bench.build builder->
         group_builder.specify "vector_decimal_fold" <|
             data.vec_decimal.fold 0 (+)
 
+    builder.group "Collections_Map" (Bench.options.set_warmup (Bench.phase_conf 3 5) . set_measure (Bench.phase_conf 3 5)) group_builder->
         group_builder.specify "build_map" <|
-            build_map 10000
+            build_map 5000
 
 
 main = collect_benches . run_main
diff --git a/test/Benchmarks/src/Column_Numeric.enso b/test/Benchmarks/src/Column_Numeric.enso
@@ -25,7 +25,7 @@ create_floats vector_size faker =
 
 
 collect_benches = Bench.build builder->
-    vector_size = 1000000
+    vector_size = 5 * 1000 * 1000
     ## No specific significance to this constant, just fixed to make generated set deterministic
     fixed_random_seed = 1644575867
     faker = Faker.new fixed_random_seed

diff --git a/test/Benchmarks/src/Natural_Order_Sort.enso b/test/Benchmarks/src/Natural_Order_Sort.enso
@@ -2,7 +2,7 @@ from Standard.Base import all
 
 from Standard.Test import Bench, Faker
 
-options = Bench.options . set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3)
+options = Bench.options . set_warmup (Bench.phase_conf 1 8) . set_measure (Bench.phase_conf 1 4)
 
 
 type Data
@@ -20,7 +20,7 @@ create_unsorted vector_size faker =
 
 
 collect_benches = Bench.build builder->
-    vector_size = 10000
+    vector_size = 7000
 
     ## No specific significance to this constant, just fixed to make generated set deterministic
     fixed_random_seed = 1644575867

diff --git a/test/Benchmarks/src/Number_Parse.enso b/test/Benchmarks/src/Number_Parse.enso
@@ -22,7 +22,7 @@ create_int_strings vector_size faker =
 
 
 collect_benches = Bench.build builder->
-    vector_size = 1000000
+    vector_size = 500 * 1000
     ## No specific significance to this constant, just fixed to make generated set deterministic
     fixed_random_seed = 1644575867
     faker = Faker.new fixed_random_seed

diff --git a/test/Benchmarks/src/Table/Sorting.enso b/test/Benchmarks/src/Table/Sorting.enso
@@ -42,7 +42,7 @@ type Data
         Data.Value create_ints create_dates create_objects create_ints_table create_dates_table create_objects_table
 
 
-options = Bench.options . set_warmup (Bench.phase_conf 1 3) . set_measure (Bench.phase_conf 1 3)
+options = Bench.options . set_warmup (Bench.phase_conf 1 7) . set_measure (Bench.phase_conf 1 3)
 
 
 collect_benches = Bench.build builder->

diff --git a/test/Benchmarks/src/Text/Compare.enso b/test/Benchmarks/src/Text/Compare.enso
@@ -10,9 +10,6 @@ compare_all_adjacent text_vector =
     res
 
 
-options = Bench.options . set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3)
-
-
 create_very_short_template character_template =
     Vector.new 4 _-> character_template
 
@@ -77,13 +74,14 @@ collect_benches = Bench.build builder->
 
         data = Data.create character_template faker common_prefix
 
-        builder.group ("Text_Compare_" + suite_prefix) options group_builder->
+        builder.group ("Text_Compare_Small_" + suite_prefix) (Bench.options.set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3)) group_builder->
             group_builder.specify "very_short" <|
                 compare_all_adjacent data.very_short
 
             group_builder.specify "medium" <|
                 compare_all_adjacent data.medium
 
+        builder.group ("Text_Compare_Big_" + suite_prefix) (Bench.options.set_warmup (Bench.phase_conf 3 5) . set_measure (Bench.phase_conf 3 3)) group_builder->
             group_builder.specify "big_random" <|
                 compare_all_adjacent data.big_random
 

diff --git a/test/Benchmarks/src/Text/Contains.enso b/test/Benchmarks/src/Text/Contains.enso
@@ -3,7 +3,7 @@ from Standard.Base import all
 from Standard.Test import Bench, Faker
 
 
-options = Bench.options . set_warmup (Bench.phase_conf 1 3) . set_measure (Bench.phase_conf 1 3)
+options = Bench.options . set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 2 3)
 
 
 check_all text_vector pattern_vector mode =