diff --git a/.github/workflows/benchmark.yml b/.github/workflows/engine-benchmark.yml similarity index 99% rename from .github/workflows/benchmark.yml rename to .github/workflows/engine-benchmark.yml index 8450e3fe12a6..b30e3920350d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/engine-benchmark.yml @@ -69,7 +69,7 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 480 + timeout-minutes: 240 env: ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }} ENSO_BUILD_SKIP_VERSION_CHECK: "true" diff --git a/.github/workflows/std-libs-benchmark.yml b/.github/workflows/std-libs-benchmark.yml new file mode 100644 index 000000000000..7644af00e8ac --- /dev/null +++ b/.github/workflows/std-libs-benchmark.yml @@ -0,0 +1,75 @@ +# This file is auto-generated. Do not edit it manually! +# Edit the enso_build::ci_gen module instead and run `cargo run --package enso-build-ci-gen`. + +name: Benchmark Standard Libraries +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + inputs: + just-check: + description: If set, benchmarks will be only checked to run correctly, not to measure actual performance. + required: true + type: boolean + default: false +jobs: + benchmark-standard-libraries: + name: Benchmark Standard Libraries + runs-on: + - benchmark + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.0.6 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.3.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v6 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - if: runner.os == 'Windows' + name: Workaround for https://github.com/actions/checkout/issues/590 (Windows) + run: '"c:\Program Files\Git\bin\bash.exe" -c "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"' + shell: cmd + - if: runner.os != 'Windows' + name: Workaround for https://github.com/actions/checkout/issues/590 (non-Windows) + run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :" + shell: bash + - name: Checking out the repository + uses: actions/checkout@v2 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: "contains(github.event.pull_request.labels.*.name, 'CI: Clean build required')" + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend benchmark enso-jmh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: "always() && always() && contains(github.event.pull_request.labels.*.name, 'CI: Clean build required')" + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 240 +env: + ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }} + ENSO_BUILD_SKIP_VERSION_CHECK: "true" diff --git a/build.sbt b/build.sbt index 0571c2f2ca08..ac8083cff276 100644 --- a/build.sbt +++ b/build.sbt @@ -1886,7 +1886,14 @@ lazy val `std-benchmarks` = (project in file("std-bits/benchmarks")) } ) .settings( - bench := (Benchmark / run).toTask("").tag(Exclusive).value, + bench := Def + .task { + (Benchmark / run).toTask("").tag(Exclusive).value + } + .dependsOn( + buildEngineDistribution + ) + .value, benchOnly := Def.inputTaskDyn { import complete.Parsers.spaceDelimited val name = spaceDelimited("").parsed match { @@ -2387,6 +2394,12 @@ buildEngineDistribution := { log.info(s"Engine package created at $root") } +// This makes the buildEngineDistribution task usable as a dependency +// of other tasks. +ThisBuild / buildEngineDistribution := { + buildEngineDistribution.result.value +} + lazy val buildEngineDistributionNoIndex = taskKey[Unit]("Builds the engine distribution without generating indexes") buildEngineDistributionNoIndex := { diff --git a/build/build/paths.yaml b/build/build/paths.yaml index e0e64699f9ac..e0a319d4a14d 100644 --- a/build/build/paths.yaml +++ b/build/build/paths.yaml @@ -4,7 +4,8 @@ /: .github/: workflows/: - benchmark.yml: + engine-benchmark.yml: + std-libs-benchmark.yml: changelog.yml: gui.yml: nightly.yml: @@ -100,6 +101,9 @@ docker-entrypoint.sh: Dockerfile: simple-library-server/: + std-bits/: + benchmarks/: + bench-report.xml: build.sbt: run: runner: # The runner native image (Linux only). diff --git a/build/build/src/ci_gen.rs b/build/build/src/ci_gen.rs index caaba18f9a31..e74e59cd213d 100644 --- a/build/build/src/ci_gen.rs +++ b/build/build/src/ci_gen.rs @@ -470,7 +470,15 @@ pub fn backend() -> Result { Ok(workflow) } -pub fn benchmark() -> Result { +pub fn engine_benchmark() -> Result { + benchmark("Benchmark Engine", "backend benchmark runtime", Some(4 * 60)) +} + +pub fn std_libs_benchmark() -> Result { + benchmark("Benchmark Standard Libraries", "backend benchmark enso-jmh", Some(4 * 60)) +} + +fn benchmark(name: &str, cmd_line: &str, timeout: Option) -> Result { let just_check_input_name = "just-check"; let just_check_input = WorkflowDispatchInput { r#type: WorkflowDispatchInputType::Boolean{default: Some(false)}, @@ -483,7 +491,7 @@ pub fn benchmark() -> Result { schedule: vec![Schedule::new("0 0 * * *")?], ..default() }; - let mut workflow = Workflow { name: "Benchmark Engine".into(), on, ..default() }; + let mut workflow = Workflow { name: name.into(), on, ..default() }; // Note that we need to use `true == input` instead of `input` because that interprets input as // `false` rather than empty string. Empty string is not falsy enough. workflow.env( @@ -491,13 +499,13 @@ pub fn benchmark() -> Result { wrap_expression(format!("true == inputs.{just_check_input_name}")), ); - let mut benchmark_job = - plain_job(&BenchmarkRunner, "Benchmark Engine", "backend benchmark runtime"); - benchmark_job.timeout_minutes = Some(60 * 8); + let mut benchmark_job = plain_job(&BenchmarkRunner, name, cmd_line); + benchmark_job.timeout_minutes = timeout; workflow.add_job(benchmark_job); Ok(workflow) } + /// Generate workflows for the CI. pub fn generate( repo_root: &crate::paths::generated::RepoRootGithubWorkflows, @@ -507,7 +515,8 @@ pub fn generate( (repo_root.nightly_yml.to_path_buf(), nightly()?), (repo_root.scala_new_yml.to_path_buf(), backend()?), (repo_root.gui_yml.to_path_buf(), gui()?), - (repo_root.benchmark_yml.to_path_buf(), benchmark()?), + (repo_root.engine_benchmark_yml.to_path_buf(), engine_benchmark()?), + (repo_root.std_libs_benchmark_yml.to_path_buf(), std_libs_benchmark()?), (repo_root.release_yml.to_path_buf(), release()?), (repo_root.promote_yml.to_path_buf(), promote()?), ]; diff --git a/build/build/src/engine.rs b/build/build/src/engine.rs index f28217f47985..111a03a31170 100644 --- a/build/build/src/engine.rs +++ b/build/build/src/engine.rs @@ -88,6 +88,8 @@ pub enum Benchmarks { Runtime, /// Run benchmarks written in pure Enso. Enso, + /// Run Enso benchmarks via JMH + EnsoJMH, } #[derive(Clone, Copy, Debug, Display, PartialEq, Eq, PartialOrd, Ord, clap::ArgEnum)] @@ -103,6 +105,7 @@ impl Benchmarks { Benchmarks::All => Some("bench"), Benchmarks::Runtime => Some("runtime/bench"), Benchmarks::Enso => None, + Benchmarks::EnsoJMH => Some("std-benchmarks/bench"), } } } diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index c975b367264e..61b807a53894 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -323,6 +323,7 @@ impl RunContext { "runtime/Benchmark/compile", "language-server/Benchmark/compile", "searcher/Benchmark/compile", + "std-benchmarks/Benchmark/compile", ]); } @@ -372,6 +373,9 @@ impl RunContext { // Check Searcher Benchmark Compilation sbt.call_arg("searcher/Benchmark/compile").await?; + + // Check Enso JMH benchmark compilation + sbt.call_arg("std-benchmarks/Benchmark/compile").await?; } for benchmark in &self.config.execute_benchmarks { @@ -419,15 +423,42 @@ impl RunContext { // If we were running any benchmarks, they are complete by now. Upload the report. if is_in_env() { - let path = &self.paths.repo_root.engine.runtime.bench_report_xml; - if path.exists() { - ide_ci::actions::artifacts::upload_single_file( - &self.paths.repo_root.engine.runtime.bench_report_xml, - "Runtime Benchmark Report", - ) - .await?; - } else { - info!("No benchmark file found at {}, nothing to upload.", path.display()); + for bench in &self.config.execute_benchmarks { + match bench { + Benchmarks::Runtime => { + let runtime_bench_report = + &self.paths.repo_root.engine.runtime.bench_report_xml; + if runtime_bench_report.exists() { + ide_ci::actions::artifacts::upload_single_file( + runtime_bench_report, + "Runtime Benchmark Report", + ) + .await?; + } else { + warn!( + "No Runtime Benchmark Report file found at {}, nothing to upload.", + runtime_bench_report.display() + ); + } + } + Benchmarks::EnsoJMH => { + let enso_jmh_report = + &self.paths.repo_root.std_bits.benchmarks.bench_report_xml; + if enso_jmh_report.exists() { + ide_ci::actions::artifacts::upload_single_file( + enso_jmh_report, + "Enso JMH Benchmark Report", + ) + .await?; + } else { + warn!( + "No Enso JMH Benchmark Report file found at {}, nothing to upload.", + enso_jmh_report.display() + ); + } + } + _ => {} + } } } diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 3a0458d6f5a5..b4281bd41a2d 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -226,9 +226,11 @@ private void generateClassForGroup( out.println(" "); out.println(" @Setup"); out.println(" public void setup(BenchmarkParams params) throws Exception {"); + // Workaround for compilation failures on Windows. + String projectRootDirPath = projectRootDir.getPath().contains("\\") ? projectRootDir.getPath().replace("\\", "\\\\") : projectRootDir.getPath(); out .append(" File projectRootDir = Utils.findRepoRootDir().toPath().resolve(\"") - .append(projectRootDir.toString()) + .append(projectRootDirPath) .append("\").toFile();\n"); out.println( " if (projectRootDir == null || !projectRootDir.exists() || !projectRootDir.canRead()) {"); diff --git a/test/Benchmarks/src/Collections.enso b/test/Benchmarks/src/Collections.enso index c6b5c9ac8ea3..1ce0def136a6 100644 --- a/test/Benchmarks/src/Collections.enso +++ b/test/Benchmarks/src/Collections.enso @@ -22,8 +22,6 @@ build_map size = rand = Java_Random.new 0.up_to size . fold Map.empty (m -> i -> m.insert (rand.nextInt 10000) i) -options = Bench.options - type Data Value ~list ~vec ~vec_decimal @@ -34,7 +32,7 @@ type Data collect_benches = Bench.build builder-> data = Data.create - builder.group "Collections" options group_builder-> + builder.group "Collections" (Bench.options.set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 2 3)) group_builder-> group_builder.specify "list_meta_fold" <| sum_list_meta data.list @@ -47,8 +45,9 @@ collect_benches = Bench.build builder-> group_builder.specify "vector_decimal_fold" <| data.vec_decimal.fold 0 (+) + builder.group "Collections_Map" (Bench.options.set_warmup (Bench.phase_conf 3 5) . set_measure (Bench.phase_conf 3 5)) group_builder-> group_builder.specify "build_map" <| - build_map 10000 + build_map 5000 main = collect_benches . run_main diff --git a/test/Benchmarks/src/Column_Numeric.enso b/test/Benchmarks/src/Column_Numeric.enso index 1dd55625cc12..69f4b08b0b53 100644 --- a/test/Benchmarks/src/Column_Numeric.enso +++ b/test/Benchmarks/src/Column_Numeric.enso @@ -25,7 +25,7 @@ create_floats vector_size faker = collect_benches = Bench.build builder-> - vector_size = 1000000 + vector_size = 5 * 1000 * 1000 ## No specific significance to this constant, just fixed to make generated set deterministic fixed_random_seed = 1644575867 faker = Faker.new fixed_random_seed diff --git a/test/Benchmarks/src/Natural_Order_Sort.enso b/test/Benchmarks/src/Natural_Order_Sort.enso index 0ae2dad97434..bead4d5885a6 100644 --- a/test/Benchmarks/src/Natural_Order_Sort.enso +++ b/test/Benchmarks/src/Natural_Order_Sort.enso @@ -2,7 +2,7 @@ from Standard.Base import all from Standard.Test import Bench, Faker -options = Bench.options . set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3) +options = Bench.options . set_warmup (Bench.phase_conf 1 8) . set_measure (Bench.phase_conf 1 4) type Data @@ -20,7 +20,7 @@ create_unsorted vector_size faker = collect_benches = Bench.build builder-> - vector_size = 10000 + vector_size = 7000 ## No specific significance to this constant, just fixed to make generated set deterministic fixed_random_seed = 1644575867 diff --git a/test/Benchmarks/src/Number_Parse.enso b/test/Benchmarks/src/Number_Parse.enso index fd58ef5ff72a..14dd9258a665 100644 --- a/test/Benchmarks/src/Number_Parse.enso +++ b/test/Benchmarks/src/Number_Parse.enso @@ -22,7 +22,7 @@ create_int_strings vector_size faker = collect_benches = Bench.build builder-> - vector_size = 1000000 + vector_size = 500 * 1000 ## No specific significance to this constant, just fixed to make generated set deterministic fixed_random_seed = 1644575867 faker = Faker.new fixed_random_seed diff --git a/test/Benchmarks/src/Table/Sorting.enso b/test/Benchmarks/src/Table/Sorting.enso index 97033b89c1ca..def0978e2ed2 100644 --- a/test/Benchmarks/src/Table/Sorting.enso +++ b/test/Benchmarks/src/Table/Sorting.enso @@ -42,7 +42,7 @@ type Data Data.Value create_ints create_dates create_objects create_ints_table create_dates_table create_objects_table -options = Bench.options . set_warmup (Bench.phase_conf 1 3) . set_measure (Bench.phase_conf 1 3) +options = Bench.options . set_warmup (Bench.phase_conf 1 7) . set_measure (Bench.phase_conf 1 3) collect_benches = Bench.build builder-> diff --git a/test/Benchmarks/src/Text/Compare.enso b/test/Benchmarks/src/Text/Compare.enso index d2b46a937797..97d0f8babb5a 100644 --- a/test/Benchmarks/src/Text/Compare.enso +++ b/test/Benchmarks/src/Text/Compare.enso @@ -10,9 +10,6 @@ compare_all_adjacent text_vector = res -options = Bench.options . set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3) - - create_very_short_template character_template = Vector.new 4 _-> character_template @@ -77,13 +74,14 @@ collect_benches = Bench.build builder-> data = Data.create character_template faker common_prefix - builder.group ("Text_Compare_" + suite_prefix) options group_builder-> + builder.group ("Text_Compare_Small_" + suite_prefix) (Bench.options.set_warmup (Bench.phase_conf 1 5) . set_measure (Bench.phase_conf 1 3)) group_builder-> group_builder.specify "very_short" <| compare_all_adjacent data.very_short group_builder.specify "medium" <| compare_all_adjacent data.medium + builder.group ("Text_Compare_Big_" + suite_prefix) (Bench.options.set_warmup (Bench.phase_conf 3 5) . set_measure (Bench.phase_conf 3 3)) group_builder-> group_builder.specify "big_random" <| compare_all_adjacent data.big_random diff --git a/test/Benchmarks/src/Text/Contains.enso b/test/Benchmarks/src/Text/Contains.enso index 3f3361f1d5ca..28a2e24c84cc 100644 --- a/test/Benchmarks/src/Text/Contains.enso +++ b/test/Benchmarks/src/Text/Contains.enso @@ -3,7 +3,7 @@ from Standard.Base import all from Standard.Test import Bench, Faker -options = Bench.options . set_warmup (Bench.phase_conf 1 3) . set_measure (Bench.phase_conf 1 3) +options = Bench.options . set_warmup (Bench.phase_conf 2 3) . set_measure (Bench.phase_conf 2 3) check_all text_vector pattern_vector mode = diff --git a/test/Benchmarks/src/Time/Work_Days.enso b/test/Benchmarks/src/Time/Work_Days.enso index 09c775942249..072e273a2cce 100644 --- a/test/Benchmarks/src/Time/Work_Days.enso +++ b/test/Benchmarks/src/Time/Work_Days.enso @@ -2,7 +2,7 @@ from Standard.Base import all from Standard.Test import Bench -options = Bench.options . set_warmup (Bench.phase_conf 1 2) . set_measure (Bench.phase_conf 1 2) +options = Bench.options . set_warmup (Bench.phase_conf 1 4) . set_measure (Bench.phase_conf 1 4) type Data diff --git a/test/Benchmarks/src/Vector/Array_Proxy_Bench.enso b/test/Benchmarks/src/Vector/Array_Proxy_Bench.enso index 1d307ef5f84c..fb83e8831337 100644 --- a/test/Benchmarks/src/Vector/Array_Proxy_Bench.enso +++ b/test/Benchmarks/src/Vector/Array_Proxy_Bench.enso @@ -32,7 +32,7 @@ type Data collect_benches = Bench.build builder-> - vector_size = 10000 + vector_size = 1000 * 1000 data = Data.create vector_size builder.group "Vector_Array_Proxy" Bench.options group_builder-> diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index c867930bf8a2..8591f5ebeb93 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -9,7 +9,7 @@ import project.Vector.Utils polyglot java import java.util.Random as Java_Random -options = Bench.options . set_warmup (Bench.phase_conf 1 2) . set_measure (Bench.phase_conf 1 2) +options = Bench.options . set_warmup (Bench.phase_conf 2 5) . set_measure (Bench.phase_conf 1 5) collect_benches = Bench.build builder-> diff --git a/test/Benchmarks/src/Vector/Sort.enso b/test/Benchmarks/src/Vector/Sort.enso index f9d8c7f15599..a9af7102b5ff 100644 --- a/test/Benchmarks/src/Vector/Sort.enso +++ b/test/Benchmarks/src/Vector/Sort.enso @@ -54,7 +54,7 @@ type Int # The Benchmarks ============================================================== -options = Bench.options . set_warmup (Bench.phase_conf 1 3) . set_measure (Bench.phase_conf 1 3) +options = Bench.options . set_warmup (Bench.phase_conf 1 4) . set_measure (Bench.phase_conf 1 3) type Data @@ -71,7 +71,7 @@ type Data collect_benches = Bench.build builder-> - vector_size = 1000 * 1000 + vector_size = 100 * 1000 data = Data.create vector_size projection = x -> x % 10 comparator = l -> r -> Ordering.compare l r