diff --git a/Cargo.lock b/Cargo.lock index 612e769d..41b4a044 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,9 +660,11 @@ name = "heroku-nodejs-engine-buildpack" version = "0.0.0" dependencies = [ "heroku-nodejs-utils", + "indoc", "libcnb 0.26.0", "libcnb-test", "libherokubuildpack 0.26.0", + "regex", "serde", "serde_json", "sha2", diff --git a/buildpacks/nodejs-engine/CHANGELOG.md b/buildpacks/nodejs-engine/CHANGELOG.md index 00d39008..e5971ba5 100644 --- a/buildpacks/nodejs-engine/CHANGELOG.md +++ b/buildpacks/nodejs-engine/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Set `HEROKU_AVAILABLE_PARALLELISM` environment variable at build and run time + ## [3.3.4] - 2024-12-05 ### Added @@ -722,55 +726,107 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [unreleased]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.4...HEAD [3.3.4]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.3...v3.3.4 [3.3.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.2...v3.3.3 + [3.3.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.1...v3.3.2 + [3.3.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.3.0...v3.3.1 + [3.3.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.18...v3.3.0 + [3.2.18]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.17...v3.2.18 + [3.2.17]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.16...v3.2.17 + [3.2.16]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.15...v3.2.16 + [3.2.15]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.14...v3.2.15 + [3.2.14]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.13...v3.2.14 + [3.2.13]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.12...v3.2.13 + [3.2.12]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.11...v3.2.12 + [3.2.11]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.10...v3.2.11 + [3.2.10]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.9...v3.2.10 + [3.2.9]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.8...v3.2.9 + [3.2.8]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.7...v3.2.8 + [3.2.7]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.6...v3.2.7 + [3.2.6]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.5...v3.2.6 + [3.2.5]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.4...v3.2.5 + [3.2.4]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.3...v3.2.4 + [3.2.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.2...v3.2.3 + [3.2.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.1...v3.2.2 + [3.2.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.2.0...v3.2.1 + [3.2.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.1.0...v3.2.0 + [3.1.0]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.6...v3.1.0 + [3.0.6]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.5...v3.0.6 + [3.0.5]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.4...v3.0.5 + [3.0.4]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.3...v3.0.4 + [3.0.3]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.2...v3.0.3 + [3.0.2]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.1...v3.0.2 + [3.0.1]: https://github.com/heroku/buildpacks-nodejs/compare/v3.0.0...v3.0.1 + [3.0.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.6...v3.0.0 + [2.6.6]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.5...v2.6.6 + [2.6.5]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.4...v2.6.5 + [2.6.4]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.3...v2.6.4 + [2.6.3]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.2...v2.6.3 + [2.6.2]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.1...v2.6.2 + [2.6.1]: https://github.com/heroku/buildpacks-nodejs/compare/v2.6.0...v2.6.1 + [2.6.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.5.0...v2.6.0 + [2.5.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.4.1...v2.5.0 + [2.4.1]: https://github.com/heroku/buildpacks-nodejs/compare/v2.4.0...v2.4.1 + [2.4.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.3.0...v2.4.0 + [2.3.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.2.0...v2.3.0 + [2.2.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.1.0...v2.2.0 + [2.1.0]: https://github.com/heroku/buildpacks-nodejs/compare/v2.0.0...v2.1.0 + [2.0.0]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.7...v2.0.0 + [1.1.7]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.6...v1.1.7 + [1.1.6]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.5...v1.1.6 + [1.1.5]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.4...v1.1.5 + [1.1.4]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.3...v1.1.4 + [1.1.3]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.2...v1.1.3 + [1.1.2]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.1...v1.1.2 + [1.1.1]: https://github.com/heroku/buildpacks-nodejs/compare/v1.1.0...v1.1.1 + [1.1.0]: https://github.com/heroku/buildpacks-nodejs/releases/tag/v1.1.0 diff --git a/buildpacks/nodejs-engine/Cargo.toml b/buildpacks/nodejs-engine/Cargo.toml index 5c128a70..78695c35 100644 --- a/buildpacks/nodejs-engine/Cargo.toml +++ b/buildpacks/nodejs-engine/Cargo.toml @@ -17,7 +17,9 @@ thiserror = "2" toml = "0.8" [dev-dependencies] +indoc = "2" libcnb-test = "=0.26.0" +regex = "1" serde_json = "1" test_support.workspace = true ureq = "2" diff --git a/buildpacks/nodejs-engine/src/bin/available_parallelism.rs b/buildpacks/nodejs-engine/src/bin/available_parallelism.rs new file mode 100644 index 00000000..de5461ac --- /dev/null +++ b/buildpacks/nodejs-engine/src/bin/available_parallelism.rs @@ -0,0 +1,17 @@ +// Required due to: https://github.com/rust-lang/rust/issues/95513 +#![allow(unused_crate_dependencies)] + +use heroku_nodejs_utils::available_parallelism::available_parallelism_env; +use libcnb::data::exec_d::ExecDProgramOutputKey; +use libcnb::exec_d::write_exec_d_program_output; +use std::collections::HashMap; + +fn main() { + let mut output: HashMap = HashMap::with_capacity(1); + let (available_parallelism_env_key, available_parallelism_env_value) = + available_parallelism_env(); + if let Ok(exec_d_output_key) = available_parallelism_env_key.parse::() { + output.insert(exec_d_output_key, available_parallelism_env_value); + } + write_exec_d_program_output(output); +} diff --git a/buildpacks/nodejs-engine/src/configure_available_parallelism.rs b/buildpacks/nodejs-engine/src/configure_available_parallelism.rs new file mode 100644 index 00000000..7cf85208 --- /dev/null +++ b/buildpacks/nodejs-engine/src/configure_available_parallelism.rs @@ -0,0 +1,38 @@ +use crate::{NodeJsEngineBuildpack, NodeJsEngineBuildpackError}; +use heroku_nodejs_utils::available_parallelism::available_parallelism_env; +use libcnb::additional_buildpack_binary_path; +use libcnb::build::BuildContext; +use libcnb::data::layer_name; +use libcnb::layer::UncachedLayerDefinition; +use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; + +pub(crate) fn configure_available_parallelism( + context: &BuildContext, +) -> Result<(), libcnb::Error> { + let available_parallelism_layer = context.uncached_layer( + layer_name!("available_parallelism"), + UncachedLayerDefinition { + build: true, + launch: true, + }, + )?; + + let (available_parallelism_env_key, available_parallelism_env_value) = + available_parallelism_env(); + + // set for the build time env (for webpack plugins or other tools that spin up processes) + available_parallelism_layer.write_env(LayerEnv::new().chainable_insert( + Scope::Build, + ModificationBehavior::Override, + available_parallelism_env_key, + available_parallelism_env_value, + ))?; + + // set for the run time env + available_parallelism_layer.write_exec_d_programs([( + "available_parallelism", + additional_buildpack_binary_path!("available_parallelism"), + )])?; + + Ok(()) +} diff --git a/buildpacks/nodejs-engine/src/main.rs b/buildpacks/nodejs-engine/src/main.rs index da5078f3..e09d55d1 100644 --- a/buildpacks/nodejs-engine/src/main.rs +++ b/buildpacks/nodejs-engine/src/main.rs @@ -1,10 +1,13 @@ use std::env::consts; use crate::attach_runtime_metrics::{attach_runtime_metrics, NodeRuntimeMetricsError}; +use crate::configure_available_parallelism::configure_available_parallelism; use crate::configure_web_env::configure_web_env; use crate::install_node::{install_node, DistLayerError}; use heroku_nodejs_utils::package_json::{PackageJson, PackageJsonError}; use heroku_nodejs_utils::vrs::{Requirement, Version}; +#[cfg(test)] +use indoc as _; use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder}; use libcnb::data::build_plan::BuildPlanBuilder; use libcnb::data::launch::{LaunchBuilder, ProcessBuilder}; @@ -19,6 +22,8 @@ use libherokubuildpack::inventory::artifact::{Arch, Os}; use libherokubuildpack::inventory::Inventory; use libherokubuildpack::log::{log_error, log_header, log_info}; #[cfg(test)] +use regex as _; +#[cfg(test)] use serde_json as _; use sha2::Sha256; #[cfg(test)] @@ -28,6 +33,7 @@ use thiserror::Error; use ureq as _; mod attach_runtime_metrics; +mod configure_available_parallelism; mod configure_web_env; mod install_node; @@ -108,6 +114,7 @@ impl Buildpack for NodeJsEngineBuildpack { install_node(&context, target_artifact)?; configure_web_env(&context)?; + configure_available_parallelism(&context)?; if Requirement::parse(MINIMUM_NODE_VERSION_FOR_METRICS) .expect("should be a valid version range") diff --git a/buildpacks/nodejs-engine/tests/integration_test.rs b/buildpacks/nodejs-engine/tests/integration_test.rs index 01599466..17fef3d3 100644 --- a/buildpacks/nodejs-engine/tests/integration_test.rs +++ b/buildpacks/nodejs-engine/tests/integration_test.rs @@ -1,11 +1,16 @@ // Required due to: https://github.com/rust-lang/rust/issues/95513 #![allow(unused_crate_dependencies)] -use libcnb_test::{assert_contains, assert_not_contains, ContainerConfig}; +use indoc::indoc; +use libcnb::data::buildpack_id; +use libcnb_test::{ + assert_contains, assert_contains_match, assert_not_contains, BuildpackReference, + ContainerConfig, +}; use std::time::Duration; use test_support::{ - assert_web_response, nodejs_integration_test, nodejs_integration_test_with_config, - set_node_engine, wait_for, PORT, + assert_web_response, custom_buildpack, integration_test_with_config, nodejs_integration_test, + nodejs_integration_test_with_config, set_node_engine, wait_for, PORT, }; const APPLICATION_STARTUP_TIMEOUT: Duration = Duration::from_secs(10); @@ -67,6 +72,34 @@ fn reinstalls_node_if_version_changes() { ); } +#[test] +#[ignore] +fn heroku_available_parallelism_is_set_at_build_and_runtime() { + integration_test_with_config( + "./fixtures/node-with-indexjs", + |_| {}, + |ctx| { + assert_contains_match!(ctx.pack_stdout, "HEROKU_AVAILABLE_PARALLELISM=\\d+"); + assert_contains_match!( + ctx.run_shell_command("env").stdout, + "HEROKU_AVAILABLE_PARALLELISM=\\d+" + ); + }, + &[ + BuildpackReference::WorkspaceBuildpack(buildpack_id!("heroku/nodejs")), + BuildpackReference::Other( + custom_buildpack() + .id("test/echo-build-parallelism") + .build(indoc! { " + #!/usr/bin/env bash + env + " }) + .call(), + ), + ], + ); +} + #[test] #[ignore] fn runtime_metrics_script_is_activated_when_heroku_metrics_url_is_set() { diff --git a/common/nodejs-utils/src/available_parallelism.rs b/common/nodejs-utils/src/available_parallelism.rs new file mode 100644 index 00000000..08e4fe33 --- /dev/null +++ b/common/nodejs-utils/src/available_parallelism.rs @@ -0,0 +1,16 @@ +pub const HEROKU_AVAILABLE_PARALLELISM: &str = "HEROKU_AVAILABLE_PARALLELISM"; + +#[must_use] +pub fn available_parallelism_env() -> (String, String) { + ( + HEROKU_AVAILABLE_PARALLELISM.to_string(), + std::thread::available_parallelism() + // XXX: The Rust implementation always rounds down the value reported here if the + // (quota / period) calculated from cgroups cpu.max produces a fractional value. + // For Heroku Fir Dynos this will always end up reducing the cpu allocation + // value by 1 since a small amount of quota is reserved for the system so we need + // to add that back unless Rust changes how they deal with rounding. + .map(|value| (value.get() + 1).to_string()) + .unwrap_or_default(), + ) +} diff --git a/common/nodejs-utils/src/lib.rs b/common/nodejs-utils/src/lib.rs index 3e3233da..0c399486 100644 --- a/common/nodejs-utils/src/lib.rs +++ b/common/nodejs-utils/src/lib.rs @@ -2,6 +2,7 @@ use keep_a_changelog_file as _; use sha2 as _; pub mod application; +pub mod available_parallelism; pub mod buildplan; pub mod distribution; pub mod inv; diff --git a/test_support/src/lib.rs b/test_support/src/lib.rs index b1dddd08..fc9e7927 100644 --- a/test_support/src/lib.rs +++ b/test_support/src/lib.rs @@ -188,7 +188,11 @@ pub fn add_package_json_dependency(app_dir: &Path, package_name: &str, package_v pub fn add_build_script(app_dir: &Path, script: &str) { update_package_json(app_dir, |json| { - let scripts = json["scripts"].as_object_mut().unwrap(); + let scripts = json + .entry("scripts") + .or_insert(serde_json::Value::Object(serde_json::Map::new())) + .as_object_mut() + .unwrap(); scripts.insert( script.to_string(), serde_json::Value::String(format!("echo 'executed {script}'")),