diff --git a/Cargo.lock b/Cargo.lock index 7ded79c6..7cb79f66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,24 +176,12 @@ dependencies = [ "derive_arbitrary", ] -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - [[package]] name = "arrayvec" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - [[package]] name = "assert2" version = "0.3.15" @@ -523,19 +511,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "blake3" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" -dependencies = [ - "arrayref", - "arrayvec 0.7.6", - "cc", - "cfg-if", - "constant_time_eq", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -947,12 +922,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - [[package]] name = "cookie" version = "0.18.1" @@ -1950,7 +1919,6 @@ version = "0.0.0" dependencies = [ "anyhow", "assert2", - "blake3", "cargo-component", "cargo-component-core", "cargo_toml", @@ -1976,7 +1944,6 @@ dependencies = [ "serde 1.0.215", "serde_json", "serde_yaml", - "shlex", "syn 2.0.90", "tempfile", "test-r", @@ -2666,7 +2633,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ - "arrayvec 0.5.2", + "arrayvec", "bitflags 1.3.2", "cfg-if", "ryu", diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index bcc15249..0a732e28 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -66,8 +66,6 @@ walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = "=0.221.2" wit-parser = "=0.221.2" -shlex = "1.3.0" -blake3 = "1.5.5" [dev-dependencies] diff --git a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json index 58f76945..a8186770 100644 --- a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json +++ b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json @@ -228,13 +228,6 @@ "command": { "type": "string", "description": "External command to execute" - }, - "mkdirs": { - "type": "array", - "description": "List of directories that should be created before running the command", - "items": { - "type": "string" - } } }, "additionalProperties": false, @@ -248,21 +241,14 @@ "type": "string", "description": "External command to execute" }, - "mkdirs": { - "type": "array", - "description": "List of directories that should be created before running the command", - "items": { - "type": "string" - } - }, - "sources": { + "inputs": { "type": "array", "description": "Inputs (paths and globs) for the external command", "items": { "type": "string" } }, - "targets": { + "outputs": { "type": "array", "description": "Output (paths and globs) for the external command", "items": { diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index f0b7a965..68bc6afd 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -10,6 +10,7 @@ use crate::model::app::{ ComponentPropertiesExtensions, ProfileName, DEFAULT_CONFIG_FILE_NAME, }; use crate::model::app_raw; +use crate::model::app_raw::ExternalCommand; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; use crate::wit_generate::{ @@ -23,7 +24,6 @@ use colored::Colorize; use glob::{glob_with, MatchOptions}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; -use serde::Serialize; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -336,7 +336,10 @@ async fn gen_rpc( let mut any_changed = false; for component_name in ctx.application.component_names() { let changed = create_generated_wit(ctx, component_name)?; - update_cargo_toml(ctx, changed, component_name)?; + if changed { + // TODO: if this fails, it won't be retried, add done file for this + update_cargo_toml(ctx, component_name)?; + } any_changed |= changed; } if any_changed { @@ -714,7 +717,7 @@ fn load_app_validated( app } -fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { +fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { log_action("Collecting", "sources"); let _indent = LogIndent::new(); @@ -729,12 +732,12 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult ValidatedResult( .application .component_source_wit(component_name, ctx.profile()); let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); - let task_result_marker = TaskResultMarker::new( - &ctx.application.task_result_marker_dir(), - ComponentGeneratorMarkerHash { - component_name, - generator_kind: "base_wit", - }, - )?; + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_base_wit); if is_up_to_date( ctx.config.skip_up_to_date_checks - || !task_result_marker.is_up_to_date() + || !gen_dir_done_marker.is_done() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_source_wit.clone()], || [component_generated_base_wit.clone()], @@ -1059,17 +1053,12 @@ fn create_generated_base_wit( ); let _indent = LogIndent::new(); - match extract_main_interface_as_wit_dep(&component_generated_base_wit) { - Ok(()) => { - task_result_marker.success()?; - Ok(true) - } - Err(err) => { - task_result_marker.failure()?; - Err(err) - } - } + extract_main_interface_as_wit_dep(&component_generated_base_wit)?; } + + gen_dir_done_marker.mark_as_done()?; + + Ok(true) } } @@ -1081,17 +1070,11 @@ fn create_generated_wit( let component_generated_wit = ctx .application .component_generated_wit(component_name, ctx.profile()); - let task_result_marker = TaskResultMarker::new( - &ctx.application.task_result_marker_dir(), - ComponentGeneratorMarkerHash { - component_name, - generator_kind: "wit", - }, - )?; + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_wit); if is_up_to_date( ctx.config.skip_up_to_date_checks - || !task_result_marker.is_up_to_date() + || !gen_dir_done_marker.is_done() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_generated_base_wit.clone()], || [component_generated_wit.clone()], @@ -1115,7 +1098,7 @@ fn create_generated_wit( copy_wit_sources(&component_generated_base_wit, &component_generated_wit)?; add_stub_deps(ctx, component_name)?; - task_result_marker.success()?; + gen_dir_done_marker.mark_as_done()?; Ok(true) } @@ -1123,7 +1106,6 @@ fn create_generated_wit( fn update_cargo_toml( ctx: &ApplicationContext, - mut skip_up_to_date_checks: bool, component_name: &ComponentName, ) -> anyhow::Result<()> { let component_source_wit = PathExtra::new( @@ -1138,43 +1120,16 @@ fn update_cargo_toml( })?; let cargo_toml = component_source_wit_parent.join("Cargo.toml"); - if !cargo_toml.exists() { - return Ok(()); - } - - let task_result_marker = TaskResultMarker::new( - &ctx.application.task_result_marker_dir(), - ComponentGeneratorMarkerHash { - component_name, - generator_kind: "Cargo.toml", - }, - )?; - - skip_up_to_date_checks |= skip_up_to_date_checks || ctx.config.skip_up_to_date_checks; - if !skip_up_to_date_checks && task_result_marker.is_up_to_date() { - log_skipping_up_to_date(format!( - "updating Cargo.toml for {}", - component_name.as_str().log_color_highlight() - )); - return Ok(()); + if cargo_toml.exists() { + regenerate_cargo_package_component( + &cargo_toml, + &ctx.application + .component_generated_wit(component_name, ctx.profile()), + None, + )? } - let result = regenerate_cargo_package_component( - &cargo_toml, - &ctx.application - .component_generated_wit(component_name, ctx.profile()), - None, - ); - match result { - Ok(()) => { - task_result_marker.success()?; - Ok(()) - } - Err(err) => { - task_result_marker.failure()?; - Err(err) - } - } + Ok(()) } async fn build_stub( @@ -1210,16 +1165,10 @@ async fn build_stub( let stub_wasm = ctx.application.stub_wasm(component_name); let stub_wit = ctx.application.stub_wit(component_name); - let task_result_marker = TaskResultMarker::new( - &ctx.application.task_result_marker_dir(), - ComponentGeneratorMarkerHash { - component_name, - generator_kind: "stub", - }, - )?; + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&stub_wit); if is_up_to_date( - ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(), + ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done(), || stub_sources, || [stub_wit.clone(), stub_wasm.clone()], ) { @@ -1248,19 +1197,13 @@ async fn build_stub( ); fs::create_dir_all(&target_root)?; - let result = - commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await; - match result { - Ok(()) => { - task_result_marker.success()?; - delete_path("stub temp build dir", &target_root)?; - Ok(true) - } - Err(err) => { - task_result_marker.failure()?; - Err(err) - } - } + commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await?; + + gen_dir_done_marker.mark_as_done()?; + + delete_path("stub temp build dir", &target_root)?; + + Ok(true) } } @@ -1352,7 +1295,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { fn execute_external_command( ctx: &ApplicationContext, component_name: &ComponentName, - command: &app_raw::ExternalCommand, + command: &ExternalCommand, ) -> anyhow::Result<()> { let build_dir = command .dir @@ -1368,22 +1311,11 @@ fn execute_external_command( .to_path_buf() }); - let task_result_marker = TaskResultMarker::new( - &ctx.application.task_result_marker_dir(), - ResolvedExternalCommandMarkerHash { - build_dir: &build_dir, - command, - }, - )?; - - let skip_up_to_date_checks = - ctx.config.skip_up_to_date_checks || !task_result_marker.is_up_to_date(); - if !command.sources.is_empty() && !command.targets.is_empty() { let sources = compile_and_collect_globs(&build_dir, &command.sources)?; let targets = compile_and_collect_globs(&build_dir, &command.targets)?; - if is_up_to_date(skip_up_to_date_checks, || sources, || targets) { + if is_up_to_date(ctx.config.skip_up_to_date_checks, || sources, || targets) { log_skipping_up_to_date(format!( "executing external command '{}' in directory {}", command.command.log_color_highlight(), @@ -1402,149 +1334,46 @@ fn execute_external_command( ), ); - if !command.mkdirs.is_empty() { - let _ident = LogIndent::new(); - for dir in &command.mkdirs { - let dir = ctx - .application - .component_source_dir(component_name) - .join(dir); - if !std::fs::exists(&dir)? { - log_action( - "Creating", - format!("directory {}", dir.log_color_highlight()), - ); - std::fs::create_dir_all(dir)? - } - } - } - - let command_tokens = shlex::split(&command.command) - .ok_or_else(|| anyhow::anyhow!("Failed to parse external command: {}", command.command))?; + let command_tokens = command.command.split(' ').collect::>(); if command_tokens.is_empty() { return Err(anyhow!("Empty command!")); } - let result = Command::new(command_tokens[0].clone()) + let result = Command::new(command_tokens[0]) .args(command_tokens.iter().skip(1)) .current_dir(build_dir) .status() .with_context(|| "Failed to execute command".to_string())?; - if result.success() { - task_result_marker.success()?; - Ok(()) - } else { - task_result_marker.failure()?; - Err(anyhow!(format!( + if !result.success() { + return Err(anyhow!(format!( "Command failed with exit code: {}", result .code() .map(|code| code.to_string().log_color_error_highlight().to_string()) .unwrap_or_else(|| "?".to_string()) - ))) + ))); } -} - -trait TaskResultMarkerHashInput { - fn task_kind() -> &'static str; - - fn hash_input(&self) -> anyhow::Result>; -} -#[derive(Serialize)] -struct ResolvedExternalCommandMarkerHash<'a> { - build_dir: &'a Path, - command: &'a app_raw::ExternalCommand, + Ok(()) } -impl TaskResultMarkerHashInput for ResolvedExternalCommandMarkerHash<'_> { - fn task_kind() -> &'static str { - "ResolvedExternalCommandMarkerHash" - } - - fn hash_input(&self) -> anyhow::Result> { - Ok(serde_yaml::to_string(self)?.into_bytes()) - } -} +static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = ".done"; -struct ComponentGeneratorMarkerHash<'a> { - component_name: &'a ComponentName, - generator_kind: &'a str, +struct GeneratedDirDoneMarker<'a> { + dir: &'a Path, } -impl TaskResultMarkerHashInput for ComponentGeneratorMarkerHash<'_> { - fn task_kind() -> &'static str { - "ComponentGeneratorMarkerHash" - } - - fn hash_input(&self) -> anyhow::Result> { - Ok(format!("{}-{}", self.component_name, self.generator_kind).into_bytes()) - } -} - -struct TaskResultMarker { - success_marker_file_path: PathBuf, - failure_marker_file_path: PathBuf, - success_before: bool, - failure_before: bool, -} - -static TASK_RESULT_MARKER_SUCCESS_SUFFIX: &str = "-success"; -static TASK_RESULT_MARKER_FAILURE_SUFFIX: &str = "-failure"; - -impl TaskResultMarker { - fn new(dir: &Path, task: T) -> anyhow::Result { - let mut hasher = blake3::Hasher::new(); - hasher.update(T::task_kind().as_bytes()); - hasher.update(&task.hash_input()?); - let hex_hash = hasher.finalize().to_hex().to_string(); - - let success_marker_file_path = dir.join(format!( - "{}{}", - &hex_hash, TASK_RESULT_MARKER_SUCCESS_SUFFIX - )); - let failure_marker_file_path = dir.join(format!( - "{}{}", - &hex_hash, TASK_RESULT_MARKER_FAILURE_SUFFIX - )); - - let success_marker_exists = success_marker_file_path.exists(); - let failure_marker_exists = failure_marker_file_path.exists(); - - let (success_before, failure_before) = match (success_marker_exists, failure_marker_exists) - { - (true, false) => (true, false), - (false, false) => (false, false), - (_, true) => (false, true), - }; - - if failure_marker_exists || !success_marker_exists { - if success_marker_exists { - fs::remove(&success_marker_file_path)? - } - if failure_marker_exists { - fs::remove(&failure_marker_file_path)? - } - } - - Ok(Self { - success_marker_file_path, - failure_marker_file_path, - success_before, - failure_before, - }) - } - - fn is_up_to_date(&self) -> bool { - !self.failure_before && self.success_before +impl<'a> GeneratedDirDoneMarker<'a> { + fn new(dir: &'a Path) -> Self { + Self { dir } } - fn success(&self) -> anyhow::Result<()> { - fs::write_str(&self.success_marker_file_path, "") + fn is_done(&self) -> bool { + self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME).exists() } - fn failure(&self) -> anyhow::Result<()> { - fs::write_str(&self.failure_marker_file_path, "") + fn mark_as_done(&self) -> anyhow::Result<()> { + fs::write_str(self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME), "") } } diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 4d8ac7e7..047c7466 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -826,10 +826,6 @@ impl Application { } } - pub fn task_result_marker_dir(&self) -> PathBuf { - self.temp_dir().join("task_results") - } - fn component(&self, component_name: &ComponentName) -> &Component { self.components .get(component_name) @@ -1039,17 +1035,12 @@ pub struct Component { impl Component { pub fn source_dir(&self) -> &Path { - let parent = self.source.parent().unwrap_or_else(|| { + self.source.parent().unwrap_or_else(|| { panic!( "Failed to get parent for component, source: {}", self.source.display() ) - }); - if parent.as_os_str().is_empty() { - Path::new(".") - } else { - parent - } + }) } } diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs index 8d1f47b2..f2a740a0 100644 --- a/wasm-rpc-stubgen/src/model/app_raw.rs +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -142,8 +142,6 @@ pub struct ExternalCommand { #[serde(default, skip_serializing_if = "Option::is_none")] pub dir: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub mkdirs: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub sources: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub targets: Vec, diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index 4841c967..3dc7741b 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -77,7 +77,6 @@ impl Template for app_raw::ExternalCommand { ) -> Result { Ok(app_raw::ExternalCommand { command: self.command.render(env, ctx)?, - mkdirs: self.mkdirs.render(env, ctx)?, dir: self.dir.render(env, ctx)?, sources: self.sources.render(env, ctx)?, targets: self.targets.render(env, ctx)?, diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index acf59877..56a1fdfb 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -186,9 +186,6 @@ impl ResolvedWitApplication { app: &Application, profile: Option<&ProfileName>, ) -> ValidatedResult { - // TODO: Can be removed once we fixed all docs and examples - std::env::set_var("WIT_REQUIRE_F32_F64", "0"); - log_action("Resolving", "application wit directories"); let _indent = LogIndent::new(); @@ -692,9 +689,6 @@ pub struct WitDepsResolver { impl WitDepsResolver { pub fn new(sources: Vec) -> anyhow::Result { - // TODO: Can be removed once we fixed all docs and examples - std::env::set_var("WIT_REQUIRE_F32_F64", "0"); - let mut packages = HashMap::>::new(); for source in &sources {