From 133e859bfc64658a2ffa95c2148090db98a2130c Mon Sep 17 00:00:00 2001 From: Staszek Krotki <16387248+staszek-krotki@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:51:27 +0100 Subject: [PATCH 01/17] init --- .gitignore | 2 ++ Cargo.lock | 34 ++++++++++++++++++++++++++++++ Cargo.toml | 4 ++++ src/main.rs | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) diff --git a/.gitignore b/.gitignore index 00ae12c..fd30ba5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.idea/ + # Generated by Cargo # will have compiled files and executables debug/ diff --git a/Cargo.lock b/Cargo.lock index 5cd7c60..ce6ef60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -389,6 +389,28 @@ dependencies = [ "xz2", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.42", +] + [[package]] name = "async-trait" version = "0.1.75" @@ -1239,6 +1261,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gsb-http-proxy" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "thiserror", + "ya-service-bus", +] + [[package]] name = "h2" version = "0.3.22" @@ -3474,12 +3506,14 @@ dependencies = [ "actix", "actix-rt", "anyhow", + "async-stream", "chrono", "clap 4.4.11", "dotenv", "env_logger", "flexi_logger", "futures 0.3.29", + "gsb-http-proxy", "log", "regex", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index a2ccd0a..d588616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,10 @@ futures = "0.3" flexi_logger = { version = "0.27", features = ["colors"] } regex = "1" reqwest = "0.11" +async-stream = "0.3.5" + +[dependencies.gsb-http-proxy] +path = "../yagna/core/http-proxy" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3", features = ["jobapi2", "processthreadsapi", "handleapi"] } diff --git a/src/main.rs b/src/main.rs index f7d4c37..01bbd10 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,8 @@ use actix::prelude::*; use chrono::Utc; use clap::Parser; use futures::prelude::*; +use async_stream::stream; +use gsb_http_proxy::{GsbHttpCall, GsbHttpCallEvent}; use ya_client_model::activity::activity_state::*; use ya_client_model::activity::ExeScriptCommand; @@ -174,9 +176,15 @@ async fn run(cli: Cli) -> any log::info!("CLI args: {:?}", &cli); log::info!("Binding to GSB ..."); + log::info!("Binding to GSB ... 1"); let agreement_path = args.agreement.clone(); + + log::info!("Binding to GSB ... 1..1"); + let agreement = AgreementDesc::load(agreement_path)?; + log::info!("Binding to GSB ... 2"); + let ctx = ExeUnitContext { activity_id: activity_id.clone(), report_url: report_url.clone(), @@ -191,20 +199,29 @@ async fn run(cli: Cli) -> any batches: Rc::new(RefCell::new(Default::default())), }; + log::info!("Binding to GSB ... 3"); + let activity_pinger = activity_loop( report_url, activity_id, ctx.process_controller.clone(), ctx.agreement.clone(), ); + + log::info!("Binding to GSB ... 4"); #[cfg(target_os = "windows")] let _job = process::win::JobObject::new()?; + log::info!("Binding to GSB ... 5"); { let batch = ctx.batches.clone(); let batch_results = batch.clone(); let ctx = ctx.clone(); + log::info!("exe_unit_url: {}", exe_unit_url); + + log::info!("[GSB Bind]: Exec"); + gsb::bind(&exe_unit_url, move |exec: activity::Exec| { let ctx = ctx.clone(); let exec = exec.clone(); @@ -363,6 +380,8 @@ async fn run(cli: Cli) -> any future::ok(batch_id) }); + log::info!("[GSB Bind]: GetExecBatchResults"); + gsb::bind(&exe_unit_url, move |exec: activity::GetExecBatchResults| { if let Some(result) = batch_results.borrow().get(&exec.batch_id) { future::ok(result.clone()) @@ -373,6 +392,47 @@ async fn run(cli: Cli) -> any ))) } }); + + log::info!("[GSB Bind]: GsbHttpCall"); + + gsb::bind_stream(&exe_unit_url, move |http_call: GsbHttpCall| { + + log::info!(">>>>>>>>>>>>>>>> GsbHttpCall: {}", http_call.host); + + let _interval = tokio::time::interval(Duration::from_secs(1)); + let stream = Box::pin(stream! { + for i in 0..10 { + let msg = format!("called {} element #{}", http_call.host, i); + + let response = GsbHttpCallEvent { + index: i, + timestamp: Utc::now().naive_local().to_string(), + val: msg, + }; + println!("sending nr {}", i); + yield Ok(response); + } + }); + + // let stream = tokio_stream::wrappers::IntervalStream::new(interval) + // .map(move |_ts| { + // println!("Creating response"); + // let msg = format!("response from {}", http_call.host); + // count += 1; + // let response = GsbHttpCallEvent { + // index: count, + // timestamp: Utc::now().naive_local(), + // val: msg, + // }; + // if count == 7 { + // return Err(HttpProxyStatusError::RuntimeException("end".to_string())); + // } + // Ok(response) + // }) + // .take(5); + println!("returning stream"); + stream + }); }; send_state( &ctx, From 338a911d607aa1068829f57f4ed44504c6b40836 Mon Sep 17 00:00:00 2001 From: Staszek Krotki <16387248+staszek-krotki@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:57:18 +0100 Subject: [PATCH 02/17] http gsb proxy --- Cargo.lock | 107 +++++++++++++++++++++++++++++++++------------------- Cargo.toml | 13 ++++--- src/main.rs | 61 ++---------------------------- 3 files changed, 79 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce6ef60..ddedb46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,7 +65,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http", + "http 0.2.11", "httparse", "httpdate", "itoa", @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -100,7 +100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" dependencies = [ "bytestring", - "http", + "http 0.2.11", "regex", "serde", "tracing", @@ -155,7 +155,7 @@ dependencies = [ "actix-service", "actix-utils", "futures-core", - "http", + "http 0.2.11", "impl-more", "openssl", "pin-project-lite", @@ -228,7 +228,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -239,7 +239,7 @@ checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -408,7 +408,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -419,7 +419,7 @@ checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -459,7 +459,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.11", "itoa", "log", "mime", @@ -733,7 +733,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1120,7 +1120,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1239,7 +1239,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1265,9 +1265,15 @@ dependencies = [ name = "gsb-http-proxy" version = "0.1.0" dependencies = [ + "async-stream", "chrono", + "http 1.0.0", + "log", + "reqwest", "serde", + "serde_json", "thiserror", + "tokio", "ya-service-bus", ] @@ -1282,7 +1288,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", "indexmap 2.1.0", "slab", "tokio", @@ -1358,6 +1364,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -1365,7 +1382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.11", "pin-project-lite", ] @@ -1398,7 +1415,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.11", "http-body", "httparse", "httpdate", @@ -1756,6 +1773,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + [[package]] name = "num_enum" version = "0.5.11" @@ -1821,7 +1848,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1994,9 +2021,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -2109,9 +2136,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2262,7 +2289,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.11", "http-body", "hyper", "hyper-tls", @@ -2447,9 +2474,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -2465,20 +2492,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2659,9 +2686,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2747,7 +2774,7 @@ checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2815,6 +2842,7 @@ dependencies = [ "bytes 1.5.0", "libc", "mio", + "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -2841,7 +2869,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3124,7 +3152,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -3158,7 +3186,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3454,8 +3482,6 @@ dependencies = [ [[package]] name = "ya-core-model" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dddc3bccc5be20eb58baa2f1bc61d62221b28e4d307c5c4bdf363e3c3149ddaa" dependencies = [ "chrono", "derive_more", @@ -3473,7 +3499,8 @@ dependencies = [ [[package]] name = "ya-core-model" version = "0.9.0" -source = "git+https://github.com/golemfactory/yagna.git?rev=6fd7b003e3d0947284d47d99b2944d293d329c53#6fd7b003e3d0947284d47d99b2944d293d329c53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dddc3bccc5be20eb58baa2f1bc61d62221b28e4d307c5c4bdf363e3c3149ddaa" dependencies = [ "chrono", "derive_more", @@ -3507,6 +3534,7 @@ dependencies = [ "actix-rt", "anyhow", "async-stream", + "bytes 1.5.0", "chrono", "clap 4.4.11", "dotenv", @@ -3514,6 +3542,7 @@ dependencies = [ "flexi_logger", "futures 0.3.29", "gsb-http-proxy", + "http 0.2.11", "log", "regex", "reqwest", @@ -3523,7 +3552,7 @@ dependencies = [ "winapi", "ya-agreement-utils", "ya-client-model", - "ya-core-model 0.9.0 (git+https://github.com/golemfactory/yagna.git?rev=6fd7b003e3d0947284d47d99b2944d293d329c53)", + "ya-core-model 0.9.0", "ya-runtime-api", "ya-service-bus", "ya-transfer", @@ -3694,7 +3723,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d588616..1775f15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,9 +11,10 @@ members = [ ] [dependencies] -ya-core-model = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53", features = ["activity", "appkey"] } +#ya-core-model = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53", features = ["activity", "appkey"] } +ya-core-model = { path = "../yagna/core/model", features = ["activity", "appkey"] } ya-runtime-api = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53" } -ya-service-bus = "0.6.1" +ya-service-bus = "0.6.3" ya-client-model = "0.5.0" ya-agreement-utils = "0.5" ya-transfer = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53" } @@ -33,11 +34,11 @@ tokio = { version = "1.32", features = ["macros"] } futures = "0.3" flexi_logger = { version = "0.27", features = ["colors"] } regex = "1" -reqwest = "0.11" +reqwest = { version = "0.11", features = ["blocking", "json"] } async-stream = "0.3.5" - -[dependencies.gsb-http-proxy] -path = "../yagna/core/http-proxy" +gsb-http-proxy = { path = "../yagna/core/http-proxy" } +http = "0.2.11" +bytes = "1.5.0" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3", features = ["jobapi2", "processthreadsapi", "handleapi"] } diff --git a/src/main.rs b/src/main.rs index 01bbd10..ba59025 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,8 +12,7 @@ use actix::prelude::*; use chrono::Utc; use clap::Parser; use futures::prelude::*; -use async_stream::stream; -use gsb_http_proxy::{GsbHttpCall, GsbHttpCallEvent}; +use gsb_http_proxy::GsbHttpCall; use ya_client_model::activity::activity_state::*; use ya_client_model::activity::ExeScriptCommand; @@ -172,19 +171,10 @@ async fn run(cli: Cli) -> any } }; - log::info!("{:?}", args); - log::info!("CLI args: {:?}", &cli); - log::info!("Binding to GSB ..."); - - log::info!("Binding to GSB ... 1"); let agreement_path = args.agreement.clone(); - log::info!("Binding to GSB ... 1..1"); - let agreement = AgreementDesc::load(agreement_path)?; - log::info!("Binding to GSB ... 2"); - let ctx = ExeUnitContext { activity_id: activity_id.clone(), report_url: report_url.clone(), @@ -199,8 +189,6 @@ async fn run(cli: Cli) -> any batches: Rc::new(RefCell::new(Default::default())), }; - log::info!("Binding to GSB ... 3"); - let activity_pinger = activity_loop( report_url, activity_id, @@ -208,11 +196,8 @@ async fn run(cli: Cli) -> any ctx.agreement.clone(), ); - log::info!("Binding to GSB ... 4"); #[cfg(target_os = "windows")] let _job = process::win::JobObject::new()?; - - log::info!("Binding to GSB ... 5"); { let batch = ctx.batches.clone(); let batch_results = batch.clone(); @@ -380,8 +365,6 @@ async fn run(cli: Cli) -> any future::ok(batch_id) }); - log::info!("[GSB Bind]: GetExecBatchResults"); - gsb::bind(&exe_unit_url, move |exec: activity::GetExecBatchResults| { if let Some(result) = batch_results.borrow().get(&exec.batch_id) { future::ok(result.clone()) @@ -393,45 +376,9 @@ async fn run(cli: Cli) -> any } }); - log::info!("[GSB Bind]: GsbHttpCall"); - - gsb::bind_stream(&exe_unit_url, move |http_call: GsbHttpCall| { - - log::info!(">>>>>>>>>>>>>>>> GsbHttpCall: {}", http_call.host); - - let _interval = tokio::time::interval(Duration::from_secs(1)); - let stream = Box::pin(stream! { - for i in 0..10 { - let msg = format!("called {} element #{}", http_call.host, i); - - let response = GsbHttpCallEvent { - index: i, - timestamp: Utc::now().naive_local().to_string(), - val: msg, - }; - println!("sending nr {}", i); - yield Ok(response); - } - }); - - // let stream = tokio_stream::wrappers::IntervalStream::new(interval) - // .map(move |_ts| { - // println!("Creating response"); - // let msg = format!("response from {}", http_call.host); - // count += 1; - // let response = GsbHttpCallEvent { - // index: count, - // timestamp: Utc::now().naive_local(), - // val: msg, - // }; - // if count == 7 { - // return Err(HttpProxyStatusError::RuntimeException("end".to_string())); - // } - // Ok(response) - // }) - // .take(5); - println!("returning stream"); - stream + gsb::bind_stream(&exe_unit_url, move |mut http_call: GsbHttpCall| { + let stream = http_call.execute("http://localhost:7861/".to_string()); + Box::pin(stream.map(|e| Ok(e))) }); }; send_state( From 7d2e422a96677c850d67f21f4ec0c3d8599748b4 Mon Sep 17 00:00:00 2001 From: Staszek Krotki <16387248+staszek-krotki@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:44:49 +0100 Subject: [PATCH 03/17] git ref to http gsb proxy --- Cargo.lock | 292 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 5 +- 2 files changed, 156 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddedb46..696fc5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,9 +46,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" +checksum = "129d4c88e98860e1758c5de288d1632b07970a16d59bdf7b8d66053d582bb71f" dependencies = [ "actix-codec", "actix-rt", @@ -95,9 +95,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http 0.2.11", @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" +checksum = "e43428f3bf11dee6d166b00ec2df4e3aa8cc1606aaa0b7433c146852e2f4e03b" dependencies = [ "actix-codec", "actix-http", @@ -259,9 +259,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "getrandom", @@ -320,9 +320,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" dependencies = [ "anstyle", "anstyle-parse", @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "async-compression" @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", @@ -441,9 +441,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "awc" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa3c705a9c7917ac0f41c0757a0a747b43bbc29b0b364b081bd7c5fc67fb223" +checksum = "b625cad34428b3b82d0bd548b26a1cd0a3d70b6109e9b4e3355d8f1802a8b1c6" dependencies = [ "actix-codec", "actix-http", @@ -491,9 +491,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bigdecimal" @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "serde", @@ -704,9 +704,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "58e54881c004cec7895b0068a0a954cd5d62da01aef83fa35b1e594497bf5445" dependencies = [ "clap_builder", "clap_derive", @@ -714,9 +714,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "59cb82d7f531603d2fd1f507441cdd35184fa81beff7bd489570de7f773460bb" dependencies = [ "anstream", "anstyle", @@ -792,9 +792,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -810,22 +810,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -839,9 +835,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -1066,9 +1062,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1081,9 +1077,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1091,15 +1087,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1108,15 +1104,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1125,21 +1121,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -1184,9 +1180,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1201,7 +1197,7 @@ dependencies = [ "actix-rt", "anyhow", "digest 0.8.1", - "futures 0.3.29", + "futures 0.3.30", "log", "rand 0.8.5", "serde", @@ -1264,9 +1260,11 @@ dependencies = [ [[package]] name = "gsb-http-proxy" version = "0.1.0" +source = "git+https://github.com/golemfactory/yagna.git?rev=09d4f43ef6ccc2865b3ffd00a6c3a62a33294940#09d4f43ef6ccc2865b3ffd00a6c3a62a33294940" dependencies = [ "async-stream", "chrono", + "futures 0.3.30", "http 1.0.0", "log", "reqwest", @@ -1279,9 +1277,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" dependencies = [ "bytes 1.5.0", "fnv", @@ -1443,9 +1441,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1508,13 +1506,13 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi 0.3.3", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1552,18 +1550,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] @@ -1582,9 +1580,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libredox" @@ -1655,9 +1653,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "metrics" @@ -1806,9 +1804,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -1827,9 +1825,9 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if 1.0.0", @@ -1859,9 +1857,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1906,9 +1904,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" dependencies = [ "memchr", "thiserror", @@ -2341,14 +2339,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.20", + "semver 1.0.21", ] [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ "bitflags 2.4.1", "errno", @@ -2402,11 +2400,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2459,9 +2457,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "semver-parser" @@ -2483,9 +2481,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] @@ -2580,9 +2578,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" [[package]] name = "socket2" @@ -2728,22 +2726,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -2759,18 +2757,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", @@ -3133,9 +3131,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3143,9 +3141,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -3158,9 +3156,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3170,9 +3168,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3180,9 +3178,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -3193,15 +3191,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -3258,11 +3256,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -3399,9 +3397,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.30" +version = "0.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" dependencies = [ "memchr", ] @@ -3418,9 +3416,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.1.3" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys", @@ -3447,7 +3445,7 @@ dependencies = [ "serde_json", "serde_yaml", "thiserror", - "ya-client-model", + "ya-client-model 0.5.0", ] [[package]] @@ -3467,6 +3465,23 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ya-client-model" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" +dependencies = [ + "bigdecimal", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", +] + [[package]] name = "ya-compile-time-utils" version = "0.2.1" @@ -3492,7 +3507,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.6.0", "ya-service-bus", ] @@ -3511,7 +3526,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.5.0", "ya-service-bus", ] @@ -3536,11 +3551,11 @@ dependencies = [ "async-stream", "bytes 1.5.0", "chrono", - "clap 4.4.11", + "clap 4.4.16", "dotenv", "env_logger", "flexi_logger", - "futures 0.3.29", + "futures 0.3.30", "gsb-http-proxy", "http 0.2.11", "log", @@ -3551,7 +3566,7 @@ dependencies = [ "tokio", "winapi", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model 0.9.0", "ya-runtime-api", "ya-service-bus", @@ -3566,7 +3581,7 @@ source = "git+https://github.com/golemfactory/yagna.git?rev=6fd7b003e3d0947284d4 dependencies = [ "anyhow", "bytes 1.5.0", - "futures 0.3.29", + "futures 0.3.30", "log", "prost 0.10.4", "prost-build 0.10.4", @@ -3601,7 +3616,7 @@ checksum = "c525504f7ca7d13825ef10a44005663b91b4b9f3eeebd6768188884bec78a37d" dependencies = [ "actix", "bitflags 1.3.2", - "futures 0.3.29", + "futures 0.3.30", "pin-project", ] @@ -3613,7 +3628,7 @@ checksum = "0f791365387355ca43c844e3709bb72e5538db1009ece21cc2ea538a2bb63722" dependencies = [ "actix", "flexbuffers", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "miniz_oxide 0.5.4", @@ -3644,7 +3659,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.5.0", - "futures 0.3.29", + "futures 0.3.30", "gftp", "globset", "h2", @@ -3663,7 +3678,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.5.0", "ya-core-model 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "ya-runtime-api", "ya-service-bus", @@ -3677,7 +3692,7 @@ name = "ya-utils-futures" version = "0.3.0" source = "git+https://github.com/golemfactory/yagna.git?rev=6fd7b003e3d0947284d47d99b2944d293d329c53#6fd7b003e3d0947284d47d99b2944d293d329c53" dependencies = [ - "futures 0.3.29", + "futures 0.3.30", "tokio", ] @@ -3744,20 +3759,19 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 1775f15..6f34ffe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ ya-core-model = { path = "../yagna/core/model", features = ["activity", "appkey"] } ya-runtime-api = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53" } ya-service-bus = "0.6.3" -ya-client-model = "0.5.0" +ya-client-model = "0.6.0" ya-agreement-utils = "0.5" ya-transfer = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53" } @@ -36,7 +36,8 @@ flexi_logger = { version = "0.27", features = ["colors"] } regex = "1" reqwest = { version = "0.11", features = ["blocking", "json"] } async-stream = "0.3.5" -gsb-http-proxy = { path = "../yagna/core/http-proxy" } +#gsb-http-proxy = { path = "../yagna/exe-unit/components/http-proxy" } +gsb-http-proxy = { git = "https://github.com/golemfactory/yagna.git", rev = "09d4f43ef6ccc2865b3ffd00a6c3a62a33294940" } http = "0.2.11" bytes = "1.5.0" From b58e1491f1cc8349e38941497cde1bd9236165f2 Mon Sep 17 00:00:00 2001 From: Staszek Krotki <16387248+staszek-krotki@users.noreply.github.com> Date: Mon, 15 Jan 2024 21:27:41 +0100 Subject: [PATCH 04/17] update rev --- Cargo.lock | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 696fc5a..e63485d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -372,6 +372,16 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-compression" version = "0.3.7" @@ -757,6 +767,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -1260,13 +1280,15 @@ dependencies = [ [[package]] name = "gsb-http-proxy" version = "0.1.0" -source = "git+https://github.com/golemfactory/yagna.git?rev=09d4f43ef6ccc2865b3ffd00a6c3a62a33294940#09d4f43ef6ccc2865b3ffd00a6c3a62a33294940" dependencies = [ + "actix-http", + "actix-web", "async-stream", "chrono", "futures 0.3.30", "http 1.0.0", "log", + "mockito", "reqwest", "serde", "serde_json", @@ -1708,6 +1730,25 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mockito" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d3038e23466858569c2d30a537f691fa0d53b51626630ae08262943e3bbb8b" +dependencies = [ + "assert-json-diff", + "colored", + "futures 0.3.30", + "hyper", + "log", + "rand 0.8.5", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + [[package]] name = "multimap" version = "0.8.3" @@ -2567,6 +2608,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" + [[package]] name = "slab" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index 6f34ffe..e1386dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ regex = "1" reqwest = { version = "0.11", features = ["blocking", "json"] } async-stream = "0.3.5" #gsb-http-proxy = { path = "../yagna/exe-unit/components/http-proxy" } -gsb-http-proxy = { git = "https://github.com/golemfactory/yagna.git", rev = "09d4f43ef6ccc2865b3ffd00a6c3a62a33294940" } +gsb-http-proxy = { git = "https://github.com/golemfactory/yagna.git", rev = "6e07db56ac05e98fcc5ac355e054bd383d1173ba" } http = "0.2.11" bytes = "1.5.0" From 98997c8f0568f7073c837918f609dfbc2261e4c1 Mon Sep 17 00:00:00 2001 From: Staszek Krotki <16387248+staszek-krotki@users.noreply.github.com> Date: Mon, 15 Jan 2024 21:52:09 +0100 Subject: [PATCH 05/17] revision update --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e1386dc..ce797a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ regex = "1" reqwest = { version = "0.11", features = ["blocking", "json"] } async-stream = "0.3.5" #gsb-http-proxy = { path = "../yagna/exe-unit/components/http-proxy" } -gsb-http-proxy = { git = "https://github.com/golemfactory/yagna.git", rev = "6e07db56ac05e98fcc5ac355e054bd383d1173ba" } +gsb-http-proxy = { git = "https://github.com/golemfactory/yagna.git", rev = "04bcd84614f9085033d1c09c3f7fc2359dc60296" } http = "0.2.11" bytes = "1.5.0" From 72574bd9abf576b9cc431a97e6c87f0f7e96f998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Thu, 11 Jan 2024 11:52:12 +0100 Subject: [PATCH 06/17] Close on CTRL-BREAK --- Cargo.toml | 2 +- src/main.rs | 16 +++++++++++++ src/signal.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 src/signal.rs diff --git a/Cargo.toml b/Cargo.toml index ce797a6..a1e22d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ anyhow = "1.0" env_logger = "0.10" yansi = "0.5" chrono = "0.4" -tokio = { version = "1.32", features = ["macros"] } +tokio = { version = "1.32", features = ["macros", "signal"] } futures = "0.3" flexi_logger = { version = "0.27", features = ["colors"] } regex = "1" diff --git a/src/main.rs b/src/main.rs index ba59025..a10cd8b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ use clap::Parser; use futures::prelude::*; use gsb_http_proxy::GsbHttpCall; +use tokio::select; use ya_client_model::activity::activity_state::*; use ya_client_model::activity::ExeScriptCommand; use ya_client_model::activity::{ActivityUsage, CommandResult, ExeScriptCommandResult}; @@ -26,12 +27,14 @@ use crate::agreement::AgreementDesc; use crate::cli::*; use crate::logger::*; use crate::process::ProcessController; +use crate::signal::SignalMonitor; mod agreement; mod cli; mod logger; mod offer_template; mod process; +mod signal; async fn send_state(ctx: &ExeUnitContext, new_state: ActivityState) -> anyhow::Result<()> { Ok(gsb::service(ctx.report_url.clone()) @@ -123,6 +126,13 @@ async fn main() -> anyhow::Result<()> { } }; + select! { + res = handle_cli(cli) => res, + res = handle_signals() => res, + } +} + +async fn handle_cli(cli: Cli) -> anyhow::Result<()> { match cli.runtime.to_lowercase().as_str() { "dummy" => run::(cli).await, _ => { @@ -133,6 +143,12 @@ async fn main() -> anyhow::Result<()> { } } +async fn handle_signals() -> anyhow::Result<()> { + let signal = SignalMonitor::default().recv().await?; + log::info!("{} received, Shutting down runtime...", signal); + Ok(()) +} + #[derive(Clone)] struct ExeUnitContext { pub activity_id: String, diff --git a/src/signal.rs b/src/signal.rs new file mode 100644 index 0000000..7b0ce2b --- /dev/null +++ b/src/signal.rs @@ -0,0 +1,66 @@ +pub(crate) type Signal = &'static str; + +use tokio::task::JoinHandle; +use tokio::{ + select, + sync::{ + oneshot, + oneshot::{Receiver, Sender}, + }, +}; + +#[cfg(target_family = "unix")] +use tokio::signal::unix; +#[cfg(target_family = "windows")] +use tokio::signal::windows; + +pub struct SignalMonitor { + stop_tx: Sender, + stop_rx: Receiver, +} + +impl Default for SignalMonitor { + fn default() -> Self { + let (stop_tx, stop_rx) = oneshot::channel(); + Self { stop_tx, stop_rx } + } +} + +impl SignalMonitor { + pub async fn recv(self) -> anyhow::Result { + Self::start(self.stop_tx)?; + Ok(self.stop_rx.await?) + } + + #[cfg(target_family = "unix")] + fn start(stop_tx: Sender) -> anyhow::Result> { + let mut sigterm = unix::signal(unix::SignalKind::terminate())?; + let mut sigint = unix::signal(unix::SignalKind::interrupt())?; + let mut sigquit = unix::signal(unix::SignalKind::quit())?; + Ok(tokio::spawn(async move { + select! { + _ = sigterm.recv() => stop_tx.send("SIGTERM").expect("Failed to handle SIGTERM event"), + _ = sigint.recv() => stop_tx.send("SIGINT").expect("Failed to handle SIGINT event"), + _ = sigquit.recv() => stop_tx.send("SIGQUIT").expect("Failed to handle SIGQUIT event"), + }; + })) + } + + #[cfg(target_family = "windows")] + fn start(stop_tx: Sender) -> anyhow::Result> { + let mut ctrl_c = windows::ctrl_c()?; + let mut ctrl_close = windows::ctrl_close()?; + let mut ctrl_logoff = windows::ctrl_logoff()?; + let mut ctrl_shutdown = windows::ctrl_shutdown()?; + let mut ctrl_break = windows::ctrl_break()?; + Ok(tokio::spawn(async move { + select! { + _ = ctrl_c.recv() => stop_tx.send("CTRL-C").expect("Failed to handle CTRL-C event"), + _ = ctrl_close.recv() => stop_tx.send("CTRL-CLOSE").expect("Failed to handle CTRL-CLOSE event"), + _ = ctrl_logoff.recv() => stop_tx.send("CTRL-LOGOFF").expect("Failed to handle CTRL-LOGOFF event"), + _ = ctrl_shutdown.recv() => stop_tx.send("CTRL-SHUTDOWN").expect("Failed to handle CTRL-HUTDOWN event"), + _ = ctrl_break.recv() => stop_tx.send("CTRL-BREAK").expect("Failed to handle CTRL-BREAK event") + }; + })) + } +} From a1a62d0aa8db9ca0afab37198bdc9c86459bbdc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Wed, 13 Dec 2023 20:55:32 +0100 Subject: [PATCH 07/17] Version update. Dependencies update. --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e63485d..21b666e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3590,7 +3590,7 @@ dependencies = [ [[package]] name = "ya-runtime-ai" -version = "0.1.0" +version = "0.1.1" dependencies = [ "actix", "actix-rt", diff --git a/Cargo.toml b/Cargo.toml index a1e22d3..46d6398 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-runtime-ai" -version = "0.1.0" +version = "0.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From c0a0501f6e34d597ff54eeb01ccf844d3ce2135c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Thu, 14 Dec 2023 15:56:19 +0100 Subject: [PATCH 08/17] Automatic WIP --- src/main.rs | 1 + src/process.rs | 1 + src/process/automatic.rs | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 src/process/automatic.rs diff --git a/src/main.rs b/src/main.rs index a10cd8b..aa7da3b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -135,6 +135,7 @@ async fn main() -> anyhow::Result<()> { async fn handle_cli(cli: Cli) -> anyhow::Result<()> { match cli.runtime.to_lowercase().as_str() { "dummy" => run::(cli).await, + "automatic" => run::(cli).await, _ => { let err = anyhow::format_err!("Unsupported framework {}", cli.runtime); log::error!("{}", err); diff --git a/src/process.rs b/src/process.rs index 3ad45b5..f61030d 100644 --- a/src/process.rs +++ b/src/process.rs @@ -13,6 +13,7 @@ use std::task::Poll; use tokio::process::*; pub mod dummy; +pub mod automatic; pub mod win; #[derive(Default, Clone)] diff --git a/src/process/automatic.rs b/src/process/automatic.rs new file mode 100644 index 0000000..3eb2074 --- /dev/null +++ b/src/process/automatic.rs @@ -0,0 +1,33 @@ +use std::{path::Path, process::Stdio}; + +use tokio::process::Command; + +use super::{AiFramework, RuntimeArgs}; + +#[derive(Clone)] +pub struct Automatic { + +} + +static _STARTUP_SCRIPT: &str = "automatic/run.bat"; + +impl AiFramework for Automatic { + fn parse_args(args: &[String]) -> anyhow::Result { + RuntimeArgs::new(&_STARTUP_SCRIPT.into(), args) + } + + fn start(args: &super::RuntimeArgs) -> anyhow::Result { + log::info!("Start cmd"); + let exe = super::find_exe(_STARTUP_SCRIPT)?; + let mut cmd = Command::new(&exe); + let work_dir = exe.parent().unwrap(); + cmd.stdout(Stdio::piped()) + .stdin(Stdio::null()) + .current_dir(work_dir); + Ok(cmd.kill_on_drop(true).spawn()?) + } + + fn run(stdout: tokio::process::ChildStdout, report_fn: ReportFn) { + log::info!("Run cmd"); + } +} From 527e20a1ebf8ddfcfa9f10f13183ba6ceab10abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Tue, 2 Jan 2024 16:11:51 +0100 Subject: [PATCH 09/17] Logging process stdout --- src/process/automatic.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/process/automatic.rs b/src/process/automatic.rs index 3eb2074..67c9a5c 100644 --- a/src/process/automatic.rs +++ b/src/process/automatic.rs @@ -1,6 +1,6 @@ use std::{path::Path, process::Stdio}; -use tokio::process::Command; +use tokio::{process::Command, io::BufReader, io::AsyncBufReadExt, }; use super::{AiFramework, RuntimeArgs}; @@ -28,6 +28,15 @@ impl AiFramework for Automatic { } fn run(stdout: tokio::process::ChildStdout, report_fn: ReportFn) { - log::info!("Run cmd"); + tokio::spawn(async move { + let mut reader = BufReader::new(stdout).lines(); + + while let Some(line) = reader.next_line().await.unwrap_or_else(|e| { + log::debug!("Error reading line from stdout: {}", e); + None + }) { + log::debug!("{}", line); + } + }); } } From 25e37693efa8947cffa6db2b8768210e381c761c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Mon, 15 Jan 2024 17:12:38 +0100 Subject: [PATCH 10/17] Automatic API kill server request --- Cargo.lock | 2 ++ Cargo.toml | 1 + src/main.rs | 23 ++++++------- src/process.rs | 38 +++++++++------------ src/process/automatic.rs | 69 +++++++++++++++++++++++++++---------- src/process/dummy.rs | 73 ++++++++++++++++++++++++---------------- 6 files changed, 124 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21b666e..9f99a1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1280,6 +1280,7 @@ dependencies = [ [[package]] name = "gsb-http-proxy" version = "0.1.0" +source = "git+https://github.com/golemfactory/yagna.git?rev=04bcd84614f9085033d1c09c3f7fc2359dc60296#04bcd84614f9085033d1c09c3f7fc2359dc60296" dependencies = [ "actix-http", "actix-web", @@ -3596,6 +3597,7 @@ dependencies = [ "actix-rt", "anyhow", "async-stream", + "async-trait", "bytes 1.5.0", "chrono", "clap 4.4.16", diff --git a/Cargo.toml b/Cargo.toml index 46d6398..0054ae4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ ya-transfer = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b0 actix = "0.13" actix-rt = "2" +async-trait = "0.1.77" log = "0.4" serde = { version = "^1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/main.rs b/src/main.rs index aa7da3b..2ea0150 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use std::cell::RefCell; use std::collections::HashMap; use std::io; @@ -14,6 +12,7 @@ use clap::Parser; use futures::prelude::*; use gsb_http_proxy::GsbHttpCall; +use process::Runtime; use tokio::select; use ya_client_model::activity::activity_state::*; use ya_client_model::activity::ExeScriptCommand; @@ -36,7 +35,10 @@ mod offer_template; mod process; mod signal; -async fn send_state(ctx: &ExeUnitContext, new_state: ActivityState) -> anyhow::Result<()> { +async fn send_state( + ctx: &ExeUnitContext, + new_state: ActivityState, +) -> anyhow::Result<()> { Ok(gsb::service(ctx.report_url.clone()) .call(activity::local::SetState::new( ctx.activity_id.clone(), @@ -46,7 +48,7 @@ async fn send_state(ctx: &ExeUnitContext, new_state: ActivityState) -> any .await??) } -async fn activity_loop( +async fn activity_loop( report_url: &str, activity_id: &str, mut process: ProcessController, @@ -151,7 +153,7 @@ async fn handle_signals() -> anyhow::Result<()> { } #[derive(Clone)] -struct ExeUnitContext { +struct ExeUnitContext { pub activity_id: String, pub report_url: String, @@ -162,7 +164,7 @@ struct ExeUnitContext { pub batches: Rc>>>, } -async fn run(cli: Cli) -> anyhow::Result<()> { +async fn run(cli: Cli) -> anyhow::Result<()> { dotenv::dotenv().ok(); let (exe_unit_url, report_url, activity_id, args) = match &cli.command { @@ -218,14 +220,9 @@ async fn run(cli: Cli) -> any { let batch = ctx.batches.clone(); let batch_results = batch.clone(); - let ctx = ctx.clone(); - - log::info!("exe_unit_url: {}", exe_unit_url); - - log::info!("[GSB Bind]: Exec"); + let ctx = ctx.clone(); gsb::bind(&exe_unit_url, move |exec: activity::Exec| { - let ctx = ctx.clone(); let exec = exec.clone(); let batch = batch.clone(); let batch_id = exec.batch_id.clone(); @@ -237,7 +234,7 @@ async fn run(cli: Cli) -> any .borrow_mut() .insert(exec.batch_id.clone(), vec![]); } - + let ctx = ctx.clone(); let script_future = async move { let mut result = Vec::new(); for exe in &exec.exe_script { diff --git a/src/process.rs b/src/process.rs index f61030d..17f79ab 100644 --- a/src/process.rs +++ b/src/process.rs @@ -1,19 +1,17 @@ use anyhow::Context; +use async_trait::async_trait; use clap::Parser; use std::cell::RefCell; use std::env::current_exe; use std::future::Future; -use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::pin::{pin, Pin}; use std::process::ExitStatus; use std::rc::Rc; use std::task::Poll; -use tokio::process::*; - -pub mod dummy; pub mod automatic; +pub mod dummy; pub mod win; #[derive(Default, Clone)] @@ -21,12 +19,15 @@ pub struct Usage { pub cnt: u64, } -pub trait AiFramework { +#[async_trait] +pub trait Runtime: Sized { fn parse_args(args: &[String]) -> anyhow::Result; - fn start(args: &RuntimeArgs) -> anyhow::Result; + fn start(args: &RuntimeArgs) -> anyhow::Result; - fn run(stdout: ChildStdout, report_fn: ReportFn); + async fn stop(&mut self) -> anyhow::Result<()>; + + async fn wait(&mut self) -> std::io::Result; } #[derive(Parser)] @@ -43,15 +44,14 @@ impl RuntimeArgs { } #[derive(Clone)] -pub struct ProcessController { - inner: Rc>, - _marker: PhantomData, +pub struct ProcessController { + inner: Rc>>, } #[allow(clippy::large_enum_variant)] -enum ProcessControllerInner { +enum ProcessControllerInner { Deployed {}, - Working { child: Child }, + Working { child: T }, Stopped {}, } @@ -67,11 +67,10 @@ pub fn find_exe(file_name: impl AsRef) -> anyhow::Result { anyhow::bail!("Unable to get dummy runtime base dir"); } -impl ProcessController { +impl ProcessController { pub fn new() -> Self { ProcessController { inner: Rc::new(RefCell::new(ProcessControllerInner::Deployed {})), - _marker: Default::default(), } } @@ -87,26 +86,21 @@ impl ProcessController { let () = self.report().unwrap_or_default(); let old = self.inner.replace(ProcessControllerInner::Stopped {}); if let ProcessControllerInner::Working { mut child, .. } = old { - let _ = child.kill().await; + let _ = child.stop().await; } } pub async fn start(&self, args: &RuntimeArgs) -> anyhow::Result<()> { - let mut child = T::start(args)?; + let child = T::start(args)?; - let opt_stdout = child.stdout.take(); self.inner .replace(ProcessControllerInner::Working { child }); - if let Some(stdout) = opt_stdout { - let _me: ProcessController = self.clone(); - T::run(stdout, move |_| {}); - } Ok(()) } } -impl Future for ProcessController { +impl Future for ProcessController { type Output = std::io::Result; fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { diff --git a/src/process/automatic.rs b/src/process/automatic.rs index 67c9a5c..f90f8d9 100644 --- a/src/process/automatic.rs +++ b/src/process/automatic.rs @@ -1,22 +1,36 @@ -use std::{path::Path, process::Stdio}; +use std::{ + process::{ExitStatus, Stdio}, + sync::Arc, +}; -use tokio::{process::Command, io::BufReader, io::AsyncBufReadExt, }; +use async_trait::async_trait; +use tokio::{ + io::AsyncBufReadExt, + io::BufReader, + process::{Child, Command}, + sync::Mutex, +}; -use super::{AiFramework, RuntimeArgs}; +use super::{Runtime, RuntimeArgs}; #[derive(Clone)] pub struct Automatic { - + child: Arc>, } static _STARTUP_SCRIPT: &str = "automatic/run.bat"; -impl AiFramework for Automatic { +static _API_HOST: &str = "http://localhost:7861"; + +static _API_KILL_PATH: &str = "sdapi/v1/server-stop"; + +#[async_trait] +impl Runtime for Automatic { fn parse_args(args: &[String]) -> anyhow::Result { RuntimeArgs::new(&_STARTUP_SCRIPT.into(), args) } - fn start(args: &super::RuntimeArgs) -> anyhow::Result { + fn start(_args: &super::RuntimeArgs) -> anyhow::Result { log::info!("Start cmd"); let exe = super::find_exe(_STARTUP_SCRIPT)?; let mut cmd = Command::new(&exe); @@ -24,19 +38,38 @@ impl AiFramework for Automatic { cmd.stdout(Stdio::piped()) .stdin(Stdio::null()) .current_dir(work_dir); - Ok(cmd.kill_on_drop(true).spawn()?) + let mut child = cmd.kill_on_drop(true).spawn()?; + + let stdout = child.stdout.take(); + + if let Some(stdout) = stdout { + tokio::spawn(async move { + let mut reader = BufReader::new(stdout).lines(); + + while let Some(line) = reader.next_line().await.unwrap_or_else(|e| { + log::debug!("Error reading line from stdout: {}", e); + None + }) { + log::debug!("{}", line); + } + }); + } + + let child = Arc::new(Mutex::new(child)); + Ok(Self { child }) + } + + async fn stop(&mut self) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + client + .post(format!("{_API_HOST}/{_API_KILL_PATH}")) + .send() + .await?; + Ok(()) } - fn run(stdout: tokio::process::ChildStdout, report_fn: ReportFn) { - tokio::spawn(async move { - let mut reader = BufReader::new(stdout).lines(); - - while let Some(line) = reader.next_line().await.unwrap_or_else(|e| { - log::debug!("Error reading line from stdout: {}", e); - None - }) { - log::debug!("{}", line); - } - }); + async fn wait(&mut self) -> std::io::Result { + let mut child = self.child.lock().await; + child.wait().await } } diff --git a/src/process/dummy.rs b/src/process/dummy.rs index b5ce3fb..760318c 100644 --- a/src/process/dummy.rs +++ b/src/process/dummy.rs @@ -1,24 +1,30 @@ -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; +use std::sync::Arc; +use async_trait::async_trait; use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio::process::{Child, ChildStdout, Command}; +use tokio::process::{Child, Command}; +use tokio::sync::Mutex; -use super::{AiFramework, RuntimeArgs, Usage}; +use super::{Runtime, RuntimeArgs}; #[derive(Clone)] -pub struct Dummy {} - -impl Dummy {} +pub struct Dummy { + child: Arc>, +} -impl Unpin for Dummy {} +fn dummy_filename() -> String { + format!("dummy{}", std::env::consts::EXE_SUFFIX) +} -impl AiFramework for Dummy { +#[async_trait] +impl Runtime for Dummy { fn parse_args(args: &[String]) -> anyhow::Result { let dummy_filename = dummy_filename(); RuntimeArgs::new(&dummy_filename, args) } - fn start(args: &RuntimeArgs) -> anyhow::Result { + fn start(args: &super::RuntimeArgs) -> anyhow::Result { let dummy_filename = dummy_filename(); let exe = super::find_exe(dummy_filename)?; let mut cmd = Command::new(&exe); @@ -28,29 +34,38 @@ impl AiFramework for Dummy { .current_dir(work_dir) .arg("--model") .arg(&args.model); - Ok(cmd.kill_on_drop(true).spawn()?) - } + let mut child = cmd.kill_on_drop(true).spawn()?; - fn run(stdout: ChildStdout, _report_fn: ReportFn) { - tokio::task::spawn_local(async move { - let mut stdout = BufReader::new(stdout); - loop { - let mut line_buf = String::new(); - match stdout.read_line(&mut line_buf).await { - Err(e) => { - log::error!("no line: {}", e); - break; + let stdout = child.stdout.take(); + if let Some(stdout) = stdout { + tokio::task::spawn_local(async move { + let mut stdout = BufReader::new(stdout); + loop { + let mut line_buf = String::new(); + match stdout.read_line(&mut line_buf).await { + Err(e) => { + log::error!("no line: {}", e); + break; + } + Ok(0) => break, + Ok(_) => (), } - Ok(0) => break, - Ok(_) => (), + let line = line_buf.trim_end(); + log::info!("dummy response: {line}"); } - let line = line_buf.trim_end(); - log::info!("dummy response: {line}"); - } - }); + }); + } + + let child = Arc::new(Mutex::new(child)); + Ok(Self { child }) } -} -fn dummy_filename() -> String { - format!("dummy{}", std::env::consts::EXE_SUFFIX) + async fn stop(&mut self) -> anyhow::Result<()> { + Ok(()) + } + + async fn wait(&mut self) -> std::io::Result { + let mut child = self.child.lock().await; + child.wait().await + } } From 2a19cce16f6892cd5884be4a03ed6fabda79a2b2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Mon, 15 Jan 2024 19:10:09 +0100 Subject: [PATCH 11/17] Stopping runtime on CTRL-BREAK --- src/main.rs | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2ea0150..4214834 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,6 @@ use std::cell::RefCell; use std::collections::HashMap; use std::io; use std::io::Write; -use std::pin::pin; use std::rc::Rc; use std::time::Duration; @@ -14,6 +13,7 @@ use gsb_http_proxy::GsbHttpCall; use process::Runtime; use tokio::select; +use tokio::sync::{mpsc, mpsc::Receiver, mpsc::Sender}; use ya_client_model::activity::activity_state::*; use ya_client_model::activity::ExeScriptCommand; use ya_client_model::activity::{ActivityUsage, CommandResult, ExeScriptCommandResult}; @@ -35,6 +35,8 @@ mod offer_template; mod process; mod signal; +pub type Signal = &'static str; + async fn send_state( ctx: &ExeUnitContext, new_state: ActivityState, @@ -51,8 +53,9 @@ async fn send_state( async fn activity_loop( report_url: &str, activity_id: &str, - mut process: ProcessController, + process: ProcessController, agreement: AgreementDesc, + mut signal_receiver: Receiver, ) -> anyhow::Result<()> { let report_service = gsb::service(report_url); let start = Utc::now(); @@ -81,12 +84,17 @@ async fn activity_loop( Ok(Err(rpc_message_error)) => log::error!("rpcMessageError : {:?}", rpc_message_error), Err(err) => log::error!("other error : {:?}", err), } - log::debug!("Looping2 ..."); - - let sleep = pin!(actix_rt::time::sleep(Duration::from_secs(1))); - process = match future::select(sleep, process).await { - future::Either::Left((_, p)) => p, - future::Either::Right((status, _)) => { + log::debug!("Looping ..."); + + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + signal = signal_receiver.recv() => { + if let Some(signal) = signal { + log::debug!("Received signal {signal}. Stopping runtime"); + process.stop().await; + } + }, + status = process.clone() => { let _err = report_service .call(activity::local::SetState { activity_id: activity_id.to_string(), @@ -100,8 +108,9 @@ async fn activity_loop( }) .await; log::error!("process exit: {:?}", status); - anyhow::bail!("Runtime exited") + anyhow::bail!("Runtime exited"); } + } } Ok(()) @@ -128,16 +137,18 @@ async fn main() -> anyhow::Result<()> { } }; + let (signal_sender, signal_receiver) = mpsc::channel::(1); + select! { - res = handle_cli(cli) => res, - res = handle_signals() => res, + res = handle_cli(cli, signal_receiver) => res, + res = handle_signals(signal_sender) => res, } } -async fn handle_cli(cli: Cli) -> anyhow::Result<()> { +async fn handle_cli(cli: Cli, signal_receiver: Receiver) -> anyhow::Result<()> { match cli.runtime.to_lowercase().as_str() { - "dummy" => run::(cli).await, - "automatic" => run::(cli).await, + "dummy" => run::(cli, signal_receiver).await, + "automatic" => run::(cli, signal_receiver).await, _ => { let err = anyhow::format_err!("Unsupported framework {}", cli.runtime); log::error!("{}", err); @@ -146,10 +157,10 @@ async fn handle_cli(cli: Cli) -> anyhow::Result<()> { } } -async fn handle_signals() -> anyhow::Result<()> { +async fn handle_signals(signal_receiver: Sender) -> anyhow::Result<()> { let signal = SignalMonitor::default().recv().await?; log::info!("{} received, Shutting down runtime...", signal); - Ok(()) + Ok(signal_receiver.send(signal).await?) } #[derive(Clone)] @@ -164,7 +175,10 @@ struct ExeUnitContext { pub batches: Rc>>>, } -async fn run(cli: Cli) -> anyhow::Result<()> { +async fn run( + cli: Cli, + signal_receiver: Receiver, +) -> anyhow::Result<()> { dotenv::dotenv().ok(); let (exe_unit_url, report_url, activity_id, args) = match &cli.command { @@ -213,6 +227,7 @@ async fn run(cli: Cli) -> anyhow: activity_id, ctx.process_controller.clone(), ctx.agreement.clone(), + signal_receiver, ); #[cfg(target_os = "windows")] From dff706de2c5a9857687bad6a53926e2dbab44824 Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Mon, 15 Jan 2024 19:30:47 +0100 Subject: [PATCH 12/17] Stopping runtime on GSB Terminate --- src/process/automatic.rs | 1 + src/process/dummy.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/process/automatic.rs b/src/process/automatic.rs index f90f8d9..6249d4c 100644 --- a/src/process/automatic.rs +++ b/src/process/automatic.rs @@ -60,6 +60,7 @@ impl Runtime for Automatic { } async fn stop(&mut self) -> anyhow::Result<()> { + log::info!("Stopping automatic server"); let client = reqwest::Client::new(); client .post(format!("{_API_HOST}/{_API_KILL_PATH}")) diff --git a/src/process/dummy.rs b/src/process/dummy.rs index 760318c..a421f89 100644 --- a/src/process/dummy.rs +++ b/src/process/dummy.rs @@ -61,6 +61,7 @@ impl Runtime for Dummy { } async fn stop(&mut self) -> anyhow::Result<()> { + log::info!("Stopping runtime"); Ok(()) } From 832c4eebfa882ccbae4bf5b82f00ade5895b5da1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Mon, 15 Jan 2024 23:38:35 +0100 Subject: [PATCH 13/17] Automatic model argument --- src/main.rs | 17 ++++++----- src/process.rs | 22 ++------------ src/process/automatic.rs | 65 ++++++++++++++++++++++++++++++++++++---- src/process/dummy.rs | 17 +++++------ 4 files changed, 78 insertions(+), 43 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4214834..1dcbbe4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::collections::HashMap; use std::io; use std::io::Write; +use std::path::PathBuf; use std::rc::Rc; use std::time::Duration; @@ -173,6 +174,8 @@ struct ExeUnitContext { pub process_controller: ProcessController, pub batches: Rc>>>, + + pub model_path: Option, } async fn run( @@ -220,6 +223,7 @@ async fn run( .start(), process_controller: process::ProcessController::::new(), batches: Rc::new(RefCell::new(Default::default())), + model_path: None, }; let activity_pinger = activity_loop( @@ -249,7 +253,7 @@ async fn run( .borrow_mut() .insert(exec.batch_id.clone(), vec![]); } - let ctx = ctx.clone(); + let mut ctx = ctx.clone(); let script_future = async move { let mut result = Vec::new(); for exe in &exec.exe_script { @@ -270,7 +274,8 @@ async fn run( ctx.agreement.model ); - ctx.transfers + ctx.model_path = ctx + .transfers .send(DeployImage { task_package: Some(ctx.agreement.model.clone()), }) @@ -298,11 +303,7 @@ async fn run( }); } ExeScriptCommand::Start { args, .. } => { - log::debug!("Raw Start cmd args: {args:?}"); - let args = T::parse_args(args).map_err(|e| { - RpcMessageError::Activity(format!("invalid args: {}", e)) - })?; - log::debug!("Start cmd model: {}", args.model); + log::debug!("Raw Start cmd args: {args:?} [ignored]"); send_state( &ctx, @@ -312,7 +313,7 @@ async fn run( .map_err(|e| RpcMessageError::Service(e.to_string()))?; ctx.process_controller - .start(&args) + .start(ctx.model_path.clone()) .await .map_err(|e| RpcMessageError::Activity(e.to_string()))?; log::debug!("Started process"); diff --git a/src/process.rs b/src/process.rs index 17f79ab..da623bc 100644 --- a/src/process.rs +++ b/src/process.rs @@ -1,6 +1,5 @@ use anyhow::Context; use async_trait::async_trait; -use clap::Parser; use std::cell::RefCell; use std::env::current_exe; use std::future::Future; @@ -21,28 +20,13 @@ pub struct Usage { #[async_trait] pub trait Runtime: Sized { - fn parse_args(args: &[String]) -> anyhow::Result; - - fn start(args: &RuntimeArgs) -> anyhow::Result; + fn start(mode: Option) -> anyhow::Result; async fn stop(&mut self) -> anyhow::Result<()>; async fn wait(&mut self) -> std::io::Result; } -#[derive(Parser)] -#[cfg_attr(test, derive(Debug, Eq, PartialEq))] -pub struct RuntimeArgs { - #[arg(long)] - pub model: String, -} - -impl RuntimeArgs { - pub fn new(cmd: &String, args: &[String]) -> anyhow::Result { - Ok(Self::try_parse_from(std::iter::once(cmd).chain(args))?) - } -} - #[derive(Clone)] pub struct ProcessController { inner: Rc>>, @@ -90,8 +74,8 @@ impl ProcessController { } } - pub async fn start(&self, args: &RuntimeArgs) -> anyhow::Result<()> { - let child = T::start(args)?; + pub async fn start(&self, model: Option) -> anyhow::Result<()> { + let child = T::start(model)?; self.inner .replace(ProcessControllerInner::Working { child }); diff --git a/src/process/automatic.rs b/src/process/automatic.rs index 6249d4c..35ffb14 100644 --- a/src/process/automatic.rs +++ b/src/process/automatic.rs @@ -1,4 +1,6 @@ use std::{ + ffi::OsStr, + path::PathBuf, process::{ExitStatus, Stdio}, sync::Arc, }; @@ -11,7 +13,7 @@ use tokio::{ sync::Mutex, }; -use super::{Runtime, RuntimeArgs}; +use super::Runtime; #[derive(Clone)] pub struct Automatic { @@ -24,20 +26,31 @@ static _API_HOST: &str = "http://localhost:7861"; static _API_KILL_PATH: &str = "sdapi/v1/server-stop"; +static _MODEL_ARG: &str = "--ckpt"; + +static _MODEL_DIR_ARG: &str = "--ckpt-dir"; + #[async_trait] impl Runtime for Automatic { - fn parse_args(args: &[String]) -> anyhow::Result { - RuntimeArgs::new(&_STARTUP_SCRIPT.into(), args) - } - - fn start(_args: &super::RuntimeArgs) -> anyhow::Result { + fn start(model: Option) -> anyhow::Result { log::info!("Start cmd"); let exe = super::find_exe(_STARTUP_SCRIPT)?; + let mut cmd = Command::new(&exe); + + if let Some(model) = model { + let ckpt_dir = model.parent().and_then(ckpt_dir); + let model_file = model.file_name().and_then(OsStr::to_str); + if let (Some(ckpt_dir), Some(model_file)) = (ckpt_dir, model_file) { + cmd.args([_MODEL_DIR_ARG, &ckpt_dir, _MODEL_ARG, model_file]); + } + } + let work_dir = exe.parent().unwrap(); cmd.stdout(Stdio::piped()) .stdin(Stdio::null()) .current_dir(work_dir); + let mut child = cmd.kill_on_drop(true).spawn()?; let stdout = child.stdout.take(); @@ -74,3 +87,43 @@ impl Runtime for Automatic { child.wait().await } } + +// Automatic needs following ckpt-dir format: C:\\some/path +#[cfg(target_family = "windows")] +fn ckpt_dir(ckpt_dir: &std::path::Path) -> Option { + use std::{collections::VecDeque, path::Path}; + + if ckpt_dir.has_root() { + let mut path_parts = VecDeque::new(); + let mut dir = Some(ckpt_dir); + while let Some(name) = dir.and_then(Path::file_name).and_then(OsStr::to_str) { + path_parts.push_front(name); + dir = dir.and_then(Path::parent); + } + if let Some(disk) = dir.and_then(Path::to_str) { + let relative_path = Into::>::into(path_parts).join("/"); + return Some(format!("{disk}\\{relative_path}")); + } + } + log::error!("Unable to build ckpt_dir in correct format from path: {ckpt_dir:?}"); + None +} + +#[cfg(target_family = "unix")] +fn ckpt_dir(ckpt_dir: &std::path::Path) -> Option { + ckpt_dir.to_str().map(str::to_string) +} + +#[cfg(target_family = "windows")] +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + + #[test] + fn ckpt_dir_test() { + let path = Path::new("C:\\my\\model\\dir"); + assert_eq!(ckpt_dir(path), Some("C:\\\\my/model/dir".to_string())); + } +} diff --git a/src/process/dummy.rs b/src/process/dummy.rs index a421f89..0b61fc0 100644 --- a/src/process/dummy.rs +++ b/src/process/dummy.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::process::{ExitStatus, Stdio}; use std::sync::Arc; @@ -6,7 +7,7 @@ use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::sync::Mutex; -use super::{Runtime, RuntimeArgs}; +use super::Runtime; #[derive(Clone)] pub struct Dummy { @@ -19,21 +20,17 @@ fn dummy_filename() -> String { #[async_trait] impl Runtime for Dummy { - fn parse_args(args: &[String]) -> anyhow::Result { - let dummy_filename = dummy_filename(); - RuntimeArgs::new(&dummy_filename, args) - } - - fn start(args: &super::RuntimeArgs) -> anyhow::Result { + fn start(model: Option) -> anyhow::Result { let dummy_filename = dummy_filename(); let exe = super::find_exe(dummy_filename)?; let mut cmd = Command::new(&exe); let work_dir = exe.parent().unwrap(); + if let Some(model) = model { + cmd.args(["--model", &model.to_string_lossy()]); + } cmd.stdout(Stdio::piped()) .stdin(Stdio::null()) - .current_dir(work_dir) - .arg("--model") - .arg(&args.model); + .current_dir(work_dir); let mut child = cmd.kill_on_drop(true).spawn()?; let stdout = child.stdout.take(); From c83f48a5fae523057cdc573f4d9b44f04fabc375 Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Wed, 17 Jan 2024 02:38:44 +0100 Subject: [PATCH 14/17] Dummy framework rename. Clippy --- conf/ya-dummy-ai.json | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/ya-dummy-ai.json b/conf/ya-dummy-ai.json index 26345be..02a3258 100644 --- a/conf/ya-dummy-ai.json +++ b/conf/ya-dummy-ai.json @@ -1,6 +1,6 @@ [ { - "name": "ai", + "name": "dummy", "version": "0.1.0", "supervisor-path": "ya-runtime-ai.exe", "extra-args": [ diff --git a/src/main.rs b/src/main.rs index 1dcbbe4..f9c8b6d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -408,7 +408,7 @@ async fn run( gsb::bind_stream(&exe_unit_url, move |mut http_call: GsbHttpCall| { let stream = http_call.execute("http://localhost:7861/".to_string()); - Box::pin(stream.map(|e| Ok(e))) + Box::pin(stream.map(Ok)) }); }; send_state( From 69a92d22458d53bcafb175f1e301f171f8aaca8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Walski?= Date: Wed, 17 Jan 2024 02:50:06 +0100 Subject: [PATCH 15/17] ya-core-model update --- Cargo.lock | 11 ++++++----- Cargo.toml | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f99a1f..b7ae509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3545,6 +3545,8 @@ dependencies = [ [[package]] name = "ya-core-model" version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dddc3bccc5be20eb58baa2f1bc61d62221b28e4d307c5c4bdf363e3c3149ddaa" dependencies = [ "chrono", "derive_more", @@ -3555,15 +3557,14 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.6.0", + "ya-client-model 0.5.0", "ya-service-bus", ] [[package]] name = "ya-core-model" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dddc3bccc5be20eb58baa2f1bc61d62221b28e4d307c5c4bdf363e3c3149ddaa" +source = "git+https://github.com/golemfactory/yagna.git?rev=04bcd84614f9085033d1c09c3f7fc2359dc60296#04bcd84614f9085033d1c09c3f7fc2359dc60296" dependencies = [ "chrono", "derive_more", @@ -3574,7 +3575,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.5.0", + "ya-client-model 0.6.0", "ya-service-bus", ] @@ -3616,7 +3617,7 @@ dependencies = [ "winapi", "ya-agreement-utils", "ya-client-model 0.6.0", - "ya-core-model 0.9.0", + "ya-core-model 0.9.0 (git+https://github.com/golemfactory/yagna.git?rev=04bcd84614f9085033d1c09c3f7fc2359dc60296)", "ya-runtime-api", "ya-service-bus", "ya-transfer", diff --git a/Cargo.toml b/Cargo.toml index 0054ae4..c369fc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,7 @@ members = [ ] [dependencies] -#ya-core-model = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53", features = ["activity", "appkey"] } -ya-core-model = { path = "../yagna/core/model", features = ["activity", "appkey"] } +ya-core-model = { git = "https://github.com/golemfactory/yagna.git", rev = "04bcd84614f9085033d1c09c3f7fc2359dc60296", features = ["activity", "appkey"] } ya-runtime-api = { git = "https://github.com/golemfactory/yagna.git", rev = "6fd7b003e3d0947284d47d99b2944d293d329c53" } ya-service-bus = "0.6.3" ya-client-model = "0.6.0" From 0c8a58bd56077d8250592b178420feef0fb00eaa Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Wed, 17 Jan 2024 13:39:18 +0100 Subject: [PATCH 16/17] Using model arg only (no model dir). Additional skip test args. Startup script path matching automatic package. --- src/process/automatic.rs | 46 +++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/process/automatic.rs b/src/process/automatic.rs index 35ffb14..b209e6d 100644 --- a/src/process/automatic.rs +++ b/src/process/automatic.rs @@ -1,5 +1,4 @@ use std::{ - ffi::OsStr, path::PathBuf, process::{ExitStatus, Stdio}, sync::Arc, @@ -20,15 +19,21 @@ pub struct Automatic { child: Arc>, } -static _STARTUP_SCRIPT: &str = "automatic/run.bat"; +//TODO parameterize it + +static _STARTUP_SCRIPT: &str = "sd.webui_noxformers/run.bat"; static _API_HOST: &str = "http://localhost:7861"; -static _API_KILL_PATH: &str = "sdapi/v1/server-stop"; +static _API_KILL_PATH: &str = "sdapi/v1/server-kill"; static _MODEL_ARG: &str = "--ckpt"; -static _MODEL_DIR_ARG: &str = "--ckpt-dir"; +static _SKIP_TEST_ARGS: [&str; 3] = [ + "--skip-torch-cuda-test", + "--skip-python-version-check", + "--skip-version-check", +]; #[async_trait] impl Runtime for Automatic { @@ -38,12 +43,12 @@ impl Runtime for Automatic { let mut cmd = Command::new(&exe); - if let Some(model) = model { - let ckpt_dir = model.parent().and_then(ckpt_dir); - let model_file = model.file_name().and_then(OsStr::to_str); - if let (Some(ckpt_dir), Some(model_file)) = (ckpt_dir, model_file) { - cmd.args([_MODEL_DIR_ARG, &ckpt_dir, _MODEL_ARG, model_file]); - } + cmd.args(_SKIP_TEST_ARGS); + + if let Some(model) = model.and_then(format_path) { + cmd.args([_MODEL_ARG, &model]); + } else { + log::warn!("No model arg"); } let work_dir = exe.parent().unwrap(); @@ -90,12 +95,12 @@ impl Runtime for Automatic { // Automatic needs following ckpt-dir format: C:\\some/path #[cfg(target_family = "windows")] -fn ckpt_dir(ckpt_dir: &std::path::Path) -> Option { - use std::{collections::VecDeque, path::Path}; +fn format_path(path: std::path::PathBuf) -> Option { + use std::{collections::VecDeque, ffi::OsStr, path::Path}; - if ckpt_dir.has_root() { + if path.has_root() { let mut path_parts = VecDeque::new(); - let mut dir = Some(ckpt_dir); + let mut dir = Some(path.as_path()); while let Some(name) = dir.and_then(Path::file_name).and_then(OsStr::to_str) { path_parts.push_front(name); dir = dir.and_then(Path::parent); @@ -105,13 +110,13 @@ fn ckpt_dir(ckpt_dir: &std::path::Path) -> Option { return Some(format!("{disk}\\{relative_path}")); } } - log::error!("Unable to build ckpt_dir in correct format from path: {ckpt_dir:?}"); + log::error!("Unable to build ckpt_dir in correct format from path: {path:?}"); None } #[cfg(target_family = "unix")] -fn ckpt_dir(ckpt_dir: &std::path::Path) -> Option { - ckpt_dir.to_str().map(str::to_string) +fn format_path(path: std::path::PathBuf) -> Option { + path.to_str().map(str::to_string) } #[cfg(target_family = "windows")] @@ -123,7 +128,10 @@ mod tests { #[test] fn ckpt_dir_test() { - let path = Path::new("C:\\my\\model\\dir"); - assert_eq!(ckpt_dir(path), Some("C:\\\\my/model/dir".to_string())); + let path = PathBuf::from(Path::new("C:\\my\\model\\model.ckpt")); + assert_eq!( + format_path(path), + Some("C:\\\\my/model/model.ckpt".to_string()) + ); } } From 013319dde5b54a87764365828fbe16f75ded608c Mon Sep 17 00:00:00 2001 From: Przemyslaw Walski Date: Wed, 17 Jan 2024 17:58:35 +0100 Subject: [PATCH 17/17] README with warning notice --- conf/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 conf/README.md diff --git a/conf/README.md b/conf/README.md new file mode 100644 index 0000000..50b157d --- /dev/null +++ b/conf/README.md @@ -0,0 +1,7 @@ +# Runtime descriptor + +When changing `name` of the _dummy_ runtime in descriptor update the name in: + +- [gamerhash-facade/Golem.Tools/GolemPackageBuilder.cs](https://github.com/golemfactory/gamerhash-facade/blob/d99ec9399262ab0a2c533a6642ae293935861152/Golem.Tools/GolemPackageBuilder.cs#L81C24-L81C24) + +- [gamerhash-facade/modules.sh](https://github.com/golemfactory/gamerhash-facade/blob/d99ec9399262ab0a2c533a6642ae293935861152/modules.sh#L20)