diff --git a/Cargo.lock b/Cargo.lock index 34a5b836b5..42154f28ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,7 +430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" dependencies = [ "cipher 0.2.5", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -440,7 +440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" dependencies = [ "cipher 0.2.5", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -599,6 +599,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" +[[package]] +name = "arrayref" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" + [[package]] name = "arrayvec" version = "0.5.2" @@ -938,15 +944,26 @@ dependencies = [ ] [[package]] -name = "block-buffer" -version = "0.7.3" +name = "blake2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "block-padding 0.1.5", - "byte-tools", - "byteorder", - "generic-array 0.12.4", + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec 0.7.6", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", + "digest 0.10.7", ] [[package]] @@ -955,8 +972,8 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "block-padding 0.2.1", - "generic-array 0.14.7", + "block-padding", + "generic-array", ] [[package]] @@ -965,7 +982,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -974,19 +991,10 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57a0e8073e8baa88212fb5823574c02ebccb395136ba9a164ab89379ec6072f0" dependencies = [ - "block-padding 0.2.1", + "block-padding", "cipher 0.2.5", ] -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - [[package]] name = "block-padding" version = "0.2.1" @@ -1100,12 +1108,6 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "byte-unit" version = "4.0.19" @@ -1252,7 +1254,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -1373,6 +1375,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -1603,7 +1611,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array", "typenum", ] @@ -1613,7 +1621,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ - "generic-array 0.14.7", + "generic-array", "subtle", ] @@ -1912,22 +1920,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array 0.12.4", -] - [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -2569,12 +2568,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fastrand" version = "2.1.1" @@ -2936,15 +2929,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2983,7 +2967,7 @@ version = "0.4.1" dependencies = [ "actix-rt", "anyhow", - "digest 0.8.1", + "digest 0.10.7", "dotenv", "env_logger 0.7.1", "futures 0.3.30", @@ -2991,7 +2975,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha3 0.8.2", + "sha3 0.10.8", "structopt", "tempdir", "thiserror", @@ -3788,7 +3772,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -4793,12 +4777,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -5907,7 +5885,7 @@ checksum = "2eca4ecc81b7f313189bf73ce724400a07da2a6dac19588b03c8bd76a2dcc251" dependencies = [ "block-buffer 0.9.0", "digest 0.9.0", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -6070,9 +6048,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", @@ -6749,7 +6727,7 @@ dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -6763,18 +6741,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" -dependencies = [ - "block-buffer 0.7.3", - "digest 0.8.1", - "fake-simd", - "opaque-debug 0.2.3", -] - [[package]] name = "sha2" version = "0.9.9" @@ -6785,7 +6751,7 @@ dependencies = [ "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -6799,19 +6765,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha3" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd26bc0e7a2e3a7c959bc494caf58b72ee0c71d67704e9520f736ca7e4853ecf" -dependencies = [ - "block-buffer 0.7.3", - "byte-tools", - "digest 0.8.1", - "keccak", - "opaque-debug 0.2.3", -] - [[package]] name = "sha3" version = "0.9.1" @@ -6821,7 +6774,7 @@ dependencies = [ "block-buffer 0.9.0", "digest 0.9.0", "keccak", - "opaque-debug 0.3.1", + "opaque-debug", ] [[package]] @@ -7164,7 +7117,7 @@ dependencies = [ "futures-core", "futures-io", "futures-util", - "generic-array 0.14.7", + "generic-array", "hex", "hkdf", "hmac 0.12.1", @@ -8186,7 +8139,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "url", "webpki-roots 0.26.5", @@ -9036,7 +8989,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha3 0.8.2", + "sha3 0.10.8", "structopt", "thiserror", "tiny-keccak", @@ -9085,7 +9038,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", - "sha3 0.8.2", + "sha3 0.10.8", "shell-words", "signal-hook", "socket2 0.4.10", @@ -9151,6 +9104,7 @@ dependencies = [ "awc", "bytes 1.7.1", "crossterm 0.26.1", + "digest 0.10.7", "env_logger 0.7.1", "futures 0.3.30", "gftp", @@ -9164,8 +9118,8 @@ dependencies = [ "regex", "secp256k1 0.27.0", "serde", - "sha2 0.8.2", - "sha3 0.8.2", + "sha2 0.10.8", + "sha3 0.10.8", "tempdir", "test-context", "thiserror", @@ -9338,7 +9292,7 @@ dependencies = [ "rustc-hex", "serde_json", "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", - "sha2 0.9.9", + "sha2 0.10.8", "structopt", "test-context", "thiserror", @@ -9422,7 +9376,7 @@ dependencies = [ "derive_more", "diesel", "diesel_migrations", - "digest 0.8.1", + "digest 0.10.7", "env_logger 0.7.1", "futures 0.3.30", "humantime 2.1.0", @@ -9441,7 +9395,7 @@ dependencies = [ "serde_bytes", "serde_json", "serial_test 0.5.1 (git+https://github.com/golemfactory/serial_test.git?branch=actix_rt_test)", - "sha3 0.8.2", + "sha3 0.10.8", "structopt", "strum 0.24.1", "strum_macros 0.24.3", @@ -9654,7 +9608,7 @@ dependencies = [ "num-traits", "r2d2", "serde_json_canonicalizer", - "sha3 0.9.1", + "sha3 0.10.8", "thiserror", "tokio", "ya-client-model", @@ -10145,8 +10099,11 @@ dependencies = [ "async-compression", "async-trait", "awc", + "blake2", + "blake3", "bytes 1.7.1", "crossterm 0.26.1", + "digest 0.10.7", "env_logger 0.7.1", "futures 0.3.30", "gftp", @@ -10162,8 +10119,8 @@ dependencies = [ "secp256k1 0.27.0", "serde", "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", - "sha2 0.8.2", - "sha3 0.8.2", + "sha2 0.10.8", + "sha3 0.10.8", "structopt", "tempdir", "test-case 3.3.1", @@ -10321,6 +10278,7 @@ dependencies = [ "actix-web-actors", "anyhow", "bytes 1.7.1", + "digest 0.10.7", "env_logger 0.7.1", "futures 0.3.30", "hex", @@ -10331,7 +10289,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_json", - "sha3 0.8.2", + "sha3 0.10.8", "structopt", "thiserror", "tokio", diff --git a/core/gftp/Cargo.toml b/core/gftp/Cargo.toml index cdacdc72a7..24df55fecf 100644 --- a/core/gftp/Cargo.toml +++ b/core/gftp/Cargo.toml @@ -24,7 +24,7 @@ ya-service-bus = { workspace = true } actix-rt = "2.7" anyhow = "1.0" -digest = "0.8.1" +digest = "0.10" dotenv = { version = "0.15.0", optional = true } env_logger = { version = "0.7.1", optional = true } futures = "0.3" @@ -32,7 +32,7 @@ log = "0.4" rand = "0.8" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sha3 = "0.8.2" +sha3 = "0.10" structopt = "0.3.9" thiserror = "1.0.20" tokio = { version = "1", features = ["io-std", "time"] } @@ -41,7 +41,6 @@ url = { version = "2.1.1", features = ["serde"] } [dev-dependencies] dotenv = { version = "0.15.0" } env_logger = { version = "0.7.1" } -sha3 = "0.8.2" tempdir = "0.3.7" tokio = { version = "1", features = ["process"] } diff --git a/core/gftp/examples/gftp-server.rs b/core/gftp/examples/gftp-server.rs index cc4be517b3..38c999ef62 100644 --- a/core/gftp/examples/gftp-server.rs +++ b/core/gftp/examples/gftp-server.rs @@ -1,8 +1,7 @@ use anyhow::{anyhow, Result}; +use digest::{Digest, Output}; use futures::future::{FutureExt, LocalBoxFuture}; use gftp::rpc::*; -use sha3::digest::generic_array::GenericArray; -use sha3::Digest; use std::ffi::OsString; use std::fs::OpenOptions; use std::io::Read; @@ -14,7 +13,7 @@ use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::process::{ChildStdin, ChildStdout, Command}; static SEQ: AtomicUsize = AtomicUsize::new(0); -type HashOutput = GenericArray::OutputSize>; +type HashOutput = Output; /// Build the GFTP binary, start the daemon and run: /// @@ -94,12 +93,12 @@ fn hash_file(path: &Path) -> Result { let mut chunk = vec![0; 4096]; while let Ok(count) = file_src.read(&mut chunk[..]) { - hasher.input(&chunk[..count]); + hasher.update(&chunk[..count]); if count != 4096 { break; } } - Ok(hasher.result()) + Ok(hasher.finalize()) } #[actix_rt::main] diff --git a/core/gftp/src/gftp.rs b/core/gftp/src/gftp.rs index f881df924e..45fa69428e 100644 --- a/core/gftp/src/gftp.rs +++ b/core/gftp/src/gftp.rs @@ -323,7 +323,7 @@ fn hash_file_sha256(mut file: &mut fs::File) -> Result { .with_context(|| "Can't seek file at offset 0.".to_string())?; io::copy(&mut file, &mut hasher)?; - Ok(format!("{:x}", hasher.result())) + Ok(format!("{:x}", hasher.finalize())) } /// Returns NodeId and file hash from gftp url. diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 10238f7299..ce57f96350 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -32,7 +32,7 @@ r2d2 = "0.8.8" rand = "0.8" rpassword = "3.0.2" serde_json = "1.0" -sha2 = "0.9.1" +sha2 = "0.10" structopt = "0.3" thiserror = "1.0" tokio = { version = "1", features = ["fs", "io-std", "signal", "io-util"] } diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 8036bd3c3d..c74d90442f 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -37,7 +37,7 @@ chrono = { version = "0.4", features = ["serde"] } derive_more = { workspace = true } diesel = { version = "1.4", features = ["chrono", "sqlite", "r2d2"] } diesel_migrations = "1.4" -digest = "0.8.1" +digest = "0.10" env_logger = { version = "0.7" } futures = "0.3" humantime = "2" @@ -55,7 +55,7 @@ regex = "1.4.2" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11.14" serde_json = "1.0" -sha3 = "0.8.2" +sha3 = "0.10" structopt = "0.3" strum = { workspace = true } strum_macros = "0.24" diff --git a/core/market/src/db/model/proposal_id.rs b/core/market/src/db/model/proposal_id.rs index 8710f4ace8..207d74b9e1 100644 --- a/core/market/src/db/model/proposal_id.rs +++ b/core/market/src/db/model/proposal_id.rs @@ -137,11 +137,11 @@ pub fn hash_proposal( ) -> String { let mut hasher = Sha3_256::new(); - hasher.input(offer_id.to_string()); - hasher.input(demand_id.to_string()); - hasher.input(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string()); + hasher.update(offer_id.to_string()); + hasher.update(demand_id.to_string()); + hasher.update(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string()); - format!("{:x}", hasher.result()) + format!("{:x}", hasher.finalize()) } impl FromStr for ProposalId { diff --git a/core/market/src/db/model/subscription_id.rs b/core/market/src/db/model/subscription_id.rs index 849be0c7cb..39c4575d1b 100644 --- a/core/market/src/db/model/subscription_id.rs +++ b/core/market/src/db/model/subscription_id.rs @@ -84,15 +84,15 @@ pub fn hash( ) -> String { let mut hasher = Sha3_256::new(); - hasher.input(properties); - hasher.input(constraints); - hasher.input(node_id); + hasher.update(properties); + hasher.update(constraints); + hasher.update(node_id); // We can't change format freely, because it is important to compute hash. // Is there any other solution, to compute hash, that is format independent? - hasher.input(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string()); - hasher.input(expiration_ts.format("%Y-%m-%d %H:%M:%f").to_string()); + hasher.update(creation_ts.format("%Y-%m-%d %H:%M:%f").to_string()); + hasher.update(expiration_ts.format("%Y-%m-%d %H:%M:%f").to_string()); - format!("{:x}", hasher.result()) + format!("{:x}", hasher.finalize()) } impl FromStr for SubscriptionId { diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 8d470c42ab..bce3a305ae 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -24,7 +24,7 @@ num-bigint = { version = "0.3", features = ["serde"] } num-traits = "0.2" num-derive = "0.3" r2d2 = "0.8" -sha3 = "0.9" +sha3 = "0.10" thiserror = "1.0" tokio = { version = "1", features = ["macros"] } diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index 94563eefe1..15440bca45 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -28,7 +28,7 @@ num-traits = "0.2" rlp = "0.5" serde = "1.0" serde_json = "^1.0" -sha3 = "0.8" +sha3 = "0.10" thiserror = "1.0" tiny-keccak = { version = "2.0", features = ["keccak"] } tokio = { version = "1", features = ["full"] } diff --git a/core/payment/src/timeout_lock.rs b/core/payment/src/timeout_lock.rs index aea6b841b6..fd96e5b5f5 100644 --- a/core/payment/src/timeout_lock.rs +++ b/core/payment/src/timeout_lock.rs @@ -106,6 +106,7 @@ impl TimedMutex { match tokio::time::timeout(Duration::from_secs(10), receiver.recv()).await { Err(_) => { log::warn!("[TimedMutex] Long running task: {task_name}!"); + counter += 1; } Ok(None) => log::warn!("[TimedMutex] Unexpected mpsc close."), Ok(Some(TimedMutexTaskMessage::Finish)) => break, diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index ac2ffa89ef..f4f217ad66 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -46,7 +46,8 @@ default = [] [dev-dependencies] ya-client = { workspace = true } actix-rt = "2.7" -sha3 = "0.8.2" +digest = "0.10" +sha3 = "0.10" structopt = "0.3" tokio = { version = "1", features = ["time", "fs"] } url = "2.1" diff --git a/core/vpn/examples/ws.rs b/core/vpn/examples/ws.rs index b19be96b06..0ee891d015 100644 --- a/core/vpn/examples/ws.rs +++ b/core/vpn/examples/ws.rs @@ -2,19 +2,19 @@ use actix::prelude::*; use actix_web_actors::ws; use actix_web_actors::ws::Frame; use bytes::Bytes; +use digest::{Digest, Output}; use futures::{SinkExt, StreamExt}; -use sha3::digest::generic_array::GenericArray; -use sha3::Digest; use std::path::{Path, PathBuf}; use structopt::StructOpt; use tokio::fs::OpenOptions; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use url::Url; + use ya_client::net::NetVpnApi; use ya_client::web::WebClient; use ya_client_model::net::{Address, NewNetwork, Node}; -type HashOutput = GenericArray::OutputSize>; +type HashOutput = Output; #[derive(StructOpt, Clone, Debug)] struct Cli { @@ -50,10 +50,10 @@ async fn file_checksum>(path: P) -> anyhow::Result { if count == 0 { break; } - hasher.input(&chunk[..count]); + hasher.update(&chunk[..count]); } - Ok(hasher.result()) + Ok(hasher.finalize()) } #[actix_rt::main] diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index b47daadeae..eb89c0edfe 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -73,7 +73,7 @@ secp256k1 = { version = "0.27.0", optional = true } serde = { version = "^1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.8" -sha3 = "0.8.2" +sha3 = "0.10" signal-hook = "0.3" socket2 = "0.4" structopt = "0.3" @@ -104,7 +104,7 @@ actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" -sha3 = "0.8.2" +sha3 = "0.10" shell-words = "1.0.0" tempdir = "0.3.7" diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 1c09e3156b..f33a27f60b 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -22,6 +22,8 @@ actix-rt = "2.7" anyhow = "1.0" # async-compression 0.3.8+ deprecates the "stream" module async-compression = { version = "=0.3.7", features = ["tokio", "futures-io", "stream", "bzip2", "gzip", "xz"] } +blake2 = "0.10" +blake3 = { version = "1.5", features = ["traits-preview"] } bytes = "1.0" futures = "0.3.4" globset = "0.4.5" @@ -34,7 +36,8 @@ rand = "0.8" regex = "1.3.4" reqwest = { version = "0.11", optional = true } serde = "1.0.104" -sha3 = "0.8.2" +sha2 = "0.10" +sha3 = "0.10" tempdir = "0.3.7" thiserror = "1.0.11" tokio = { version = "1", features = ["fs", "io-util"] } @@ -73,10 +76,10 @@ actix-web = "4" actix-rt = "2.9.0" anyhow = "1.0" crossterm = "0.26.1" +digest = "0.10" env_logger = "0.7" secp256k1 = { version = "0.27.0" } serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } -sha2 = "0.8.1" structopt = "0.3.15" test-context = "0.1.4" test-case = "3" diff --git a/exe-unit/components/transfer/examples/gftp.rs b/exe-unit/components/transfer/examples/gftp.rs index 94a7e5b939..98ffba8231 100644 --- a/exe-unit/components/transfer/examples/gftp.rs +++ b/exe-unit/components/transfer/examples/gftp.rs @@ -1,7 +1,6 @@ use crossterm::{cursor, terminal, ExecutableCommand, QueueableCommand}; +use digest::{Digest, Output}; use rand::RngCore; -use sha3::digest::generic_array::GenericArray; -use sha3::Digest; use std::env; use std::fs::OpenOptions; use std::io::{Read, Write}; @@ -14,7 +13,7 @@ use ya_transfer::{ transfer, FileTransferProvider, GftpTransferProvider, TransferContext, TransferProvider, }; -type HashOutput = GenericArray::OutputSize>; +type HashOutput = Output; fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) -> HashOutput { let path = path.join(name); @@ -32,11 +31,11 @@ fn create_file(path: &Path, name: &str, chunk_size: usize, chunk_count: usize) - for _ in 0..chunk_count { rng.fill_bytes(&mut input); - hasher.input(&input); + hasher.update(&input); file_src.write_all(&input).unwrap(); } file_src.flush().unwrap(); - hasher.result() + hasher.finalize() } fn hash_file(path: &Path) -> HashOutput { @@ -46,12 +45,12 @@ fn hash_file(path: &Path) -> HashOutput { let mut chunk = vec![0; 4096]; while let Ok(count) = file_src.read(&mut chunk[..]) { - hasher.input(&chunk[..count]); + hasher.update(&chunk[..count]); if count != 4096 { break; } } - hasher.result() + hasher.finalize() } // processing progress updates must not panic or the transfer will be aborted diff --git a/exe-unit/components/transfer/src/cache.rs b/exe-unit/components/transfer/src/cache.rs index 956d54131b..15e1705531 100644 --- a/exe-unit/components/transfer/src/cache.rs +++ b/exe-unit/components/transfer/src/cache.rs @@ -147,9 +147,9 @@ impl CachePath { /// Creates the long version of path, including hash and the "random" token. pub fn temp_path(&self) -> PathBuf { let mut digest = sha3::Sha3_224::default(); - digest.input(&self.hash); - digest.input(&self.nonce); - let hash = digest.result(); + digest.update(&self.hash); + digest.update(&self.nonce); + let hash = digest.finalize(); PathBuf::from(hex::encode(hash)) } diff --git a/exe-unit/components/transfer/src/gftp.rs b/exe-unit/components/transfer/src/gftp.rs index 9292ea20ae..150c42713a 100644 --- a/exe-unit/components/transfer/src/gftp.rs +++ b/exe-unit/components/transfer/src/gftp.rs @@ -1,6 +1,7 @@ use crate::error::Error; use crate::{abortable_sink, abortable_stream}; use crate::{TransferContext, TransferData, TransferProvider, TransferSink, TransferStream}; + use bytes::Bytes; use futures::channel::mpsc; use futures::future::{ready, try_select, Either}; @@ -9,6 +10,7 @@ use gftp::DEFAULT_CHUNK_SIZE; use sha3::{Digest, Sha3_256}; use tokio::task::spawn_local; use url::Url; + use ya_core_model::gftp as model; use ya_core_model::gftp::Error as GftpError; use ya_core_model::gftp::GftpChunk; @@ -109,12 +111,12 @@ impl TransferProvider for GftpTransferProvider { }; offset += chunk.content.len(); - digest.input(&chunk.content); + digest.update(&chunk.content); chunk_tx.send(Ok::<_, Error>(chunk)).await?; } } - Ok::<_, Error>(digest.result()) + Ok::<_, Error>(digest.finalize()) }; let send_fut = chunk_rx.try_for_each_concurrent(concurrency, |chunk| async { diff --git a/exe-unit/components/transfer/src/hash.rs b/exe-unit/components/transfer/src/hash.rs index ad350354d9..df645f5f00 100644 --- a/exe-unit/components/transfer/src/hash.rs +++ b/exe-unit/components/transfer/src/hash.rs @@ -63,17 +63,39 @@ where 512 => Box::::default(), len => { return Err(Error::UnsupportedDigestError(format!( - "Unsupported digest {} of length {}: {}", - alg, - len, - hex::encode(&hash), + "Unsupported digest {alg} of length {len}: {}", + hex::encode(&hash) ))) } }, + "sha2" => match hash.len() * 8 { + 224 => Box::::default(), + 256 => Box::::default(), + 384 => Box::::default(), + 512 => Box::::default(), + len => { + return Err(Error::UnsupportedDigestError(format!( + "Unsupported digest {alg} of length {len}: {}", + hex::encode(&hash) + ))) + } + }, + "blake2b" => Box::::default(), + "blake2s" => Box::::default(), + "blake2" => match hash.len() * 8 { + 256 => Box::::default(), + 512 => Box::::default(), + len => { + return Err(Error::UnsupportedDigestError(format!( + "Unsupported digest {alg} of length {len}: {}", + hex::encode(&hash) + ))) + } + }, + "blake3" => Box::::default(), _ => { return Err(Error::UnsupportedDigestError(format!( - "Unsupported digest: {}", - alg + "Unsupported digest: {alg}" ))) } }; @@ -96,7 +118,7 @@ where let mut chunk = vec![0; 4096]; while let Ok(count) = file_src.read(&mut chunk[..]) { - self.hasher.input(&chunk[..count]); + self.hasher.update(&chunk[..count]); if count != 4096 { break; } @@ -119,14 +141,14 @@ where match opt { Some(item) => { if let Ok(data) = item { - self.hasher.input(data.as_ref()); + self.hasher.update(data.as_ref()); } } None => { let result = match &self.result { Some(r) => r, None => { - self.result = Some(self.hasher.result_reset().to_vec()); + self.result = Some(self.hasher.finalize_reset().to_vec()); self.result.as_ref().unwrap() } }; diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 269d9bbc7d..824dd2c12a 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -1,23 +1,24 @@ use actix::Actor; +use digest::Digest; use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::{generate_file_with_hash, generate_file_with_hasher}; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; -/// When re-deploying image, `TransferService` should uses partially downloaded image. +/// When re-deploying image, `TransferService` should use partially downloaded image. /// Hash computations should be correct in both cases. #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); @@ -81,3 +82,88 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re Ok(()) } + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_deploy_checksum(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("deploy-checksum")?; + let temp_dir = dir.path(); + + log::debug!("Creating directories in: {}", temp_dir.display()); + let work_dir = temp_dir.join("work_dir"); + let cache_dir = temp_dir.join("cache_dir"); + let sub_dir = temp_dir.join("sub_dir"); + + for dir in [work_dir.clone(), cache_dir.clone(), sub_dir.clone()] { + std::fs::create_dir_all(dir)?; + } + + log::debug!("Generating example images using different hash functions"); + + fn generate(temp_dir: &std::path::Path, name: &str) -> String { + let chunk_size = 4096_usize; + let chunk_count = 10; + + hex::encode(generate_file_with_hasher::( + temp_dir, + name, + chunk_size, + chunk_count, + )) + } + + let images = vec![ + ("sha3-256", generate::(temp_dir, "sha3-256")), + ("sha3-512", generate::(temp_dir, "sha3-512")), + ("sha3-224", generate::(temp_dir, "sha3-224")), + ("sha3-384", generate::(temp_dir, "sha3-384")), + ("sha2-256", generate::(temp_dir, "sha2-256")), + ("sha2-512", generate::(temp_dir, "sha2-512")), + ("sha2-224", generate::(temp_dir, "sha2-224")), + ("sha2-384", generate::(temp_dir, "sha2-384")), + ( + "blake2-512", + generate::(temp_dir, "blake2-512"), + ), + ( + "blake2-256", + generate::(temp_dir, "blake2-256"), + ), + ("blake3", generate::(temp_dir, "blake3")), + ]; + + log::debug!("Starting HTTP servers"); + let path = temp_dir.to_path_buf(); + start_http(ctx, path) + .await + .expect("unable to start http servers"); + + log::debug!("Starting TransferService"); + let exe_ctx = TransferServiceContext { + work_dir: work_dir.clone(), + cache_dir, + ..TransferServiceContext::default() + }; + let addr = TransferService::new(exe_ctx).start(); + + log::info!("[>>] Deployment with hash verification"); + for (name, hash) in images { + log::info!("[>>] Verifying deploy with {name}"); + + let hash_function = name.split('-').next().unwrap(); + let deploy = DeployImage { + task_package: Some(format!( + "hash://{hash_function}:{hash}:http://127.0.0.1:8001/{name}" + )), + }; + let result = addr.send(deploy).await; + let path = result.unwrap().unwrap().unwrap(); + + assert!(path.exists()); + } + + Ok(()) +} diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 4e635249b4..b112d93858 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -1,5 +1,6 @@ use actix::{Actor, Addr}; use futures::future::LocalBoxFuture; +use sha3::Sha3_512; use std::io::ErrorKind; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -173,7 +174,7 @@ async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result< transfer(&addr, "http://127.0.0.1:8001/rnd", "container:/input/rnd-1"), ) .await??; - verify_hash(&hash, work_dir.join("vol-1"), "rnd-1"); + verify_hash::(&hash, work_dir.join("vol-1"), "rnd-1"); log::warn!("Checksum verified"); Ok(()) diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index faa4b136d4..894a0eeba5 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -1,4 +1,5 @@ use actix::{Actor, Addr}; +use sha3::Sha3_512; use std::env; use test_context::test_context; @@ -113,7 +114,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu println!(); log::warn!("[>>] Transfer HTTP -> container"); transfer(&addr, "http://127.0.0.1:8001/rnd", "container:/input/rnd-1").await?; - verify_hash(&hash, work_dir.join("vol-1"), "rnd-1"); + verify_hash::(&hash, work_dir.join("vol-1"), "rnd-1"); log::warn!("Checksum verified"); println!(); @@ -124,7 +125,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu "http://127.0.0.1:8002/rnd-2", ) .await?; - verify_hash(&hash, temp_dir, "rnd-2"); + verify_hash::(&hash, temp_dir, "rnd-2"); log::warn!("Checksum verified"); println!(); @@ -135,19 +136,19 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu "http://127.0.0.1:8002/rnd-3", ) .await?; - verify_hash(&hash, temp_dir, "rnd-3"); + verify_hash::(&hash, temp_dir, "rnd-3"); log::warn!("Checksum verified"); println!(); log::warn!("[>>] Transfer container -> container"); transfer(&addr, "container:/input/rnd-1", "container:/input/rnd-4").await?; - verify_hash(&hash, work_dir.join("vol-1"), "rnd-4"); + verify_hash::(&hash, work_dir.join("vol-1"), "rnd-4"); log::warn!("Checksum verified"); println!(); log::warn!("[>>] Transfer container -> container (different volume)"); transfer(&addr, "container:/input/rnd-1", "container:/output/rnd-5").await?; - verify_hash(&hash, work_dir.join("vol-2"), "rnd-5"); + verify_hash::(&hash, work_dir.join("vol-2"), "rnd-5"); log::warn!("Checksum verified"); Ok(()) @@ -257,8 +258,8 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul ) .await?; log::warn!("Extraction complete"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-1"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-4"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-1"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-4"); log::warn!("Checksum verified"); log::warn!("Removing extracted files"); @@ -279,8 +280,8 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul ) .await?; log::warn!("Extraction complete"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-1"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-4"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-1"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-4"); log::warn!("Checksum verified"); log::warn!("Removing extracted files"); @@ -296,8 +297,8 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul // args.fileset = Some(FileSet::Pattern(SetEntry::Single("**/rnd-*".into()))); transfer_with_args(&addr, "container:/input", "container:/extract", args).await?; log::warn!("Transfer complete"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-1"); - verify_hash(&hash, work_dir.join("vol-3"), "rnd-4"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-1"); + verify_hash::(&hash, work_dir.join("vol-3"), "rnd-4"); log::warn!("Checksum verified"); transfer( diff --git a/exe-unit/src/handlers/local.rs b/exe-unit/src/handlers/local.rs index b737234415..2437a0851f 100644 --- a/exe-unit/src/handlers/local.rs +++ b/exe-unit/src/handlers/local.rs @@ -4,6 +4,7 @@ use crate::runtime::Runtime; use crate::service::ServiceAddr; use crate::state::State; use crate::{report, ExeUnit}; + use actix::prelude::*; use futures::FutureExt; @@ -160,10 +161,10 @@ impl Handler for ExeUnit { log::debug!("IAS report: {}", &evidence.report); let mut hasher = Sha3_256::new(); - hasher.input(task_package.as_bytes()); + hasher.update(task_package.as_bytes()); let mut payload_hash = [0u8; 32]; - payload_hash.copy_from_slice(hasher.result().as_ref()); + payload_hash.copy_from_slice(hasher.finalize().as_ref()); Some(Credentials::Sgx { requestor: crypto.requestor_pub_key.serialize().to_vec(), diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 1cb3f35556..9c9f370bd8 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -36,6 +36,7 @@ async-compression = { version = "=0.3.7", features = [ ] } bytes = "1.0" crossterm = "0.26.1" +digest = "0.10" env_logger = "0.7" futures = "0.3.4" globset = "0.4.5" @@ -48,8 +49,8 @@ rand = "0.8" regex = "1.3.4" secp256k1 = { version = "0.27.0" } serde = "1.0.104" -sha2 = "0.8.1" -sha3 = "0.8.2" +sha2 = "0.10" +sha3 = "0.10" tempdir = "0.3.7" test-context = "0.1.4" thiserror = "1.0.11" diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index 8d05e872e6..2e46def341 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -1,24 +1,22 @@ +use digest::{Digest, Output}; use rand::Rng; -use sha2::Digest; use std::fs::OpenOptions; use std::io::Write; use std::path::{Path, PathBuf}; -use crate::hash::HashOutput; - -pub fn generate_file_with_hash( +pub fn generate_file_with_hasher( path: &Path, name: &str, chunk_size: usize, chunk_count: usize, -) -> HashOutput { +) -> Output { let path = path.join(name); log::debug!( "Creating a random file {} of size {chunk_size} * {chunk_count}", path.display() ); - let mut hasher = sha3::Sha3_512::default(); + let mut hasher = H::new(); let mut file_src = OpenOptions::new() .write(true) .create(true) @@ -39,11 +37,20 @@ pub fn generate_file_with_hash( .map(|_| rng.gen_range(0..256) as u8) .collect(); - hasher.input(&input); + hasher.update(&input); let _ = file_src.write(&input).unwrap(); } file_src.flush().unwrap(); - hasher.result() + hasher.finalize() +} + +pub fn generate_file_with_hash( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, +) -> Output { + generate_file_with_hasher::(path, name, chunk_size, chunk_count) } pub fn generate_file(path: &PathBuf, chunk_size: usize, chunk_count: usize) { diff --git a/test-utils/test-framework/framework-basic/src/hash.rs b/test-utils/test-framework/framework-basic/src/hash.rs index 41fcd3c521..b35892b8b1 100644 --- a/test-utils/test-framework/framework-basic/src/hash.rs +++ b/test-utils/test-framework/framework-basic/src/hash.rs @@ -1,28 +1,28 @@ -use sha2::digest::generic_array::GenericArray; -use sha2::Digest; +use digest::{Digest, Output}; use std::fs::OpenOptions; use std::io::Read; use std::path::Path; -pub type HashOutput = GenericArray::OutputSize>; - -pub fn hash_file(path: &Path) -> HashOutput { +pub fn hash_file(path: &Path) -> Output { let mut file_src = OpenOptions::new().read(true).open(path).expect("rnd file"); - let mut hasher = sha3::Sha3_512::default(); + let mut hasher = H::new(); let mut chunk = vec![0; 4096]; while let Ok(count) = file_src.read(&mut chunk[..]) { - hasher.input(&chunk[..count]); + hasher.update(&chunk[..count]); if count != 4096 { break; } } - hasher.result() + hasher.finalize() } -pub fn verify_hash, P: AsRef>(hash: &HashOutput, path: P, file_name: S) { +pub fn verify_hash(hash: &Output, path: impl AsRef, file_name: impl AsRef) +where + H: Digest, +{ let path = path.as_ref().join(file_name.as_ref()); log::info!("Verifying hash of {:?}", path); - assert_eq!(hash, &hash_file(&path)); + assert_eq!(hash, &hash_file::(&path)); }