From deba819d2df79a536fb65cf0d26a13ce84cd40d7 Mon Sep 17 00:00:00 2001 From: Kirill Pimenov Date: Mon, 19 Feb 2018 18:09:30 +0100 Subject: [PATCH 01/15] Very primitive test of the Client API --- util/fetch/src/client.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index bcd534cb34f..36874a82f1d 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -358,3 +358,17 @@ impl io::Read for Response { res } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn it_should_fetch() { + let fetch = Client::new().unwrap(); + let ya = fetch.fetch("https://ya.ru").wait(); + assert!(ya.is_ok()); + let ya_response = ya.unwrap(); + assert!(ya_response.is_success()); + } +} \ No newline at end of file From 17f45c8177ddf8e21e67886a59495721d78665ad Mon Sep 17 00:00:00 2001 From: Kirill Pimenov Date: Tue, 27 Feb 2018 12:02:39 +0100 Subject: [PATCH 02/15] [WIP] getting rid of request --- Cargo.lock | 534 +++++++++++++++++---------------------- util/fetch/Cargo.toml | 7 +- util/fetch/src/client.rs | 106 +++++--- util/fetch/src/lib.rs | 8 +- 4 files changed, 313 insertions(+), 342 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bab06de5923..3f65ca4a403 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,8 +1,3 @@ -[[package]] -name = "adler32" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "aho-corasick" version = "0.6.4" @@ -70,7 +65,7 @@ name = "backtrace-sys" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -93,6 +88,15 @@ dependencies = [ "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bigint" version = "4.2.0" @@ -181,10 +185,10 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -265,23 +269,6 @@ dependencies = [ "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "core-foundation" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "core-foundation-sys" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam" version = "0.3.2" @@ -292,6 +279,14 @@ name = "crunchy" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "ct-logs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ctrlc" version = "1.1.1" @@ -369,7 +364,7 @@ name = "env_logger" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -387,7 +382,7 @@ version = "0.5.7" source = "git+https://github.com/paritytech/rust-secp256k1#c1fc9daedee67e1b4028ad1b3669275ed49c22cf" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -430,9 +425,9 @@ dependencies = [ "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -484,7 +479,7 @@ dependencies = [ "kvdb-memorydb 0.1.0", "kvdb-rocksdb 0.1.0", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", "memory-cache 0.1.0", @@ -493,7 +488,7 @@ dependencies = [ "num 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-machine 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0", "price-info 1.11.0", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -539,9 +534,9 @@ name = "ethcore-io" version = "1.11.0" dependencies = [ "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -566,7 +561,7 @@ dependencies = [ "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", "memorydb 0.1.1", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0", "plain_hasher 0.1.0", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -591,8 +586,8 @@ dependencies = [ "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "isatty 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -621,8 +616,8 @@ dependencies = [ "hyper 0.10.0-a.0 (git+https://github.com/paritytech/hyper)", "keccak-hash 0.1.0", "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "table 0.1.0", "transient-hashmap 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -660,9 +655,9 @@ dependencies = [ "ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "path 0.1.0", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -700,8 +695,8 @@ dependencies = [ "kvdb 0.1.0", "kvdb-rocksdb 0.1.0", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -740,8 +735,8 @@ dependencies = [ "jsonrpc-macros 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-tcp-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -815,7 +810,7 @@ dependencies = [ "eth-secp256k1 0.5.7 (git+https://github.com/paritytech/rust-secp256k1)", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wordlist 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", @@ -848,9 +843,9 @@ dependencies = [ "ethkey 0.3.0", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wordlist 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -872,7 +867,7 @@ dependencies = [ "ethstore 0.2.0", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -897,9 +892,9 @@ dependencies = [ "keccak-hash 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.1.0", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -919,9 +914,9 @@ dependencies = [ "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", ] @@ -966,9 +961,11 @@ version = "0.1.0" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "reqwest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -996,19 +993,6 @@ name = "fnv" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -1062,7 +1046,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1080,8 +1064,8 @@ dependencies = [ "ethkey 0.3.0", "hidapi 0.3.1 (git+https://github.com/paritytech/hidapi-rs)", "libusb 0.3.0 (git+https://github.com/paritytech/libusb-rs)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "protobuf 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "trezor-sys 1.0.0 (git+https://github.com/paritytech/trezor-sys)", @@ -1121,7 +1105,7 @@ name = "hidapi" version = "0.3.1" source = "git+https://github.com/paritytech/hidapi-rs#e77ea09c98f29ea8855dd9cd9461125a28ca9125" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1138,7 +1122,7 @@ dependencies = [ "cookie 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "rotor 0.6.3 (git+https://github.com/tailhook/rotor)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1157,7 +1141,7 @@ dependencies = [ "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1178,7 +1162,7 @@ dependencies = [ "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1190,17 +1174,21 @@ dependencies = [ ] [[package]] -name = "hyper-tls" -version = "0.1.2" +name = "hyper-rustls" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "ct-logs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "webpki-roots 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1284,9 +1272,9 @@ dependencies = [ "keccak-hash 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memorydb 0.1.1", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.1.0", "rlp 0.2.1", "util-error 0.1.0", @@ -1298,7 +1286,7 @@ version = "8.0.1" source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10#786f3e16ba6192a42955eddf57e4baa2a57dde3b" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1312,7 +1300,7 @@ dependencies = [ "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-server-utils 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1324,7 +1312,7 @@ source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10#786f3 dependencies = [ "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-server-utils 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-tokio-ipc 0.1.5 (git+https://github.com/nikvolf/parity-tokio-ipc)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1345,8 +1333,8 @@ version = "8.0.0" source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10#786f3e16ba6192a42955eddf57e4baa2a57dde3b" dependencies = [ "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1357,7 +1345,7 @@ dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "globset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1369,8 +1357,8 @@ source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10#786f3 dependencies = [ "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-server-utils 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1382,8 +1370,8 @@ dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-server-utils 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)", ] @@ -1392,7 +1380,7 @@ dependencies = [ name = "keccak-hash" version = "0.1.0" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1421,7 +1409,7 @@ name = "kvdb-memorydb" version = "0.1.0" dependencies = [ "kvdb 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1432,9 +1420,9 @@ dependencies = [ "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1447,7 +1435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazy_static" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1465,15 +1453,6 @@ name = "libc" version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "libflate" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "libusb" version = "0.3.0" @@ -1489,7 +1468,7 @@ name = "libusb-sys" version = "0.2.4" source = "git+https://github.com/paritytech/libusb-sys#14bdb698003731b6344a79e1d814704e44363e7c" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1515,8 +1494,19 @@ dependencies = [ [[package]] name = "log" -version = "0.3.8" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "lru-cache" @@ -1603,7 +1593,7 @@ dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0", "kvdb-rocksdb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1613,7 +1603,7 @@ name = "mime" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1640,7 +1630,7 @@ name = "miniz-sys" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1653,7 +1643,7 @@ dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "magenta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "magenta-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1668,7 +1658,7 @@ version = "0.1.4" source = "git+https://github.com/alexcrichton/mio-named-pipes#9c1bbb985b74374d3b7eda76937279f8e977ef81" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1721,20 +1711,6 @@ dependencies = [ "tiny-keccak 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "native-tls" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.9.23 (registry+https://github.com/rust-lang/crates.io-index)", - "schannel 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "net2" version = "0.2.31" @@ -1760,9 +1736,9 @@ dependencies = [ "ethcore-network-devp2p 1.11.0", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1771,10 +1747,10 @@ version = "0.1.0" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "ntp 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-reactor 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1794,7 +1770,7 @@ dependencies = [ "conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1889,29 +1865,6 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "openssl" -version = "0.9.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.23 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys" -version = "0.9.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "vcpkg 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "order-stat" version = "0.1.3" @@ -1978,7 +1931,7 @@ dependencies = [ "keccak-hash 0.1.0", "kvdb 0.1.0", "kvdb-rocksdb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "migration 0.1.0", "node-filter 1.11.0", "node-health 0.1.0", @@ -1995,7 +1948,7 @@ dependencies = [ "parity-updater 1.11.0", "parity-version 1.11.0", "parity-whisper 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "path 0.1.0", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2035,7 +1988,7 @@ dependencies = [ "jsonrpc-http-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "keccak-hash 0.1.0", "linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)", "node-health 0.1.0", "parity-dapps-glue 1.9.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2043,7 +1996,7 @@ dependencies = [ "parity-reactor 0.1.0", "parity-ui 1.11.0", "parity-version 1.11.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2094,11 +2047,11 @@ dependencies = [ "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-reactor 0.1.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2129,7 +2082,7 @@ dependencies = [ "ethkey 0.3.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2186,7 +2139,7 @@ dependencies = [ "jsonrpc-ws-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "keccak-hash 0.1.0", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", "multihash 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "node-health 0.1.0", @@ -2194,7 +2147,7 @@ dependencies = [ "parity-reactor 0.1.0", "parity-updater 1.11.0", "parity-version 1.11.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2222,10 +2175,10 @@ dependencies = [ "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-ws-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-rpc 1.11.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2238,7 +2191,7 @@ source = "git+https://github.com/nikvolf/parity-tokio-ipc#d6c5b3cfcc913a1b9cf0f0 dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mio-named-pipes 0.1.4 (git+https://github.com/alexcrichton/mio-named-pipes)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2305,7 +2258,7 @@ dependencies = [ "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.11.0", "parity-version 1.11.0", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "path 0.1.0", "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2329,8 +2282,8 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2347,9 +2300,9 @@ dependencies = [ "jsonrpc-core 8.0.1 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-macros 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", "jsonrpc-pubsub 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -2368,13 +2321,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "parking_lot" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2407,7 +2360,7 @@ dependencies = [ "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.1.1", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memorydb 0.1.1", "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -2455,11 +2408,6 @@ dependencies = [ "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "pkg-config" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "plain_hasher" version = "0.1.0" @@ -2487,8 +2435,8 @@ version = "1.11.0" dependencies = [ "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2638,13 +2586,22 @@ dependencies = [ "rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rayon-core" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2684,38 +2641,27 @@ dependencies = [ ] [[package]] -name = "reqwest" -version = "0.8.1" +name = "ring" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libflate 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 2.0.0-alpha.2 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "ring" -version = "0.12.1" +version = "0.13.0-alpha" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2761,7 +2707,7 @@ name = "rocksdb-sys" version = "0.3.0" source = "git+https://github.com/paritytech/rust-rocksdb#ecf06adf3148ab10f6f7686b724498382ff4f36e" dependencies = [ - "cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)", @@ -2772,7 +2718,7 @@ name = "rotor" version = "0.6.3" source = "git+https://github.com/tailhook/rotor#80ce2e4cd82fdc7f88bb2d737407fa5106799790" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2842,18 +2788,22 @@ dependencies = [ ] [[package]] -name = "safemem" -version = "0.2.0" +name = "rustls" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] -name = "schannel" -version = "0.1.10" +name = "safemem" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "scoped-tls" @@ -2866,23 +2816,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "security-framework" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "security-framework-sys" -version = "0.1.16" +name = "sct" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2942,17 +2881,6 @@ dependencies = [ "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "serde_urlencoded" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "sha1" version = "0.2.0" @@ -3041,7 +2969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "stats" version = "0.1.0" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3106,7 +3034,7 @@ version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "syntex_errors 0.58.1 (registry+https://github.com/rust-lang/crates.io-index)", "syntex_pos 0.58.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3167,7 +3095,7 @@ name = "thread_local" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3206,7 +3134,7 @@ dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3220,7 +3148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3241,7 +3169,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3253,31 +3181,32 @@ dependencies = [ ] [[package]] -name = "tokio-service" -version = "0.1.0" +name = "tokio-rustls" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "tokio-timer" -version = "0.1.2" +name = "tokio-service" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "tokio-tls" -version = "0.1.3" +name = "tokio-timer" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3289,7 +3218,7 @@ dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3308,7 +3237,7 @@ dependencies = [ name = "trace-time" version = "0.1.0" dependencies = [ - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3323,7 +3252,7 @@ version = "1.11.0" dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3453,6 +3382,11 @@ name = "untrusted" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "untrusted" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "url" version = "1.5.1" @@ -3483,19 +3417,6 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "uuid" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "vcpkg" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "vec_map" version = "0.8.0" @@ -3534,7 +3455,7 @@ dependencies = [ "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethjson 0.1.0", "keccak-hash 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0", "rlp 0.2.1", ] @@ -3552,7 +3473,7 @@ dependencies = [ "ethcore-logger 1.11.0", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wasm 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", "wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)", @@ -3568,8 +3489,8 @@ dependencies = [ "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wasm 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3583,6 +3504,24 @@ dependencies = [ "parity-wasm 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "webpki" +version = "0.18.0-alpha" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "webpki-roots" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.2.8" @@ -3620,7 +3559,7 @@ dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", "sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3670,7 +3609,6 @@ dependencies = [ ] [metadata] -"checksum adler32 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6cbd0b9af8587c72beadc9f72d35b9fbb070982c9e6203e46e93f10df25f8f45" "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b3568b48b7cefa6b8ce125f9bb4989e52fbcc29ebea88df04cc7c5f12f70455" "checksum app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7d1c0d48a81bbb13043847f957971f4d87c81542d80ece5e84ba3cba4058fd4" @@ -3682,6 +3620,7 @@ dependencies = [ "checksum base-x 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2f59103b47307f76e03bef1633aec7fa9e29bfb5aa6daf5a334f94233c71f6c1" "checksum base32 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b9605ba46d61df0410d8ac686b0007add8172eba90e8e909c347856fe794d8c" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" +"checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" "checksum bigint 4.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5442186ef6560f30f1ee4b9c1e4c87a35a6879d3644550cc248ec2b955eb5fcd" "checksum bincode 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e103c8b299b28a9c6990458b7013dc4a8356a9b854c51b9883241f5866fac36e" "checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" @@ -3693,17 +3632,16 @@ dependencies = [ "checksum bn 0.4.4 (git+https://github.com/paritytech/bn)" = "" "checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" "checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" -"checksum cc 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7db2f146208d7e0fbee761b09cd65a7f51ccc38705d4e7262dad4d73b12a76b1" +"checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum cid 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d85ee025368e69063c420cbb2ed9f852cb03a5e69b73be021e65726ce03585b6" "checksum clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8f4a2b3bb7ef3c672d7c13d15613211d5a6976b6892c598b0fcb5d40765f19c2" "checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd" "checksum conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299" "checksum cookie 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d53b80dde876f47f03cda35303e368a79b91c70b0d65ecba5fd5280944a08591" -"checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" -"checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" +"checksum ct-logs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea083d2ff2ec269598a484b9a34cc04262eff81d03ed70cec89dca1794108b8" "checksum ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)" = "" "checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" "checksum daemonize 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0239832c1b4ca406d5ec73728cf4c7336d25cf85dd32db9e047e9e706ee0e935" @@ -3726,8 +3664,6 @@ dependencies = [ "checksum fixed-hash 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21c520ebc46522d519aec9cba2b7115d49cea707d771b772c46bec61aa0daeb8" "checksum flate2 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)" = "e6234dd4468ae5d1e2dbb06fe2b058696fdc50a339c68a393aefbf00bc81e423" "checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344" -"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0bab5b5e94f5c31fc764ba5dd9ad16568aae5d4825538c01d6bca680c9bf94a7" @@ -3745,7 +3681,7 @@ dependencies = [ "checksum hyper 0.10.0-a.0 (git+https://github.com/paritytech/hyper)" = "" "checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2" "checksum hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "641abc3e3fcf0de41165595f801376e01106bca1fd876dda937730e477ca004c" -"checksum hyper-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c81fa95203e2a6087242c38691a0210f23e9f3f8f944350bd676522132e2985" +"checksum hyper-rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aede89087d053868c8673a527a22d5df852dbd6291985bfb281cd11c76619cfa" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76" "checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb" @@ -3765,17 +3701,17 @@ dependencies = [ "checksum jsonrpc-ws-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.10)" = "" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" -"checksum lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3b37545ab726dd833ec6420aaba8231c5b320814b9029ad585555d2a03e94fbf" +"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3b585b7a6811fb03aa10e74b278a0f00f8dd9b45dc681f148bb29fa5cb61859b" "checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121" -"checksum libflate 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a2aa04ec0100812d31a5366130ff9e793291787bc31da845bede4a00ea329830" "checksum libusb 0.3.0 (git+https://github.com/paritytech/libusb-rs)" = "" "checksum libusb-sys 0.2.4 (git+https://github.com/paritytech/libusb-sys)" = "" "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939" "checksum linked-hash-map 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2aab0478615bb586559b0114d94dd8eca4fdbb73b443adcb0d00b61692b4bf" "checksum local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e1ceb20f39ff7ae42f3ff9795f3986b1daad821caaa1e1732a0944103a5a1a66" -"checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" "checksum lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d06ff7ff06f729ce5f4e227876cb88d10bc59cd4ae1e09fbb2bde15c850dc21" "checksum magenta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf0336886480e671965f794bc9b6fce88503563013d1bfb7a502c81fe3ac527" "checksum magenta-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40d014c7011ac470ae28e2f76a02bfea4a8480f73e701353b49ad7a8d75f4699" @@ -3794,7 +3730,6 @@ dependencies = [ "checksum msdos_time 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "65ba9d75bcea84e07812618fedf284a64776c2f2ea0cad6bca7f69739695a958" "checksum multibase 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b9c35dac080fd6e16a99924c8dfdef0af89d797dd851adab25feaffacf7850d6" "checksum multihash 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d49add5f49eb08bfc4d01ff286b84a48f53d45314f165c2d6efe477222d24f3" -"checksum native-tls 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f74dbadc8b43df7864539cedb7bc91345e532fdd913cfdc23ad94f4d2d40fbc0" "checksum net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)" = "3a80f842784ef6c9a958b68b7516bc7e35883c614004dd94959a4dca1b716c09" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum ntp 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "143149743832c6543b60a8ef2a26cd9122dfecec2b767158e852a7beecf6d7a0" @@ -3808,8 +3743,6 @@ dependencies = [ "checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d" "checksum number_prefix 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "59a14be9c211cb9c602bad35ac99f41e9a84b44d71b8cbd3040e3bd02a214902" "checksum ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c" -"checksum openssl 0.9.23 (registry+https://github.com/rust-lang/crates.io-index)" = "169a4b9160baf9b9b1ab975418c673686638995ba921683a7f1e01470dcb8854" -"checksum openssl-sys 0.9.23 (registry+https://github.com/rust-lang/crates.io-index)" = "2200ffec628e3f14c39fc0131a301db214f1a7d584e36507ee8700b0c7fb7a46" "checksum order-stat 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "efa535d5117d3661134dbf1719b6f0ffe06f2375843b13935db186cd094105eb" "checksum ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "58d25b6c0e47b20d05226d288ff434940296e7e2f8b877975da32f862152241f" "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" @@ -3821,14 +3754,13 @@ dependencies = [ "checksum parity-ui-precompiled 1.9.0 (git+https://github.com/js-dist-paritytech/parity-master-1-10-shell.git?rev=ac243b5ce9ce10d5f9f6137e974f195e8403b68e)" = "" "checksum parity-wasm 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9d3c21b85fed4e8490e716b7fcb247185ec201f28845be6e749ab864401463c7" "checksum parity-wordlist 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d0dec124478845b142f68b446cbee953d14d4b41f1bc0425024417720dce693" -"checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412" +"checksum parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9fd9d732f2de194336fb02fe11f9eed13d9e76f13f4315b4d88a14ca411750cd" "checksum parking_lot_core 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4f610cb9664da38e417ea3225f23051f589851999535290e077939838ab7a595" "checksum percent-encoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "de154f638187706bde41d9b4738748933d64e6b37bdbffc0b47a97d16a6ae356" "checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" "checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" "checksum phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6b07ffcc532ccc85e3afc45865469bf5d9e4ef5bfcf9622e3cfe80c2d275ec03" "checksum phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "07e24b0ca9643bdecd0632f2b3da6b1b89bbb0030e0b992afc1113b23a7bc2f2" -"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e5422a1ee1bc57cc47ae717b0137314258138f38fd5f3cea083f43a9725383a0" "checksum pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2412f3332a07c7a2a50168988dcc184f32180a9758ad470390e5f55e089f6b6e" "checksum primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0e31b86efadeaeb1235452171a66689682783149a6249ff334a2c5d8218d00a4" @@ -3847,12 +3779,13 @@ dependencies = [ "checksum rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)" = "512870020642bb8c221bf68baa1b2573da814f6ccfe5c9699b1c303047abe9b1" "checksum rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d5f78082e6a6d042862611e9640cf20776185fee506cf6cf67e93c6225cee31" "checksum rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b614fe08b6665cb9a231d07ac1364b0ef3cb3698f1239ee0c4c3a88a524f54c8" +"checksum rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed02d09394c94ffbdfdc755ad62a132e94c3224a8354e78a1200ced34df12edf" "checksum rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e64b609139d83da75902f88fd6c01820046840a18471e4dfcd5ac7c0f46bea53" "checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509" "checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" -"checksum reqwest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f73a8482e3b2b20ef5c07168b27048fc3778a012ce9b11a021556a450a01e9b5" "checksum ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6f7d28b30a72c01b458428e0ae988d4149c20d902346902be881e3edc4bb325c" +"checksum ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)" = "946e5e2b336032275e23152755ec2a0610ede0302101d3666fdb686795d26535" "checksum rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rocksdb-sys 0.3.0 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rotor 0.6.3 (git+https://github.com/tailhook/rotor)" = "" @@ -3863,12 +3796,11 @@ dependencies = [ "checksum rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0ceb8ce7a5e520de349e1fa172baeba4a9e8d5ef06c47471863530bc4972ee1e" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9743a7670d88d5d52950408ecdb7c71d8986251ab604d4689dd2ca25c9bca69" +"checksum rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab72e4883a4fc9fd5cd462a51c55d79f6a7b5c9483e8d73a2b7bca0b18430bcd" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" -"checksum schannel 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "acece75e0f987c48863a6c792ec8b7d6c4177d4a027f8ccc72f849794f437016" "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" "checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918" -"checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" -"checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" +"checksum sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4540aed8d71a5de961a8902cf356e28122bd62695eb5be1c214f84d8704097c" "checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)" = "4763b773978e495252615e814d2ad04773b2c1f85421c7913869a537f35cb406" @@ -3876,7 +3808,6 @@ dependencies = [ "checksum serde_derive_internals 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1fc848d073be32cd982380c06587ea1d433bc1a4c4a111de07ec2286a3ddade8" "checksum serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142" "checksum serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c9db7266c7d63a4c4b7fe8719656ccdd51acf1bed6124b174f933b009fb10bcb" -"checksum serde_urlencoded 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce0fd303af908732989354c6f02e05e2e6d597152870f2c6990efb0577137480" "checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" "checksum shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72f20b8f3c060374edb8046591ba28f62448c369ccbdc7b02075103fb3a9e38d" "checksum siphasher 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "833011ca526bd88f16778d32c699d325a9ad302fa06381cd66f7be63351d3f6d" @@ -3911,9 +3842,9 @@ dependencies = [ "checksum tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4ab83e7adb5677e42e405fa4ceff75659d93c4d7d7dd22f52fcec59ee9f02af" "checksum tokio-named-pipes 0.1.0 (git+https://github.com/nikvolf/tokio-named-pipes)" = "" "checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +"checksum tokio-rustls 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c0ba72dd900bf306bbf18c3cdb1dd526e8b5744439934fb2dfe3326d2caab9" "checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" "checksum tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" -"checksum tokio-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d88e411cac1c87e405e4090be004493c5d8072a370661033b1a64ea205ec2e13" "checksum tokio-uds 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6116c71be48f8f1656551fd16458247fdd6c03201d7893ad81189055fcde03e8" "checksum toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a7540f4ffc193e0d3c94121edb19b055670d369f77d5804db11ae053a45b6e7e" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" @@ -3932,10 +3863,9 @@ dependencies = [ "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" +"checksum untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70afa43c8c5d23a53a3c39ec9b56232c5badc19f6bb5ad529c1d6448a7241365" "checksum url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb819346883532a271eb626deb43c4a1bb4c4dd47c519bd78137c3e72a4fe27" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" -"checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" -"checksum vcpkg 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9e0a7d8bed3178a8fb112199d466eeca9ed09a14ba8ad67718179b4fd5487d0b" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" "checksum vecio 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0795a11576d29ae80525a3fda315bf7b534f8feb9d34101e5fe63fb95bb2fd24" "checksum vergen 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c3365f36c57e5df714a34be40902b27a992eeddb9996eca52d0584611cf885d" @@ -3943,6 +3873,8 @@ dependencies = [ "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)" = "" "checksum wasmi 0.0.0 (git+https://github.com/paritytech/wasmi)" = "" +"checksum webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)" = "c84d6c11545f0a9afd8757628563fe3941bd13194e30ee106d647d41ba0f7e45" +"checksum webpki-roots 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "edbd75d6abf044ef0c9d7ec92b9e8c518bcd93a15bb7bd9a92239e035248fc17" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index 1eaf788eded..a80f4bc0d3f 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -10,8 +10,11 @@ authors = ["Parity Technologies "] futures = "0.1" futures-cpupool = "0.1" parking_lot = "0.5" -log = "0.3" -reqwest = "0.8" +log = "0.4" + +hyper = "0.11" +hyper-rustls = "0.12" +tokio-core = "0.1" [features] default = [] diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 36874a82f1d..7831a9dc27c 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -20,11 +20,17 @@ use std::{io, fmt, time}; use std::sync::Arc; use std::sync::atomic::{self, AtomicBool}; -use futures::{self, Future}; +use futures::{self, Future, Async}; use futures_cpupool::{CpuPool, CpuFuture}; use parking_lot::RwLock; -use reqwest; -use reqwest::mime::Mime; + +use hyper::{self, Request, Method, StatusCode, Uri}; +use hyper::client::FutureResponse; +use hyper::header::{UserAgent, ContentType}; +use hyper::mime::Mime; + +use hyper_rustls; +use tokio_core; type BoxFuture = Box + Send>; @@ -94,18 +100,20 @@ pub trait Fetch: Clone + Send + Sync + 'static { const CLIENT_TIMEOUT_SECONDS: u64 = 5; +type HyperClient = hyper::Client; + /// Fetch client pub struct Client { - client: RwLock<(time::Instant, Arc)>, + client: RwLock>, pool: CpuPool, limit: Option, } impl Clone for Client { fn clone(&self) -> Self { - let (ref time, ref client) = *self.client.read(); + let client = *self.client.read(); Client { - client: RwLock::new((time.clone(), client.clone())), + client: RwLock::new(client.clone()), pool: self.pool.clone(), limit: self.limit.clone(), } @@ -113,15 +121,17 @@ impl Clone for Client { } impl Client { - fn new_client() -> Result, Error> { - let mut client = reqwest::ClientBuilder::new(); - client.redirect(reqwest::RedirectPolicy::limited(5)); - Ok(Arc::new(client.build()?)) + fn new_client() -> Result, Error> { + let mut core = tokio_core::reactor::Core::new()?; + let mut client = hyper::Client::configure() + .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())); + // TODO [kirushik] Now it's no longer following redirects automatically + Ok(Arc::new(client.build(&core.handle()))) } fn with_limit(limit: Option) -> Result { Ok(Client { - client: RwLock::new((time::Instant::now(), Self::new_client()?)), + client: RwLock::new(Self::new_client()?), pool: CpuPool::new(4), limit: limit, }) @@ -132,7 +142,7 @@ impl Client { self.limit = limit } - fn client(&self) -> Result, Error> { + fn client(&self) -> Result, Error> { { let (ref time, ref client) = *self.client.read(); if time.elapsed() < time::Duration::from_secs(CLIENT_TIMEOUT_SECONDS) { @@ -177,12 +187,20 @@ impl Fetch for Client { fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result { debug!(target: "fetch", "Fetching from: {:?}", url); + let url: Uri = match url.parse() { + Ok(url) => url, + Err(err) => return self.pool.spawn(futures::future::err(err.into())) + }; + + trace!(target: "fetch", "Starting fetch task: {:?}", self.url); + let mut request = Request::new(Method::Get, self.url); + request.headers_mut().set(UserAgent::new("Parity Fetch Neo")); match self.client() { Ok(client) => { self.pool.spawn(FetchTask { - url: url.into(), - client: client, + url: url, + response: client.request(request), limit: self.limit, abort: abort, }) @@ -195,8 +213,8 @@ impl Fetch for Client { } struct FetchTask { - url: String, - client: Arc, + url: Uri, + response: FutureResponse, limit: Option, abort: Abort, } @@ -212,17 +230,11 @@ impl Future for FetchTask { return Err(Error::Aborted); } - trace!(target: "fetch", "Starting fetch task: {:?}", self.url); - let result = self.client.get(&self.url) - .header(reqwest::header::UserAgent::new("Parity Fetch")) - .send()?; - - Ok(futures::Async::Ready(Response { - inner: ResponseInner::Response(result), - abort: self.abort.clone(), - limit: self.limit, - read: 0, - })) + match self.response.poll()? { + Async::Ready(response) => Ok(Async::Ready(response.into())), + Async::NotReady => Ok(Async::NotReady) + } + } } @@ -230,7 +242,11 @@ impl Future for FetchTask { #[derive(Debug)] pub enum Error { /// Internal fetch error - Fetch(reqwest::Error), + Fetch(hyper::Error), + /// IO error, from tokio core + IO(io::Error), + /// URI parse error + URI(hyper::error::UriError), /// Request aborted Aborted, } @@ -244,14 +260,26 @@ impl fmt::Display for Error { } } -impl From for Error { - fn from(error: reqwest::Error) -> Self { +impl From for Error { + fn from(error: hyper::Error) -> Self { Error::Fetch(error) } } +impl From for Error { + fn from(error: io::Error) -> Self { + Error::IO(error) + } +} + +impl From for Error { + fn from(error: hyper::error::UriError) -> Self { + Error::URI(error) + } +} + enum ResponseInner { - Response(reqwest::Response), + Response(hyper::Response), Reader(Box), NotFound, } @@ -297,17 +325,17 @@ impl Response { } /// Returns status code of this response. - pub fn status(&self) -> reqwest::StatusCode { + pub fn status(&self) -> StatusCode { match self.inner { ResponseInner::Response(ref r) => r.status(), - ResponseInner::NotFound => reqwest::StatusCode::NotFound, - _ => reqwest::StatusCode::Ok, + ResponseInner::NotFound => StatusCode::NotFound, + _ => StatusCode::Ok, } } /// Returns `true` if response status code is successful. pub fn is_success(&self) -> bool { - self.status() == reqwest::StatusCode::Ok + self.status() == StatusCode::Ok } /// Returns `true` if content type of this response is `text/html` @@ -322,7 +350,7 @@ impl Response { pub fn content_type(&self) -> Option { match self.inner { ResponseInner::Response(ref r) => { - let content_type = r.headers().get::(); + let content_type = r.headers().get::(); content_type.map(|mime| mime.0.clone()) }, _ => None, @@ -330,6 +358,12 @@ impl Response { } } +impl From for Response { + fn from(response: hyper::Response) -> Self { + unimplemented!() + } +} + impl io::Read for Response { fn read(&mut self, buf: &mut [u8]) -> io::Result { if self.abort.is_aborted() { diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index 1f68afe2c8f..d87f89f4af0 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -24,11 +24,13 @@ extern crate log; extern crate futures; extern crate futures_cpupool; extern crate parking_lot; -extern crate reqwest; + +extern crate hyper; +extern crate hyper_rustls; + +extern crate tokio_core; pub mod client; -pub use self::reqwest::StatusCode; -pub use self::reqwest::mime::Mime; pub use self::client::{Client, Fetch, Error, Response, Abort}; From b47f4c627823829fff72c5f52cf861c22763c6b6 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 2 Mar 2018 12:07:07 +0100 Subject: [PATCH 03/15] Add support for redirects. --- Cargo.lock | 98 ++--- dapps/src/apps/fetcher/installers.rs | 3 +- parity/run.rs | 2 +- util/fetch/Cargo.toml | 8 +- util/fetch/src/client.rs | 584 +++++++++++++++++++-------- util/fetch/src/lib.rs | 1 + 6 files changed, 451 insertions(+), 245 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f65ca4a403..3dd4b701d5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,15 +88,6 @@ dependencies = [ "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "base64" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "bigint" version = "4.2.0" @@ -281,10 +272,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ct-logs" -version = "0.3.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sct 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -558,7 +549,7 @@ dependencies = [ "keccak-hash 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", "memorydb 0.1.1", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -962,10 +953,11 @@ dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper-rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1175,20 +1167,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.12.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ct-logs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ct-logs 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-rustls 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", - "webpki-roots 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki-roots 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2255,7 +2246,7 @@ dependencies = [ "ethcore-bytes 0.1.0", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethsync 1.11.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.11.0", "parity-version 1.11.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2652,18 +2643,6 @@ dependencies = [ "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ring" -version = "0.13.0-alpha" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "rlp" version = "0.2.1" @@ -2789,15 +2768,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.12.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", - "sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sct 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2817,11 +2796,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "sct" -version = "0.3.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3182,14 +3161,13 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3382,11 +3360,6 @@ name = "untrusted" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "untrusted" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "url" version = "1.5.1" @@ -3506,20 +3479,20 @@ dependencies = [ [[package]] name = "webpki" -version = "0.18.0-alpha" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", - "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "webpki-roots" -version = "0.14.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)", + "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3620,7 +3593,6 @@ dependencies = [ "checksum base-x 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2f59103b47307f76e03bef1633aec7fa9e29bfb5aa6daf5a334f94233c71f6c1" "checksum base32 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b9605ba46d61df0410d8ac686b0007add8172eba90e8e909c347856fe794d8c" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" -"checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" "checksum bigint 4.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5442186ef6560f30f1ee4b9c1e4c87a35a6879d3644550cc248ec2b955eb5fcd" "checksum bincode 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e103c8b299b28a9c6990458b7013dc4a8356a9b854c51b9883241f5866fac36e" "checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" @@ -3641,7 +3613,7 @@ dependencies = [ "checksum cookie 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d53b80dde876f47f03cda35303e368a79b91c70b0d65ecba5fd5280944a08591" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" -"checksum ct-logs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea083d2ff2ec269598a484b9a34cc04262eff81d03ed70cec89dca1794108b8" +"checksum ct-logs 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "61cd11fb222fecf889f4531855c614548e92e8bd2eb178e35296885df5ee9a7c" "checksum ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)" = "" "checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" "checksum daemonize 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0239832c1b4ca406d5ec73728cf4c7336d25cf85dd32db9e047e9e706ee0e935" @@ -3681,7 +3653,7 @@ dependencies = [ "checksum hyper 0.10.0-a.0 (git+https://github.com/paritytech/hyper)" = "" "checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2" "checksum hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "641abc3e3fcf0de41165595f801376e01106bca1fd876dda937730e477ca004c" -"checksum hyper-rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aede89087d053868c8673a527a22d5df852dbd6291985bfb281cd11c76619cfa" +"checksum hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d6cdc1751771a14b8175764394f025e309a28c825ed9eaf97fa62bb831dc8c5" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum igd 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "356a0dc23a4fa0f8ce4777258085d00a01ea4923b2efd93538fc44bf5e1bda76" "checksum integer-encoding 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a053c9c7dcb7db1f2aa012c37dc176c62e4cdf14898dee0eecc606de835b8acb" @@ -3785,7 +3757,6 @@ dependencies = [ "checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" "checksum ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6f7d28b30a72c01b458428e0ae988d4149c20d902346902be881e3edc4bb325c" -"checksum ring 0.13.0-alpha (registry+https://github.com/rust-lang/crates.io-index)" = "946e5e2b336032275e23152755ec2a0610ede0302101d3666fdb686795d26535" "checksum rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rocksdb-sys 0.3.0 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rotor 0.6.3 (git+https://github.com/tailhook/rotor)" = "" @@ -3796,11 +3767,11 @@ dependencies = [ "checksum rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0ceb8ce7a5e520de349e1fa172baeba4a9e8d5ef06c47471863530bc4972ee1e" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9743a7670d88d5d52950408ecdb7c71d8986251ab604d4689dd2ca25c9bca69" -"checksum rustls 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab72e4883a4fc9fd5cd462a51c55d79f6a7b5c9483e8d73a2b7bca0b18430bcd" +"checksum rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc9f2e05fd6a3ce1530cd5dbcc553d2f94d7749fe3e4f5b443668eddd842889e" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" "checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918" -"checksum sct 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4540aed8d71a5de961a8902cf356e28122bd62695eb5be1c214f84d8704097c" +"checksum sct 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1137b767bbe1c4d30656993bdd97422ed41255d9400b105d735f8c7d9e800632" "checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)" = "4763b773978e495252615e814d2ad04773b2c1f85421c7913869a537f35cb406" @@ -3842,7 +3813,7 @@ dependencies = [ "checksum tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4ab83e7adb5677e42e405fa4ceff75659d93c4d7d7dd22f52fcec59ee9f02af" "checksum tokio-named-pipes 0.1.0 (git+https://github.com/nikvolf/tokio-named-pipes)" = "" "checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" -"checksum tokio-rustls 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c0ba72dd900bf306bbf18c3cdb1dd526e8b5744439934fb2dfe3326d2caab9" +"checksum tokio-rustls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9263e472d976e4345e50c6cce4cfe6b17c71593ea593cce1df26f1efd36debb" "checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" "checksum tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" "checksum tokio-uds 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6116c71be48f8f1656551fd16458247fdd6c03201d7893ad81189055fcde03e8" @@ -3863,7 +3834,6 @@ dependencies = [ "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" -"checksum untrusted 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70afa43c8c5d23a53a3c39ec9b56232c5badc19f6bb5ad529c1d6448a7241365" "checksum url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb819346883532a271eb626deb43c4a1bb4c4dd47c519bd78137c3e72a4fe27" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" diff --git a/dapps/src/apps/fetcher/installers.rs b/dapps/src/apps/fetcher/installers.rs index f47ab855cd0..8ca4377b8fa 100644 --- a/dapps/src/apps/fetcher/installers.rs +++ b/dapps/src/apps/fetcher/installers.rs @@ -19,9 +19,10 @@ use std::{fs, fmt}; use std::io::{self, Read, Write}; use std::path::PathBuf; use ethereum_types::H256; -use fetch::{self, Mime}; +use fetch; use futures_cpupool::CpuPool; use hash::keccak_buffer; +use mime_guess::Mime; use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest}; use handlers::{ContentValidator, ValidatorResponse}; diff --git a/parity/run.rs b/parity/run.rs index 155fcd143db..809532090b0 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -35,7 +35,7 @@ use ethcore_logger::{Config as LogConfig, RotatingLogger}; use ethcore_service::ClientService; use ethsync::{self, SyncConfig}; use fdlimit::raise_fd_limit; -use hash_fetch::fetch::{Fetch, Client as FetchClient}; +use hash_fetch::fetch::{Client as FetchClient}; use hash_fetch; use informant::{Informant, LightNodeInformantData, FullNodeInformantData}; use journaldb::Algorithm; diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index a80f4bc0d3f..d8ddc485dfc 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -9,12 +9,12 @@ authors = ["Parity Technologies "] [dependencies] futures = "0.1" futures-cpupool = "0.1" -parking_lot = "0.5" -log = "0.4" - hyper = "0.11" -hyper-rustls = "0.12" +hyper-rustls = "0.11" +log = "0.4" +parking_lot = "0.5" tokio-core = "0.1" +url = "1" [features] default = [] diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 7831a9dc27c..542cd65152d 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -16,21 +16,26 @@ //! Fetching -use std::{io, fmt, time}; +use std::cmp::min; +use std::{io, error, fmt, mem}; use std::sync::Arc; use std::sync::atomic::{self, AtomicBool}; +use std::thread; +use std::time::Duration; -use futures::{self, Future, Async}; -use futures_cpupool::{CpuPool, CpuFuture}; -use parking_lot::RwLock; +use futures::{self, Future, Async, Sink, Stream}; +use futures::future::{self, Either}; +use futures_cpupool::CpuPool; +use futures::sync::{mpsc, oneshot}; +use parking_lot::{Condvar, Mutex}; -use hyper::{self, Request, Method, StatusCode, Uri}; -use hyper::client::FutureResponse; -use hyper::header::{UserAgent, ContentType}; +use hyper::{self, Request, Method, StatusCode}; +use hyper::header::{UserAgent, Location, ContentType}; use hyper::mime::Mime; use hyper_rustls; -use tokio_core; +use tokio_core::reactor; +use url::{self, Url}; type BoxFuture = Box + Send>; @@ -56,29 +61,19 @@ pub trait Fetch: Clone + Send + Sync + 'static { /// Result type type Result: Future + Send + 'static; - /// Creates new Fetch object. - fn new() -> Result where Self: Sized; - /// Spawn the future in context of this `Fetch` thread pool. - /// Implementation is optional. - fn process(&self, f: F) -> BoxFuture where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, - { - Box::new(f) - } - - /// Spawn the future in context of this `Fetch` thread pool as "fire and forget", i.e. dropping this future without - /// canceling the underlying future. - /// Implementation is optional. - fn process_and_forget(&self, _: F) where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, - { - panic!("Attempting to process and forget future on unsupported Fetch."); - } + fn process(&self, f: F) -> BoxFuture + where F: Future + Send + 'static, + I: Send + 'static, + E: Send + 'static; + + /// Spawn the future in context of this `Fetch` thread pool as + /// "fire and forget", i.e. dropping this future without canceling + /// the underlying future. + fn process_and_forget(&self, f: F) + where F: Future + Send + 'static, + I: Send + 'static, + E: Send + 'static; /// Fetch URL and get a future for the result. /// Supports aborting the request in the middle of execution. @@ -88,71 +83,161 @@ pub trait Fetch: Clone + Send + Sync + 'static { fn fetch(&self, url: &str) -> Self::Result { self.fetch_with_abort(url, Default::default()) } - - /// Fetch URL and get the result synchronously. - fn fetch_sync(&self, url: &str) -> Result { - self.fetch(url).wait() - } - - /// Closes this client - fn close(self) where Self: Sized {} } +const THREAD_NAME: &str = "fetch"; const CLIENT_TIMEOUT_SECONDS: u64 = 5; +const MAX_REDIRECTS: usize = 5; -type HyperClient = hyper::Client; +type TxResponse = oneshot::Sender>; +type RxResponse = oneshot::Receiver>; +type StartupCond = Arc<(Mutex>, Condvar)>; -/// Fetch client -pub struct Client { - client: RwLock>, - pool: CpuPool, - limit: Option, +// `Proto`col values are sent over an mpsc channel from clients to +// their shared background thread with a tokio core and hyper cient inside. +enum Proto { + Request(Url, hyper::Request, TxResponse, usize), + Quit // terminates background thread } -impl Clone for Client { - fn clone(&self) -> Self { - let client = *self.client.read(); - Client { - client: RwLock::new(client.clone()), - pool: self.pool.clone(), - limit: self.limit.clone(), - } +impl Proto { + fn is_quit(&self) -> bool { + if let Proto::Quit = *self { true } else { false } } } +/// Fetch client +#[derive(Clone)] +pub struct Client { + pool: CpuPool, + tx_proto: mpsc::Sender, + limit: Option +} + impl Client { - fn new_client() -> Result, Error> { - let mut core = tokio_core::reactor::Core::new()?; - let mut client = hyper::Client::configure() - .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())); - // TODO [kirushik] Now it's no longer following redirects automatically - Ok(Arc::new(client.build(&core.handle()))) - } + /// Create a new client which spins up a separate thread running a + /// tokio `Core` and a `hyper::Client`. + /// Clones of this client share the same background thread. + pub fn new() -> Result { + let startup_done = Arc::new((Mutex::new(Ok(())), Condvar::new())); + let (tx_proto, rx_proto) = mpsc::channel(64); + + Client::background_thread(startup_done.clone(), tx_proto.clone(), rx_proto)?; + + let mut guard = startup_done.0.lock(); + let startup_result = startup_done.1.wait_for(&mut guard, Duration::from_secs(3)); + + if startup_result.timed_out() { + error!(target: "fetch", "timeout starting {}", THREAD_NAME); + return Err(Error::Other("timeout starting background thread".into())) + } + if let Err(e) = mem::replace(&mut *guard, Ok(())) { + error!(target: "fetch", "error starting background thread: {}", e); + return Err(e.into()) + } - fn with_limit(limit: Option) -> Result { Ok(Client { - client: RwLock::new(Self::new_client()?), - pool: CpuPool::new(4), - limit: limit, + pool: CpuPool::new(4), + tx_proto: tx_proto, + limit: Some(64 * 1024 * 1024) }) } - /// Sets a limit on the maximum download size. - pub fn set_limit(&mut self, limit: Option) { - self.limit = limit + fn background_thread(start: StartupCond, + tx_proto: mpsc::Sender, + rx_proto: mpsc::Receiver) -> io::Result> + { + thread::Builder::new().name(THREAD_NAME.into()).spawn(move || { + let mut core = match reactor::Core::new() { + Ok(c) => c, + Err(e) => { + *start.0.lock() = Err(e); + start.1.notify_one(); + return () + } + }; + let handle = core.handle(); + let client = hyper::Client::configure() + .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())) + .build(&core.handle()); + + start.1.notify_one(); + debug!(target: "fetch", "processing requests ..."); + + let maxdur = Duration::from_secs(CLIENT_TIMEOUT_SECONDS); + let work = rx_proto.take_while(|item| Ok(!item.is_quit())).for_each(|item| { + if let Proto::Request(url, rq, sender, redir) = item { + trace!(target: "fetch", "new request to {}", url); + let timeout = match reactor::Timeout::new(maxdur, &handle) { + Ok(t) => t, + Err(e) => { + error!(target: "fetch", "failed to create timeout: {}.", e); + return future::err(()) + } + }; + let reschedule = tx_proto.clone(); + let future = client.request(rq).select2(timeout).then(move |rs| { + trace!(target: "fetch", "response received from {}", url); + // When sending responses back over the oneshot channels, we treat + // the possibility that the other end is gone as normal, hence we + // use `unwrap_or(())` and do not error. + match rs { + Ok(Either::A((rs, _))) => { + if let Some(next_url) = redirect_location(url, &rs) { + if redir == 0 { + Either::A(future::ok(sender.send(Err(Error::TooManyRedirects)).unwrap_or(()))) + } else { + let next_req = get(&next_url); + Either::B(reschedule.send(Proto::Request(next_url, next_req, sender, redir - 1)).then(|result| { + if let Err(e) = result { + error!(target: "fetch", "failed to reschedule request: {}", e); + } + // We can not recover from this error. Client code will + // get a `oneshot::Canceled` error since we dropped the + // `oneshot::Sender`. This should not happen as long as + // this thread runs, as with `reschedule` we are + // sending the `Proto` value back to ourselves. + future::ok(()) + })) + } + } else { + Either::A(future::ok(sender.send(Ok(rs)).unwrap_or(()))) + } + } + Ok(Either::B((_, _))) => Either::A(future::ok(sender.send(Err(Error::Timeout)).unwrap_or(()))), + Err(Either::A((err, _))) => Either::A(future::ok(sender.send(Err(err.into())).unwrap_or(()))), + Err(Either::B((err, _))) => Either::A(future::ok(sender.send(Err(err.into())).unwrap_or(()))), + } + }); + handle.spawn(future); + trace!(target: "fetch", "waiting for next request...") + } + future::ok(()) + }); + if let Err(()) = core.run(work) { + error!(target: "fetch", "error while executing future") + } + debug!(target: "fetch", "{} background thread finished", THREAD_NAME) + }) } - fn client(&self) -> Result, Error> { - { - let (ref time, ref client) = *self.client.read(); - if time.elapsed() < time::Duration::from_secs(CLIENT_TIMEOUT_SECONDS) { - return Ok(client.clone()); - } - } + /// Close this client by shutting down the background thread. + /// + /// Please note that this will affect all clones of this `Client` as they all + /// share the same background thread. + pub fn close(self) -> Result<(), Error> { + self.tx_proto.clone().send(Proto::Quit).wait() + .map_err(|e| { + error!(target: "fetch", "failed to send quit to background thread: {}", e); + // We can not put `e: SendError` into `Other` as it is not `Send`. + Error::Other("failed to terminate background thread".into()) + })?; + Ok(()) + } - let client = Self::new_client()?; - *self.client.write() = (time::Instant::now(), client.clone()); - Ok(client) + /// (Un-)set size limit on response body. + pub fn set_limit(&mut self, limit: Option) { + self.limit = limit; } /// Returns a handle to underlying CpuPool of this client. @@ -162,134 +247,183 @@ impl Client { } impl Fetch for Client { - type Result = CpuFuture; + type Result = BoxFuture; + + fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result { + debug!(target: "fetch", "fetching: {:?}", url); + + let url: Url = match url.parse() { + Ok(u) => u, + Err(e) => return Box::new(futures::future::err(e.into())) + }; - fn new() -> Result { - // Max 64MB will be downloaded. - Self::with_limit(Some(64 * 1024 * 1024)) + let req = get(&url); + let sender = self.tx_proto.clone(); + let limit = self.limit.clone(); + let (tx_res, rx_res) = oneshot::channel(); + let future = sender.send(Proto::Request(url.clone(), req, tx_res, MAX_REDIRECTS)) + .map(|_| rx_res) + .map_err(|e| { + error!(target: "fetch", "failed to schedule request: {}", e); + Error::Other("failed to schedule request".into()) + }) + .and_then(move |rx_res| { + FetchTask { + url: url, + rx_res: rx_res, + limit: limit, + abort: abort + } + }); + Box::new(future) } - fn process(&self, f: F) -> BoxFuture where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, + fn process(&self, f: F) -> BoxFuture + where F: Future + Send + 'static, + I: Send + 'static, + E: Send + 'static { Box::new(self.pool.spawn(f)) } - fn process_and_forget(&self, f: F) where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, + fn process_and_forget(&self, f: F) + where F: Future + Send + 'static, + I: Send + 'static, + E: Send + 'static { self.pool.spawn(f).forget() } +} - fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result { - debug!(target: "fetch", "Fetching from: {:?}", url); - let url: Uri = match url.parse() { - Ok(url) => url, - Err(err) => return self.pool.spawn(futures::future::err(err.into())) - }; - - trace!(target: "fetch", "Starting fetch task: {:?}", self.url); - let mut request = Request::new(Method::Get, self.url); - request.headers_mut().set(UserAgent::new("Parity Fetch Neo")); - - match self.client() { - Ok(client) => { - self.pool.spawn(FetchTask { - url: url, - response: client.request(request), - limit: self.limit, - abort: abort, - }) - }, - Err(err) => { - self.pool.spawn(futures::future::err(err)) - }, +fn redirect_location(u: Url, r: &hyper::Response) -> Option { + use hyper::StatusCode::*; + match r.status() { + MovedPermanently + | PermanentRedirect + | TemporaryRedirect + | Found + | SeeOther => { + if let Some(loc) = r.headers().get::() { + u.join(loc).ok() + } else { + None + } } + _ => None } } +fn get(u: &Url) -> hyper::Request { + let uri = u.as_ref().parse().expect("Every valid URL is aso a URI"); + let mut rq = Request::new(Method::Get, uri); + rq.headers_mut().set(UserAgent::new("Parity Fetch Neo")); + rq +} + struct FetchTask { - url: Uri, - response: FutureResponse, - limit: Option, - abort: Abort, + url: Url, + rx_res: RxResponse, + limit: Option, + abort: Abort } impl Future for FetchTask { - // TODO [ToDr] timeouts handling? type Item = Response; type Error = Error; fn poll(&mut self) -> futures::Poll { if self.abort.is_aborted() { - trace!(target: "fetch", "Fetch of {:?} aborted.", self.url); + debug!(target: "fetch", "Fetch of {:?} aborted.", self.url); return Err(Error::Aborted); } - - match self.response.poll()? { - Async::Ready(response) => Ok(Async::Ready(response.into())), + match self.rx_res.poll()? { + Async::Ready(Err(e)) => Err(e.into()), + Async::Ready(Ok(r)) => { + let ctype = r.headers().get::().cloned(); + Ok(Async::Ready(Response { + inner: ResponseInner::Response(r.status(), ctype, BodyReader::new(r.body())), + abort: self.abort.clone(), + limit: self.limit.clone(), + read: 0 + })) + } Async::NotReady => Ok(Async::NotReady) } - } } -/// Fetch Error +/// Fetch related error cases. #[derive(Debug)] pub enum Error { - /// Internal fetch error - Fetch(hyper::Error), - /// IO error, from tokio core - IO(io::Error), - /// URI parse error - URI(hyper::error::UriError), + /// Error produced by hyper. + Hyper(hyper::Error), + /// I/O error + Io(io::Error), + /// URL parse error + Url(url::ParseError), /// Request aborted Aborted, + /// Followed too many redirects + TooManyRedirects, + /// Request took too long + Timeout, + /// The background request procesing was canceled + Canceled, + /// Some other error + Other(Box) } impl fmt::Display for Error { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match *self { - Error::Aborted => write!(fmt, "The request has been aborted."), - Error::Fetch(ref err) => write!(fmt, "{}", err), + Error::Aborted => write!(fmt, "The request has been aborted."), + Error::Hyper(ref e) => write!(fmt, "{}", e), + Error::Url(ref e) => write!(fmt, "{}", e), + Error::Io(ref e) => write!(fmt, "{}", e), + Error::Other(ref e) => write!(fmt, "{}", e), + Error::TooManyRedirects => write!(fmt, "too many redirects"), + Error::Timeout => write!(fmt, "request timed out"), + Error::Canceled => write!(fmt, "background thread canceled request processing"), } } } +impl From for Error { + fn from(_: oneshot::Canceled) -> Self { + Error::Canceled + } +} + impl From for Error { - fn from(error: hyper::Error) -> Self { - Error::Fetch(error) + fn from(e: hyper::Error) -> Self { + Error::Hyper(e) } } impl From for Error { - fn from(error: io::Error) -> Self { - Error::IO(error) + fn from(e: io::Error) -> Self { + Error::Io(e) } } -impl From for Error { - fn from(error: hyper::error::UriError) -> Self { - Error::URI(error) +impl From for Error { + fn from(e: url::ParseError) -> Self { + Error::Url(e) } } enum ResponseInner { - Response(hyper::Response), + Response(StatusCode, Option, BodyReader), Reader(Box), - NotFound, + NotFound } impl fmt::Debug for ResponseInner { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - ResponseInner::Response(ref response) => response.fmt(f), - ResponseInner::NotFound => write!(f, "Not found"), - ResponseInner::Reader(_) => write!(f, "io Reader"), + ResponseInner::Response(s, ..) => write!(f, "hyper response (status={})", s), + ResponseInner::NotFound => write!(f, "not found"), + ResponseInner::Reader(_) => write!(f, "io reader"), } } } @@ -300,7 +434,7 @@ pub struct Response { inner: ResponseInner, abort: Abort, limit: Option, - read: usize, + read: usize } impl Response { @@ -310,7 +444,7 @@ impl Response { inner: ResponseInner::Reader(Box::new(reader)), abort: Abort::default(), limit: None, - read: 0, + read: 0 } } @@ -320,16 +454,16 @@ impl Response { inner: ResponseInner::NotFound, abort: Abort::default(), limit: None, - read: 0, + read: 0 } } /// Returns status code of this response. pub fn status(&self) -> StatusCode { match self.inner { - ResponseInner::Response(ref r) => r.status(), - ResponseInner::NotFound => StatusCode::NotFound, - _ => StatusCode::Ok, + ResponseInner::Response(s, ..) => s, + ResponseInner::NotFound => StatusCode::NotFound, + _ => StatusCode::Ok } } @@ -340,30 +474,23 @@ impl Response { /// Returns `true` if content type of this response is `text/html` pub fn is_html(&self) -> bool { - match self.content_type() { - Some(ref mime) if mime.type_() == "text" && mime.subtype() == "html" => true, - _ => false, + if let Some(ref mime) = self.content_type() { + mime.type_() == "text" && mime.subtype() == "html" + } else { + false } } /// Returns content type of this response (if present) pub fn content_type(&self) -> Option { - match self.inner { - ResponseInner::Response(ref r) => { - let content_type = r.headers().get::(); - content_type.map(|mime| mime.0.clone()) - }, - _ => None, + if let ResponseInner::Response(_, ref c, _) = self.inner { + c.as_ref().map(|mime| mime.0.clone()) + } else { + None } } } -impl From for Response { - fn from(response: hyper::Response) -> Self { - unimplemented!() - } -} - impl io::Read for Response { fn read(&mut self, buf: &mut [u8]) -> io::Result { if self.abort.is_aborted() { @@ -371,38 +498,145 @@ impl io::Read for Response { } let res = match self.inner { - ResponseInner::Response(ref mut response) => response.read(buf), - ResponseInner::NotFound => return Ok(0), - ResponseInner::Reader(ref mut reader) => reader.read(buf), + ResponseInner::Response(_, _, ref mut r) => r.read(buf), + ResponseInner::NotFound => return Ok(0), + ResponseInner::Reader(ref mut r) => r.read(buf) }; // increase bytes read if let Ok(read) = res { - self.read += read; + self.read += read } // check limit match self.limit { Some(limit) if limit < self.read => { return Err(io::Error::new(io::ErrorKind::PermissionDenied, "Size limit reached.")); - }, - _ => {}, + } + _ => {} } res } } +// `BodyReader` serves as a bridge from async to sync I/O. It implements +// `io::Read` by repedately waiting for the next `Chunk` of hyper's response `Body`. +struct BodyReader { + chunk: hyper::Chunk, + body: Option, + offset: usize +} + +impl BodyReader { + fn new(b: hyper::Body) -> BodyReader { + BodyReader { body: Some(b), chunk: Default::default(), offset: 0 } + } +} + +impl io::Read for BodyReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut m = 0; + while self.body.is_some() { + // Can we still read from the current chunk? + if self.offset < self.chunk.len() { + let k = min(self.chunk.len() - self.offset, buf.len() - m); + let c = &self.chunk[self.offset .. self.offset + k]; + (&mut buf[m .. m + k]).copy_from_slice(c); + self.offset += k; + m += k; + if m == buf.len() { + break + } + } else { + // While in this loop, `self.body` is always defined => wait for the next chunk. + match self.body.take().unwrap().into_future().wait() { + Err((e, _)) => { + error!(target: "fetch", "failed to read chunk: {}", e); + return Err(io::Error::new(io::ErrorKind::Other, "failed to read body chunk")) + } + Ok((None, _)) => break, // body is exhausted, break out of the loop + Ok((Some(c), b)) => { + self.body = Some(b); + self.chunk = c; + self.offset = 0 + } + } + } + } + Ok(m) + } +} + #[cfg(test)] mod test { use super::*; + use std::io::Read; #[test] fn it_should_fetch() { - let fetch = Client::new().unwrap(); - let ya = fetch.fetch("https://ya.ru").wait(); - assert!(ya.is_ok()); - let ya_response = ya.unwrap(); - assert!(ya_response.is_success()); + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/drip?numbytes=3&duration=3&delay=1&code=200"); + let mut resp = future.wait().unwrap(); + assert!(resp.is_success()); + let mut body = Vec::new(); + resp.read_to_end(&mut body).unwrap(); + assert_eq!(body.len(), 3) + } + + #[test] + fn it_should_timeout() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/delay/7"); + match future.wait() { + Err(Error::Timeout) => {} + other => panic!("expected timeout, got {:?}", other) + } + } + + #[test] + fn it_should_follow_redirects() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/absolute-redirect/3"); + assert!(future.wait().unwrap().is_success()) + } + + #[test] + fn it_should_follow_relative_redirects() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/relative-redirect/3"); + assert!(future.wait().unwrap().is_success()) } -} \ No newline at end of file + + #[test] + fn it_should_not_follow_too_many_redirects() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/absolute-redirect/100"); + match future.wait() { + Err(Error::TooManyRedirects) => {} + other => panic!("expected too many redirects error, got {:?}", other) + } + } + + #[test] + fn it_should_read_data() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/bytes/1024"); + let mut resp = future.wait().unwrap(); + assert!(resp.is_success()); + let mut body = Vec::new(); + resp.read_to_end(&mut body).unwrap(); + assert_eq!(body.len(), 1024) + } + + #[test] + fn it_should_read_chunked_data() { + let client = Client::new().unwrap(); + let future = client.fetch("https://httpbin.org/stream-bytes/1024?chunk_size=19"); + let mut resp = future.wait().unwrap(); + assert!(resp.is_success()); + let mut body = Vec::new(); + resp.read_to_end(&mut body).unwrap(); + assert_eq!(body.len(), 1024) + } +} diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index d87f89f4af0..23a85e0da8b 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -29,6 +29,7 @@ extern crate hyper; extern crate hyper_rustls; extern crate tokio_core; +extern crate url; pub mod client; From 7fec3aec8ed4dbd66dcef02c19a73eae1e937667 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 7 Mar 2018 14:33:48 +0100 Subject: [PATCH 04/15] Remove CpuPool from `fetch::Client`. --- Cargo.lock | 4 ++- dapps/src/apps/fetcher/mod.rs | 3 ++ dapps/src/apps/mod.rs | 2 +- dapps/src/handlers/fetch.rs | 10 +++++-- dapps/src/web.rs | 5 ++++ ethcore/Cargo.toml | 1 + ethcore/src/lib.rs | 1 + ethcore/src/miner/miner.rs | 5 ++-- hash-fetch/Cargo.toml | 1 + hash-fetch/src/client.rs | 15 ++++++---- hash-fetch/src/lib.rs | 1 + parity/dapps.rs | 6 ++-- parity/params.rs | 6 ++-- parity/rpc_apis.rs | 9 ++++-- parity/run.rs | 24 ++++++++++------ price-info/Cargo.toml | 1 + price-info/src/lib.rs | 11 ++++--- rpc/src/v1/impls/light/parity_set.rs | 9 ++++-- rpc/src/v1/impls/parity_set.rs | 8 ++++-- util/fetch/Cargo.toml | 1 - util/fetch/src/client.rs | 43 +--------------------------- util/fetch/src/lib.rs | 1 - 22 files changed, 87 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dd4b701d5b..f1670909cb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,6 +460,7 @@ dependencies = [ "ethkey 0.3.0", "ethstore 0.2.0", "evm 0.1.0", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "hardware-wallet 1.11.0", "hashdb 0.1.1", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -951,7 +952,6 @@ name = "fetch" version = "0.1.0" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2037,6 +2037,7 @@ dependencies = [ "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2426,6 +2427,7 @@ version = "1.11.0" dependencies = [ "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/dapps/src/apps/fetcher/mod.rs b/dapps/src/apps/fetcher/mod.rs index 44352f76f1c..981437ffbf4 100644 --- a/dapps/src/apps/fetcher/mod.rs +++ b/dapps/src/apps/fetcher/mod.rs @@ -216,6 +216,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), + self.pool.clone() ) }, URLHintResult::GithubDapp(content) => { @@ -232,6 +233,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), + self.pool.clone() ) }, URLHintResult::Content(content) => { @@ -248,6 +250,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), + self.pool.clone() ) }, }; diff --git a/dapps/src/apps/mod.rs b/dapps/src/apps/mod.rs index e7657736d18..c065a08474c 100644 --- a/dapps/src/apps/mod.rs +++ b/dapps/src/apps/mod.rs @@ -81,7 +81,7 @@ pub fn all_endpoints( insert::(&mut pages, "v1", Embeddable::Yes(embeddable.clone()), pool.clone()); pages.insert("proxy".into(), ProxyPac::boxed(embeddable.clone(), dapps_domain.to_owned())); - pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), fetch.clone())); + pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), fetch.clone(), pool.clone())); (local_endpoints, pages) } diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index 8e0fc021e0f..079feb1340c 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -23,6 +23,7 @@ use std::time::{Instant, Duration}; use fetch::{self, Fetch}; use futures::sync::oneshot; use futures::{self, Future}; +use futures_cpupool::CpuPool; use hyper::{self, Method, StatusCode}; use parking_lot::Mutex; @@ -252,6 +253,7 @@ impl ContentFetcherHandler { installer: H, embeddable_on: Embeddable, fetch: F, + pool: CpuPool ) -> Self { let fetch_control = FetchControl::default(); let errors = Errors { embeddable_on }; @@ -262,6 +264,7 @@ impl ContentFetcherHandler { Method::Get => { trace!(target: "dapps", "Fetching content from: {:?}", url); FetchState::InProgress(Self::fetch_content( + pool, fetch, url, fetch_control.abort.clone(), @@ -282,6 +285,7 @@ impl ContentFetcherHandler { } fn fetch_content( + pool: CpuPool, fetch: F, url: &str, abort: Arc, @@ -290,7 +294,7 @@ impl ContentFetcherHandler { installer: H, ) -> Box + Send> { // Start fetching the content - let fetch2 = fetch.clone(); + let pool2 = pool.clone(); let future = fetch.fetch_with_abort(url, abort.into()).then(move |result| { trace!(target: "dapps", "Fetching content finished. Starting validation: {:?}", result); Ok(match result { @@ -303,7 +307,7 @@ impl ContentFetcherHandler { Ok(ValidatorResponse::Streaming(stream)) => { trace!(target: "dapps", "Validation OK. Streaming response."); let (reading, response) = stream.into_response(); - fetch2.process_and_forget(reading); + pool.spawn(reading).forget(); FetchState::Streaming(response) }, Err(e) => { @@ -319,7 +323,7 @@ impl ContentFetcherHandler { }); // make sure to run within fetch thread pool. - fetch.process(future) + Box::new(pool2.spawn(future)) } } diff --git a/dapps/src/web.rs b/dapps/src/web.rs index 395ab6deb46..8b4d95d5071 100644 --- a/dapps/src/web.rs +++ b/dapps/src/web.rs @@ -25,6 +25,7 @@ use hyper::{mime, StatusCode}; use apps; use endpoint::{Endpoint, EndpointPath, Request, Response}; use futures::future; +use futures_cpupool::CpuPool; use handlers::{ ContentFetcherHandler, ContentHandler, ContentValidator, ValidatorResponse, StreamingHandler, @@ -35,6 +36,7 @@ pub struct Web { embeddable_on: Embeddable, web_proxy_tokens: Arc, fetch: F, + pool: CpuPool } impl Web { @@ -42,11 +44,13 @@ impl Web { embeddable_on: Embeddable, web_proxy_tokens: Arc, fetch: F, + pool: CpuPool ) -> Box { Box::new(Web { embeddable_on, web_proxy_tokens, fetch, + pool }) } @@ -129,6 +133,7 @@ impl Endpoint for Web { }, self.embeddable_on.clone(), self.fetch.clone(), + self.pool.clone() )) } } diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 732cc938cca..63b890e0347 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -33,6 +33,7 @@ ethjson = { path = "../json" } ethkey = { path = "../ethkey" } ethstore = { path = "../ethstore" } evm = { path = "evm" } +futures-cpupool = "0.1" hardware-wallet = { path = "../hw" } heapsize = "0.4" itertools = "0.5" diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 0d2ea0f5ca5..d23fed91792 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -71,6 +71,7 @@ extern crate ethcore_transaction as transaction; extern crate ethereum_types; extern crate ethjson; extern crate ethkey; +extern crate futures_cpupool; extern crate hardware_wallet; extern crate hashdb; extern crate itertools; diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index ec4ab87fdf3..58111cd741a 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -35,6 +35,7 @@ use ethcore_miner::transaction_queue::{ AccountDetails, TransactionOrigin, }; +use futures_cpupool::CpuPool; use ethcore_miner::work_notify::{WorkPoster, NotifyWork}; use miner::service_transaction_checker::ServiceTransactionChecker; use miner::{MinerService, MinerStatus}; @@ -218,11 +219,11 @@ pub enum GasPricer { impl GasPricer { /// Create a new Calibrated `GasPricer`. - pub fn new_calibrated(options: GasPriceCalibratorOptions, fetch: FetchClient) -> GasPricer { + pub fn new_calibrated(options: GasPriceCalibratorOptions, fetch: FetchClient, p: CpuPool) -> GasPricer { GasPricer::Calibrated(GasPriceCalibrator { options: options, next_calibration: Instant::now(), - price_info: PriceInfoClient::new(fetch), + price_info: PriceInfoClient::new(fetch, p), }) } diff --git a/hash-fetch/Cargo.toml b/hash-fetch/Cargo.toml index 19b846c1cfd..4c013e1dbb6 100644 --- a/hash-fetch/Cargo.toml +++ b/hash-fetch/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Parity Technologies "] [dependencies] futures = "0.1" +futures-cpupool = "0.1" log = "0.3" mime = "0.3" mime_guess = "2.0.0-alpha.2" diff --git a/hash-fetch/src/client.rs b/hash-fetch/src/client.rs index 3ee58e3ff91..4512f817e85 100644 --- a/hash-fetch/src/client.rs +++ b/hash-fetch/src/client.rs @@ -23,6 +23,7 @@ use std::path::PathBuf; use hash::keccak_buffer; use fetch::{Fetch, Response, Error as FetchError, Client as FetchClient}; +use futures_cpupool::CpuPool; use futures::{Future, IntoFuture}; use parity_reactor::Remote; use urlhint::{URLHintContract, URLHint, URLHintResult}; @@ -113,6 +114,7 @@ fn validate_hash(path: PathBuf, hash: H256, result: Result /// Default Hash-fetching client using on-chain contract to resolve hashes to URLs. pub struct Client { + pool: CpuPool, contract: URLHintContract, fetch: F, remote: Remote, @@ -121,15 +123,16 @@ pub struct Client { impl Client { /// Creates new instance of the `Client` given on-chain contract client and task runner. - pub fn new(contract: Arc>, remote: Remote) -> Self { - Client::with_fetch(contract, FetchClient::new().unwrap(), remote) + pub fn new(contract: Arc>, pool: CpuPool, remote: Remote) -> Self { + Client::with_fetch(contract, pool, FetchClient::new().unwrap(), remote) } } impl Client { /// Creates new instance of the `Client` given on-chain contract client, fetch service and task runner. - pub fn with_fetch(contract: Arc>, fetch: F, remote: Remote) -> Self { + pub fn with_fetch(contract: Arc>, pool: CpuPool, fetch: F, remote: Remote) -> Self { Client { + pool, contract: URLHintContract::new(contract), fetch: fetch, remote: remote, @@ -144,6 +147,7 @@ impl HashFetch for Client { let random_path = self.random_path.clone(); let remote_fetch = self.fetch.clone(); + let pool = self.pool.clone(); let future = self.contract.resolve(hash) .map_err(|e| { warn!("Error resolving URL: {}", e); Error::NoResolution }) .and_then(|maybe_url| maybe_url.ok_or(Error::NoResolution)) @@ -161,7 +165,7 @@ impl HashFetch for Client { .into_future() .and_then(move |url| { debug!(target: "fetch", "Resolved {:?} to {:?}. Fetching...", hash, url); - let future = remote_fetch.fetch(&url).then(move |result| { + pool.spawn(remote_fetch.fetch(&url).then(move |result| { debug!(target: "fetch", "Content fetched, validating hash ({:?})", hash); let path = random_path(); let res = validate_hash(path.clone(), hash, result); @@ -171,8 +175,7 @@ impl HashFetch for Client { let _ = fs::remove_file(&path); } res - }); - remote_fetch.process(future) + })) }) .then(move |res| { on_done(res); Ok(()) as Result<(), ()> }); diff --git a/hash-fetch/src/lib.rs b/hash-fetch/src/lib.rs index a33639b46d7..be9df25f87b 100644 --- a/hash-fetch/src/lib.rs +++ b/hash-fetch/src/lib.rs @@ -25,6 +25,7 @@ extern crate ethabi; extern crate ethcore_bytes as bytes; extern crate ethereum_types; extern crate futures; +extern crate futures_cpupool; extern crate keccak_hash as hash; extern crate mime; extern crate mime_guess; diff --git a/parity/dapps.rs b/parity/dapps.rs index b1004276c0d..d93ae2db7cc 100644 --- a/parity/dapps.rs +++ b/parity/dapps.rs @@ -23,6 +23,7 @@ use dir::helpers::replace_home; use ethcore::client::{Client, BlockChainClient, BlockId, CallContract}; use ethsync::LightSync; use futures::{Future, future, IntoFuture}; +use futures_cpupool::CpuPool; use hash_fetch::fetch::Client as FetchClient; use registrar::{RegistrarClient, Asynchronous}; use light::client::LightChainClient; @@ -160,6 +161,7 @@ pub struct Dependencies { pub sync_status: Arc, pub contract_client: Arc>, pub fetch: FetchClient, + pub pool: CpuPool, pub signer: Arc, pub ui_address: Option<(String, u16)>, } @@ -253,7 +255,7 @@ mod server { let web_proxy_tokens = Arc::new(move |token| signer.web_proxy_access_token_domain(&token)); Ok(parity_dapps::Middleware::dapps( - deps.fetch.pool(), + deps.pool, deps.node_health, deps.ui_address, extra_embed_on, @@ -273,7 +275,7 @@ mod server { dapps_domain: &str, ) -> Result { Ok(parity_dapps::Middleware::ui( - deps.fetch.pool(), + deps.pool, deps.node_health, dapps_domain, deps.contract_client, diff --git a/parity/params.rs b/parity/params.rs index ba750ae761b..9d3eee84569 100644 --- a/parity/params.rs +++ b/parity/params.rs @@ -17,6 +17,7 @@ use std::{str, fs, fmt}; use std::time::Duration; use ethereum_types::{U256, Address}; +use futures_cpupool::CpuPool; use parity_version::version_data; use journaldb::Algorithm; use ethcore::spec::{Spec, SpecParams}; @@ -240,7 +241,7 @@ impl Default for GasPricerConfig { } impl GasPricerConfig { - pub fn to_gas_pricer(&self, fetch: FetchClient) -> GasPricer { + pub fn to_gas_pricer(&self, fetch: FetchClient, p: CpuPool) -> GasPricer { match *self { GasPricerConfig::Fixed(u) => GasPricer::Fixed(u), GasPricerConfig::Calibrated { usd_per_tx, recalibration_period, .. } => { @@ -249,7 +250,8 @@ impl GasPricerConfig { usd_per_tx: usd_per_tx, recalibration_period: recalibration_period, }, - fetch + fetch, + p ) } } diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index f2ab075911b..1ebfe04921f 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -28,6 +28,7 @@ use ethcore::miner::Miner; use ethcore::snapshot::SnapshotService; use ethcore_logger::RotatingLogger; use ethsync::{ManageNetwork, SyncProvider, LightSync}; +use futures_cpupool::CpuPool; use hash_fetch::fetch::Client as FetchClient; use jsonrpc_core::{self as core, MetaIoHandler}; use light::client::LightChainClient; @@ -225,6 +226,7 @@ pub struct FullDependencies { pub dapps_address: Option, pub ws_address: Option, pub fetch: FetchClient, + pub pool: CpuPool, pub remote: parity_reactor::Remote, pub whisper_rpc: Option<::whisper::RpcFactory>, pub gas_price_percentile: usize, @@ -253,7 +255,7 @@ impl FullDependencies { } } - let nonces = Arc::new(Mutex::new(dispatch::Reservations::with_pool(self.fetch.pool()))); + let nonces = Arc::new(Mutex::new(dispatch::Reservations::with_pool(self.pool.clone()))); let dispatcher = FullDispatcher::new( self.client.clone(), self.miner.clone(), @@ -355,6 +357,7 @@ impl FullDependencies { &self.net_service, self.dapps_service.clone(), self.fetch.clone(), + self.pool.clone() ).to_delegate()) }, Api::Traces => { @@ -430,6 +433,7 @@ pub struct LightDependencies { pub dapps_address: Option, pub ws_address: Option, pub fetch: FetchClient, + pub pool: CpuPool, pub geth_compatibility: bool, pub remote: parity_reactor::Remote, pub whisper_rpc: Option<::whisper::RpcFactory>, @@ -451,7 +455,7 @@ impl LightDependencies { self.on_demand.clone(), self.cache.clone(), self.transaction_queue.clone(), - Arc::new(Mutex::new(dispatch::Reservations::with_pool(self.fetch.pool()))), + Arc::new(Mutex::new(dispatch::Reservations::with_pool(self.pool.clone()))), self.gas_price_percentile, ); @@ -564,6 +568,7 @@ impl LightDependencies { self.sync.clone(), self.dapps_service.clone(), self.fetch.clone(), + self.pool.clone() ).to_delegate()) }, Api::Traces => { diff --git a/parity/run.rs b/parity/run.rs index 809532090b0..7e913bbea5f 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -35,8 +35,8 @@ use ethcore_logger::{Config as LogConfig, RotatingLogger}; use ethcore_service::ClientService; use ethsync::{self, SyncConfig}; use fdlimit::raise_fd_limit; -use hash_fetch::fetch::{Client as FetchClient}; -use hash_fetch; +use futures_cpupool::CpuPool; +use hash_fetch::{self, fetch}; use informant::{Informant, LightNodeInformantData, FullNodeInformantData}; use journaldb::Algorithm; use kvdb_rocksdb::{Database, DatabaseConfig}; @@ -308,8 +308,10 @@ fn execute_light_impl(cmd: RunCmd, can_restart: bool, logger: Arc) // prepare account provider let account_provider = Arc::new(prepare_account_provider(&cmd.spec, &cmd.dirs, &spec.data_dir, cmd.acc_conf, &passwords)?); + let cpu_pool = CpuPool::new(4); + // fetch service - let fetch = FetchClient::new().map_err(|e| format!("Error starting fetch client: {:?}", e))?; + let fetch = fetch::Client::new().map_err(|e| format!("Error starting fetch client: {:?}", e))?; // create miner let initial_min_gas_price = cmd.gas_pricer_conf.initial_min(); - let miner = Miner::new(cmd.miner_options, cmd.gas_pricer_conf.to_gas_pricer(fetch.clone()), &spec, Some(account_provider.clone())); + let miner = Miner::new(cmd.miner_options, cmd.gas_pricer_conf.to_gas_pricer(fetch.clone(), cpu_pool.clone()), &spec, Some(account_provider.clone())); miner.set_author(cmd.miner_extras.author); miner.set_gas_floor_target(cmd.miner_extras.gas_floor_target); miner.set_gas_ceil_target(cmd.miner_extras.gas_ceil_target); @@ -702,7 +708,7 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) Arc::downgrade(&(service.client() as Arc)), Arc::downgrade(&sync_provider), update_policy, - hash_fetch::Client::with_fetch(contract_client.clone(), updater_fetch, event_loop.remote()) + hash_fetch::Client::with_fetch(contract_client.clone(), cpu_pool.clone(), updater_fetch, event_loop.remote()) ); service.add_notify(updater.clone()); @@ -738,7 +744,7 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) let sync_status = Arc::new(SyncStatus(sync, client, net_conf)); let node_health = node_health::NodeHealth::new( sync_status.clone(), - node_health::TimeChecker::new(&cmd.ntp_servers, fetch.pool()), + node_health::TimeChecker::new(&cmd.ntp_servers, cpu_pool.clone()), event_loop.remote(), ); (node_health.clone(), dapps::Dependencies { @@ -746,6 +752,7 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) node_health, contract_client, fetch: fetch.clone(), + pool: cpu_pool.clone(), signer: signer_service.clone(), ui_address: cmd.ui_conf.redirection_address(), }) @@ -773,6 +780,7 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) dapps_address: cmd.dapps_conf.address(cmd.http_conf.address()), ws_address: cmd.ws_conf.address(), fetch: fetch.clone(), + pool: cpu_pool.clone(), remote: event_loop.remote(), whisper_rpc: whisper_factory, gas_price_percentile: cmd.gas_price_percentile, diff --git a/price-info/Cargo.toml b/price-info/Cargo.toml index 459bccc1dc4..89d0e5474c0 100644 --- a/price-info/Cargo.toml +++ b/price-info/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Parity Technologies "] [dependencies] fetch = { path = "../util/fetch" } futures = "0.1" +futures-cpupool = "0.1" log = "0.3" serde_json = "1.0" diff --git a/price-info/src/lib.rs b/price-info/src/lib.rs index ec3c363a42d..5a47034e8e9 100644 --- a/price-info/src/lib.rs +++ b/price-info/src/lib.rs @@ -19,6 +19,7 @@ //! A simple client to get the current ETH price using an external API. extern crate futures; +extern crate futures_cpupool; extern crate serde_json; #[macro_use] @@ -33,6 +34,7 @@ use std::io::Read; use fetch::{Client as FetchClient, Fetch}; use futures::Future; +use futures_cpupool::CpuPool; use serde_json::Value; /// Current ETH price information. @@ -65,6 +67,7 @@ impl From for Error { /// A client to get the current ETH price using an external API. pub struct Client { + pool: CpuPool, api_endpoint: String, fetch: F, } @@ -85,14 +88,14 @@ impl cmp::PartialEq for Client { impl Client { /// Creates a new instance of the `Client` given a `fetch::Client`. - pub fn new(fetch: F) -> Client { + pub fn new(fetch: F, pool: CpuPool) -> Client { let api_endpoint = "https://api.etherscan.io/api?module=stats&action=ethprice".to_owned(); - Client { api_endpoint, fetch } + Client { pool, api_endpoint, fetch } } /// Gets the current ETH price and calls `set_price` with the result. pub fn get(&self, set_price: G) { - self.fetch.process_and_forget(self.fetch.fetch(&self.api_endpoint) + self.pool.spawn(self.fetch.fetch(&self.api_endpoint) .map_err(|err| Error::Fetch(err)) .and_then(move |mut response| { if !response.is_success() { @@ -121,7 +124,7 @@ impl Client { warn!("Failed to auto-update latest ETH price: {:?}", err); err }) - ); + ).forget(); } } diff --git a/rpc/src/v1/impls/light/parity_set.rs b/rpc/src/v1/impls/light/parity_set.rs index 6b8f0dc4d7f..25df2962517 100644 --- a/rpc/src/v1/impls/light/parity_set.rs +++ b/rpc/src/v1/impls/light/parity_set.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use ethsync::ManageNetwork; use fetch::Fetch; +use futures_cpupool::CpuPool; use hash::keccak_buffer; use jsonrpc_core::{Result, BoxFuture}; @@ -36,15 +37,17 @@ pub struct ParitySetClient { net: Arc, dapps: Option>, fetch: F, + pool: CpuPool } impl ParitySetClient { /// Creates new `ParitySetClient` with given `Fetch`. - pub fn new(net: Arc, dapps: Option>, fetch: F) -> Self { + pub fn new(net: Arc, dapps: Option>, fetch: F, p: CpuPool) -> Self { ParitySetClient { net: net, dapps: dapps, fetch: fetch, + pool: p } } } @@ -125,14 +128,14 @@ impl ParitySet for ParitySetClient { } fn hash_content(&self, url: String) -> BoxFuture { - self.fetch.process(self.fetch.fetch(&url).then(move |result| { + Box::new(self.pool.spawn(self.fetch.fetch(&url).then(move |result| { result .map_err(errors::fetch) .and_then(|response| { keccak_buffer(&mut io::BufReader::new(response)).map_err(errors::fetch) }) .map(Into::into) - })) + }))) } fn dapps_refresh(&self) -> Result { diff --git a/rpc/src/v1/impls/parity_set.rs b/rpc/src/v1/impls/parity_set.rs index 4864e780611..1e6a84c1b74 100644 --- a/rpc/src/v1/impls/parity_set.rs +++ b/rpc/src/v1/impls/parity_set.rs @@ -23,6 +23,7 @@ use ethcore::client::MiningBlockChainClient; use ethcore::mode::Mode; use ethsync::ManageNetwork; use fetch::{self, Fetch}; +use futures_cpupool::CpuPool; use hash::keccak_buffer; use updater::{Service as UpdateService}; @@ -41,6 +42,7 @@ pub struct ParitySetClient { net: Arc, dapps: Option>, fetch: F, + pool: CpuPool, eip86_transition: u64, } @@ -55,6 +57,7 @@ impl ParitySetClient net: &Arc, dapps: Option>, fetch: F, + pool: CpuPool ) -> Self { ParitySetClient { client: client.clone(), @@ -63,6 +66,7 @@ impl ParitySetClient net: net.clone(), dapps: dapps, fetch: fetch, + pool: pool, eip86_transition: client.eip86_transition(), } } @@ -166,14 +170,14 @@ impl ParitySet for ParitySetClient where } fn hash_content(&self, url: String) -> BoxFuture { - self.fetch.process(self.fetch.fetch(&url).then(move |result| { + Box::new(self.pool.spawn(self.fetch.fetch(&url).then(move |result| { result .map_err(errors::fetch) .and_then(|response| { keccak_buffer(&mut io::BufReader::new(response)).map_err(errors::fetch) }) .map(Into::into) - })) + }))) } fn dapps_refresh(&self) -> Result { diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index d8ddc485dfc..57d745d19c5 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -8,7 +8,6 @@ authors = ["Parity Technologies "] [dependencies] futures = "0.1" -futures-cpupool = "0.1" hyper = "0.11" hyper-rustls = "0.11" log = "0.4" diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 542cd65152d..049adfa1fc2 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -25,7 +25,6 @@ use std::time::Duration; use futures::{self, Future, Async, Sink, Stream}; use futures::future::{self, Either}; -use futures_cpupool::CpuPool; use futures::sync::{mpsc, oneshot}; use parking_lot::{Condvar, Mutex}; @@ -61,20 +60,6 @@ pub trait Fetch: Clone + Send + Sync + 'static { /// Result type type Result: Future + Send + 'static; - /// Spawn the future in context of this `Fetch` thread pool. - fn process(&self, f: F) -> BoxFuture - where F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static; - - /// Spawn the future in context of this `Fetch` thread pool as - /// "fire and forget", i.e. dropping this future without canceling - /// the underlying future. - fn process_and_forget(&self, f: F) - where F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static; - /// Fetch URL and get a future for the result. /// Supports aborting the request in the middle of execution. fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result; @@ -109,7 +94,6 @@ impl Proto { /// Fetch client #[derive(Clone)] pub struct Client { - pool: CpuPool, tx_proto: mpsc::Sender, limit: Option } @@ -136,11 +120,7 @@ impl Client { return Err(e.into()) } - Ok(Client { - pool: CpuPool::new(4), - tx_proto: tx_proto, - limit: Some(64 * 1024 * 1024) - }) + Ok(Client { tx_proto: tx_proto, limit: Some(64 * 1024 * 1024) }) } fn background_thread(start: StartupCond, @@ -239,11 +219,6 @@ impl Client { pub fn set_limit(&mut self, limit: Option) { self.limit = limit; } - - /// Returns a handle to underlying CpuPool of this client. - pub fn pool(&self) -> CpuPool { - self.pool.clone() - } } impl Fetch for Client { @@ -277,22 +252,6 @@ impl Fetch for Client { }); Box::new(future) } - - fn process(&self, f: F) -> BoxFuture - where F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static - { - Box::new(self.pool.spawn(f)) - } - - fn process_and_forget(&self, f: F) - where F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static - { - self.pool.spawn(f).forget() - } } fn redirect_location(u: Url, r: &hyper::Response) -> Option { diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index 23a85e0da8b..a51352bb70a 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -22,7 +22,6 @@ extern crate log; extern crate futures; -extern crate futures_cpupool; extern crate parking_lot; extern crate hyper; From ec9d156c7af41c65dcaa2c91a0511b5598f38fa2 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 9 Mar 2018 14:34:28 +0100 Subject: [PATCH 05/15] Adapt code to API changes and fix tests. --- Cargo.lock | 18 +- dapps/src/apps/fetcher/installers.rs | 2 +- dapps/src/apps/fetcher/mod.rs | 6 +- dapps/src/handlers/fetch.rs | 6 +- dapps/src/web.rs | 10 +- hash-fetch/Cargo.toml | 2 + hash-fetch/src/client.rs | 93 ++-- hash-fetch/src/lib.rs | 1 + parity/configuration.rs | 31 +- parity/params.rs | 2 +- parity/rpc_apis.rs | 4 +- parity/run.rs | 5 +- price-info/Cargo.toml | 2 + price-info/src/lib.rs | 71 +-- rpc/src/v1/impls/light/parity_set.rs | 16 +- rpc/src/v1/impls/parity_set.rs | 12 +- updater/src/updater.rs | 6 +- util/fetch/Cargo.toml | 1 + util/fetch/src/client.rs | 638 ++++++++++++--------------- util/fetch/src/lib.rs | 5 +- 20 files changed, 466 insertions(+), 465 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1670909cb7..cd04365888e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,6 +952,7 @@ name = "fetch" version = "0.1.0" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1013,6 +1014,14 @@ dependencies = [ "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "futures-timer" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "gcc" version = "0.3.54" @@ -2038,6 +2047,7 @@ dependencies = [ "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2047,6 +2057,7 @@ dependencies = [ "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2428,9 +2439,11 @@ dependencies = [ "fetch 0.1.0", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3642,6 +3655,7 @@ dependencies = [ "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0bab5b5e94f5c31fc764ba5dd9ad16568aae5d4825538c01d6bca680c9bf94a7" "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +"checksum futures-timer 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5cedfe9b6dc756220782cc1ba5bcb1fa091cdcba155e40d3556159c3db58043" "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" "checksum getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "65922871abd2f101a2eb0eaebadc66668e54a87ad9c3dd82520b5f86ede5eff9" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" @@ -3845,8 +3859,8 @@ dependencies = [ "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum wasm-utils 0.1.0 (git+https://github.com/paritytech/wasm-utils)" = "" "checksum wasmi 0.0.0 (git+https://github.com/paritytech/wasmi)" = "" -"checksum webpki 0.18.0-alpha (registry+https://github.com/rust-lang/crates.io-index)" = "c84d6c11545f0a9afd8757628563fe3941bd13194e30ee106d647d41ba0f7e45" -"checksum webpki-roots 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "edbd75d6abf044ef0c9d7ec92b9e8c518bcd93a15bb7bd9a92239e035248fc17" +"checksum webpki 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e1622384bcb5458c6a3e3fa572f53ea8fef1cc85e535a2983dea87e9154fac2" +"checksum webpki-roots 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "155d4060e5befdf3a6076bd28c22513473d9900b763c9e4521acc6f78a75415c" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/dapps/src/apps/fetcher/installers.rs b/dapps/src/apps/fetcher/installers.rs index 8ca4377b8fa..2c067d493c9 100644 --- a/dapps/src/apps/fetcher/installers.rs +++ b/dapps/src/apps/fetcher/installers.rs @@ -54,7 +54,7 @@ fn write_response_and_check_hash( // Now write the response let mut file = io::BufWriter::new(fs::File::create(&content_path)?); - let mut reader = io::BufReader::new(response); + let mut reader = io::BufReader::new(fetch::BodyReader::new(response)); io::copy(&mut reader, &mut file)?; file.flush()?; diff --git a/dapps/src/apps/fetcher/mod.rs b/dapps/src/apps/fetcher/mod.rs index 981437ffbf4..7362830125e 100644 --- a/dapps/src/apps/fetcher/mod.rs +++ b/dapps/src/apps/fetcher/mod.rs @@ -216,7 +216,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), ) }, URLHintResult::GithubDapp(content) => { @@ -233,7 +233,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), ) }, URLHintResult::Content(content) => { @@ -250,7 +250,7 @@ impl Endpoint for ContentFetcher { ), self.embeddable_on.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), ) }, }; diff --git a/dapps/src/handlers/fetch.rs b/dapps/src/handlers/fetch.rs index 079feb1340c..179bb4884c7 100644 --- a/dapps/src/handlers/fetch.rs +++ b/dapps/src/handlers/fetch.rs @@ -36,7 +36,7 @@ const FETCH_TIMEOUT: u64 = 300; pub enum ValidatorResponse { Local(local::Dapp), - Streaming(StreamingHandler), + Streaming(StreamingHandler), } pub trait ContentValidator: Sized + Send + 'static { @@ -253,7 +253,7 @@ impl ContentFetcherHandler { installer: H, embeddable_on: Embeddable, fetch: F, - pool: CpuPool + pool: CpuPool, ) -> Self { let fetch_control = FetchControl::default(); let errors = Errors { embeddable_on }; @@ -295,7 +295,7 @@ impl ContentFetcherHandler { ) -> Box + Send> { // Start fetching the content let pool2 = pool.clone(); - let future = fetch.fetch_with_abort(url, abort.into()).then(move |result| { + let future = fetch.fetch(url, abort.into()).then(move |result| { trace!(target: "dapps", "Fetching content finished. Starting validation: {:?}", result); Ok(match result { Ok(response) => match installer.validate_and_install(response) { diff --git a/dapps/src/web.rs b/dapps/src/web.rs index 8b4d95d5071..86c0ac28d6c 100644 --- a/dapps/src/web.rs +++ b/dapps/src/web.rs @@ -36,7 +36,7 @@ pub struct Web { embeddable_on: Embeddable, web_proxy_tokens: Arc, fetch: F, - pool: CpuPool + pool: CpuPool, } impl Web { @@ -44,13 +44,13 @@ impl Web { embeddable_on: Embeddable, web_proxy_tokens: Arc, fetch: F, - pool: CpuPool + pool: CpuPool, ) -> Box { Box::new(Web { embeddable_on, web_proxy_tokens, fetch, - pool + pool, }) } @@ -133,7 +133,7 @@ impl Endpoint for Web { }, self.embeddable_on.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), )) } } @@ -151,7 +151,7 @@ impl ContentValidator for WebInstaller { let is_html = response.is_html(); let mime = response.content_type().unwrap_or(mime::TEXT_HTML); let mut handler = StreamingHandler::new( - response, + fetch::BodyReader::new(response), status, mime, self.embeddable_on, diff --git a/hash-fetch/Cargo.toml b/hash-fetch/Cargo.toml index 4c013e1dbb6..4aa16312603 100644 --- a/hash-fetch/Cargo.toml +++ b/hash-fetch/Cargo.toml @@ -26,4 +26,6 @@ ethabi-derive = "5.0" ethabi-contract = "5.0" [dev-dependencies] +hyper = "0.11" parking_lot = "0.5" +url = "1" diff --git a/hash-fetch/src/client.rs b/hash-fetch/src/client.rs index 4512f817e85..751beb3abf7 100644 --- a/hash-fetch/src/client.rs +++ b/hash-fetch/src/client.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use std::path::PathBuf; use hash::keccak_buffer; -use fetch::{Fetch, Response, Error as FetchError, Client as FetchClient}; +use fetch::{self, Fetch}; use futures_cpupool::CpuPool; use futures::{Future, IntoFuture}; use parity_reactor::Remote; @@ -38,7 +38,7 @@ pub trait HashFetch: Send + Sync + 'static { /// 2. `on_done` - callback function invoked when the content is ready (or there was error during fetch) /// /// This function may fail immediately when fetch cannot be initialized or content cannot be resolved. - fn fetch(&self, hash: H256, on_done: Box) + Send>); + fn fetch(&self, hash: H256, abort: fetch::Abort, on_done: Box) + Send>); } /// Hash-fetching error. @@ -58,7 +58,7 @@ pub enum Error { /// IO Error while validating hash. IO(io::Error), /// Error during fetch. - Fetch(FetchError), + Fetch(fetch::Error), } #[cfg(test)] @@ -78,8 +78,8 @@ impl PartialEq for Error { } } -impl From for Error { - fn from(error: FetchError) -> Self { +impl From for Error { + fn from(error: fetch::Error) -> Self { Error::Fetch(error) } } @@ -90,14 +90,9 @@ impl From for Error { } } -fn validate_hash(path: PathBuf, hash: H256, result: Result) -> Result { - let response = result?; - if !response.is_success() { - return Err(Error::InvalidStatus); - } - +fn validate_hash(path: PathBuf, hash: H256, body: fetch::BodyReader) -> Result { // Read the response - let mut reader = io::BufReader::new(response); + let mut reader = io::BufReader::new(body); let mut writer = io::BufWriter::new(fs::File::create(&path)?); io::copy(&mut reader, &mut writer)?; writer.flush()?; @@ -113,7 +108,7 @@ fn validate_hash(path: PathBuf, hash: H256, result: Result } /// Default Hash-fetching client using on-chain contract to resolve hashes to URLs. -pub struct Client { +pub struct Client { pool: CpuPool, contract: URLHintContract, fetch: F, @@ -124,7 +119,7 @@ pub struct Client { impl Client { /// Creates new instance of the `Client` given on-chain contract client and task runner. pub fn new(contract: Arc>, pool: CpuPool, remote: Remote) -> Self { - Client::with_fetch(contract, pool, FetchClient::new().unwrap(), remote) + Client::with_fetch(contract, pool, fetch::Client::new().unwrap(), remote) } } @@ -142,7 +137,7 @@ impl Client { } impl HashFetch for Client { - fn fetch(&self, hash: H256, on_done: Box) + Send>) { + fn fetch(&self, hash: H256, abort: fetch::Abort, on_done: Box) + Send>) { debug!(target: "fetch", "Fetching: {:?}", hash); let random_path = self.random_path.clone(); @@ -165,18 +160,26 @@ impl HashFetch for Client { .into_future() .and_then(move |url| { debug!(target: "fetch", "Resolved {:?} to {:?}. Fetching...", hash, url); - pool.spawn(remote_fetch.fetch(&url).then(move |result| { - debug!(target: "fetch", "Content fetched, validating hash ({:?})", hash); - let path = random_path(); - let res = validate_hash(path.clone(), hash, result); - if let Err(ref err) = res { - trace!(target: "fetch", "Error: {:?}", err); - // Remove temporary file in case of error - let _ = fs::remove_file(&path); - } - res - })) + remote_fetch.fetch(&url, abort).from_err() }) + .and_then(move |response| { + if !response.is_success() { + Err(Error::InvalidStatus) + } else { + Ok(response) + } + }) + .and_then(move |response| pool.spawn_fn(move || { + debug!(target: "fetch", "Content fetched, validating hash ({:?})", hash); + let path = random_path(); + let res = validate_hash(path.clone(), hash, fetch::BodyReader::new(response)); + if let Err(ref err) = res { + trace!(target: "fetch", "Error: {:?}", err); + // Remove temporary file in case of error + let _ = fs::remove_file(&path); + } + res + })) .then(move |res| { on_done(res); Ok(()) as Result<(), ()> }); self.remote.spawn(future); @@ -197,14 +200,19 @@ fn random_temp_path() -> PathBuf { #[cfg(test)] mod tests { + extern crate hyper; + extern crate url; use rustc_hex::FromHex; use std::sync::{Arc, mpsc}; use parking_lot::Mutex; use futures::future; + use futures_cpupool::CpuPool; use fetch::{self, Fetch}; use parity_reactor::Remote; use urlhint::tests::{FakeRegistrar, URLHINT}; use super::{Error, Client, HashFetch, random_temp_path}; + use self::hyper::StatusCode; + use self::url::Url; #[derive(Clone)] @@ -215,17 +223,13 @@ mod tests { impl Fetch for FakeFetch { type Result = future::Ok; - fn new() -> Result where Self: Sized { - Ok(FakeFetch { return_success: true }) - } - - fn fetch_with_abort(&self, url: &str, _abort: fetch::Abort) -> Self::Result { + fn fetch(&self, url: &str, abort: fetch::Abort) -> Self::Result { assert_eq!(url, "https://parity.io/assets/images/ethcore-black-horizontal.png"); + let u = Url::parse(url).unwrap(); future::ok(if self.return_success { - let cursor = ::std::io::Cursor::new(b"result"); - fetch::Response::from_reader(cursor) + fetch::client::Response::new(u, hyper::Response::new().with_body(&b"result"[..]), abort) } else { - fetch::Response::not_found() + fetch::client::Response::new(u, hyper::Response::new().with_status(StatusCode::NotFound), abort) }) } } @@ -244,11 +248,11 @@ mod tests { // given let contract = Arc::new(FakeRegistrar::new()); let fetch = FakeFetch { return_success: false }; - let client = Client::with_fetch(contract.clone(), fetch, Remote::new_sync()); + let client = Client::with_fetch(contract.clone(), CpuPool::new(1), fetch, Remote::new_sync()); // when let (tx, rx) = mpsc::channel(); - client.fetch(2.into(), Box::new(move |result| { + client.fetch(2.into(), Default::default(), Box::new(move |result| { tx.send(result).unwrap(); })); @@ -262,11 +266,11 @@ mod tests { // given let registrar = Arc::new(registrar()); let fetch = FakeFetch { return_success: false }; - let client = Client::with_fetch(registrar.clone(), fetch, Remote::new_sync()); + let client = Client::with_fetch(registrar.clone(), CpuPool::new(1), fetch, Remote::new_sync()); // when let (tx, rx) = mpsc::channel(); - client.fetch(2.into(), Box::new(move |result| { + client.fetch(2.into(), Default::default(), Box::new(move |result| { tx.send(result).unwrap(); })); @@ -280,14 +284,14 @@ mod tests { // given let registrar = Arc::new(registrar()); let fetch = FakeFetch { return_success: true }; - let mut client = Client::with_fetch(registrar.clone(), fetch, Remote::new_sync()); + let mut client = Client::with_fetch(registrar.clone(), CpuPool::new(1), fetch, Remote::new_sync()); let path = random_temp_path(); let path2 = path.clone(); client.random_path = Arc::new(move || path2.clone()); // when let (tx, rx) = mpsc::channel(); - client.fetch(2.into(), Box::new(move |result| { + client.fetch(2.into(), Default::default(), Box::new(move |result| { tx.send(result).unwrap(); })); @@ -303,16 +307,17 @@ mod tests { // given let registrar = Arc::new(registrar()); let fetch = FakeFetch { return_success: true }; - let client = Client::with_fetch(registrar.clone(), fetch, Remote::new_sync()); + let client = Client::with_fetch(registrar.clone(), CpuPool::new(1), fetch, Remote::new_sync()); // when let (tx, rx) = mpsc::channel(); - client.fetch("0x06b0a4f426f6713234b2d4b2468640bc4e0bb72657a920ad24c5087153c593c8".into(), Box::new(move |result| { - tx.send(result).unwrap(); - })); + client.fetch("0x06b0a4f426f6713234b2d4b2468640bc4e0bb72657a920ad24c5087153c593c8".into(), + Default::default(), + Box::new(move |result| { tx.send(result).unwrap(); })); // then let result = rx.recv().unwrap(); assert!(result.is_ok(), "Should return path, got: {:?}", result); } } + diff --git a/hash-fetch/src/lib.rs b/hash-fetch/src/lib.rs index be9df25f87b..6e74982bb84 100644 --- a/hash-fetch/src/lib.rs +++ b/hash-fetch/src/lib.rs @@ -48,3 +48,4 @@ mod client; pub mod urlhint; pub use client::{HashFetch, Client, Error}; +pub use fetch::Abort; diff --git a/parity/configuration.rs b/parity/configuration.rs index 9632ce93555..9898f358d73 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -939,6 +939,7 @@ impl Configuration { _ => return Err("Invalid value for `--releases-track`. See `--help` for more information.".into()), }, path: default_hypervisor_path(), + max_size: 128 * 1024 * 1024, }) } @@ -1389,7 +1390,8 @@ mod tests { require_consensus: true, filter: UpdateFilter::Critical, track: ReleaseTrack::Unknown, - path: default_hypervisor_path() + path: default_hypervisor_path(), + max_size: 128 * 1024 * 1024, }, mode: Default::default(), tracing: Default::default(), @@ -1459,9 +1461,30 @@ mod tests { let conf3 = parse(&["parity", "--auto-update=xxx"]); // then - assert_eq!(conf0.update_policy().unwrap(), UpdatePolicy{enable_downloading: true, require_consensus: true, filter: UpdateFilter::Critical, track: ReleaseTrack::Testing, path: default_hypervisor_path()}); - assert_eq!(conf1.update_policy().unwrap(), UpdatePolicy{enable_downloading: true, require_consensus: false, filter: UpdateFilter::All, track: ReleaseTrack::Unknown, path: default_hypervisor_path()}); - assert_eq!(conf2.update_policy().unwrap(), UpdatePolicy{enable_downloading: false, require_consensus: true, filter: UpdateFilter::All, track: ReleaseTrack::Beta, path: default_hypervisor_path()}); + assert_eq!(conf0.update_policy().unwrap(), UpdatePolicy { + enable_downloading: true, + require_consensus: true, + filter: UpdateFilter::Critical, + track: ReleaseTrack::Testing, + path: default_hypervisor_path(), + max_size: 128 * 1024 * 1024, + }); + assert_eq!(conf1.update_policy().unwrap(), UpdatePolicy { + enable_downloading: true, + require_consensus: false, + filter: UpdateFilter::All, + track: ReleaseTrack::Unknown, + path: default_hypervisor_path(), + max_size: 128 * 1024 * 1024, + }); + assert_eq!(conf2.update_policy().unwrap(), UpdatePolicy { + enable_downloading: false, + require_consensus: true, + filter: UpdateFilter::All, + track: ReleaseTrack::Beta, + path: default_hypervisor_path(), + max_size: 128 * 1024 * 1024, + }); assert!(conf3.update_policy().is_err()); } diff --git a/parity/params.rs b/parity/params.rs index 9d3eee84569..4b5cd940937 100644 --- a/parity/params.rs +++ b/parity/params.rs @@ -251,7 +251,7 @@ impl GasPricerConfig { recalibration_period: recalibration_period, }, fetch, - p + p, ) } } diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index 1ebfe04921f..5ba18dc09bf 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -357,7 +357,7 @@ impl FullDependencies { &self.net_service, self.dapps_service.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), ).to_delegate()) }, Api::Traces => { @@ -568,7 +568,7 @@ impl LightDependencies { self.sync.clone(), self.dapps_service.clone(), self.fetch.clone(), - self.pool.clone() + self.pool.clone(), ).to_delegate()) }, Api::Traces => { diff --git a/parity/run.rs b/parity/run.rs index 7e913bbea5f..2575122dc7e 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -700,10 +700,7 @@ pub fn execute_impl(cmd: RunCmd, can_restart: bool, logger: Arc) let contract_client = Arc::new(::dapps::FullRegistrar::new(client.clone())); // the updater service - let mut updater_fetch = fetch.clone(); - // parity binaries should be smaller than 128MB - updater_fetch.set_limit(Some(128 * 1024 * 1024)); - + let updater_fetch = fetch.clone(); let updater = Updater::new( Arc::downgrade(&(service.client() as Arc)), Arc::downgrade(&sync_provider), diff --git a/price-info/Cargo.toml b/price-info/Cargo.toml index 89d0e5474c0..198cccb98bc 100644 --- a/price-info/Cargo.toml +++ b/price-info/Cargo.toml @@ -14,4 +14,6 @@ log = "0.3" serde_json = "1.0" [dev-dependencies] +hyper = "0.11" parking_lot = "0.5" +url = "1" diff --git a/price-info/src/lib.rs b/price-info/src/lib.rs index 5a47034e8e9..c065aea9610 100644 --- a/price-info/src/lib.rs +++ b/price-info/src/lib.rs @@ -30,10 +30,11 @@ pub extern crate fetch; use std::cmp; use std::fmt; use std::io; -use std::io::Read; +use std::str; use fetch::{Client as FetchClient, Fetch}; -use futures::Future; +use futures::{Future, Stream}; +use futures::future::{self, Either}; use futures_cpupool::CpuPool; use serde_json::Value; @@ -50,7 +51,7 @@ pub enum Error { /// The API returned an unexpected status code. StatusCode(&'static str), /// The API returned an unexpected status content. - UnexpectedResponse(String), + UnexpectedResponse(Option), /// There was an error when trying to reach the API. Fetch(fetch::Error), /// IO error when reading API response. @@ -95,16 +96,18 @@ impl Client { /// Gets the current ETH price and calls `set_price` with the result. pub fn get(&self, set_price: G) { - self.pool.spawn(self.fetch.fetch(&self.api_endpoint) - .map_err(|err| Error::Fetch(err)) - .and_then(move |mut response| { + let future = self.fetch.fetch(&self.api_endpoint, fetch::Abort::default()) + .from_err() + .and_then(|response| { if !response.is_success() { - return Err(Error::StatusCode(response.status().canonical_reason().unwrap_or("unknown"))); + let s = Error::StatusCode(response.status().canonical_reason().unwrap_or("unknown")); + return Either::A(future::err(s)); } - let mut result = String::new(); - response.read_to_string(&mut result)?; - - let value: Option = serde_json::from_str(&result).ok(); + Either::B(response.concat2().from_err()) + }) + .map(move |body| { + let body_str = str::from_utf8(&body).ok(); + let value: Option = body_str.and_then(|s| serde_json::from_str(s).ok()); let ethusd = value .as_ref() @@ -117,19 +120,21 @@ impl Client { set_price(PriceInfo { ethusd }); Ok(()) }, - None => Err(Error::UnexpectedResponse(result)), + None => Err(Error::UnexpectedResponse(body_str.map(From::from))), } }) .map_err(|err| { warn!("Failed to auto-update latest ETH price: {:?}", err); err - }) - ).forget(); + }); + self.pool.spawn(future).forget() } } #[cfg(test)] mod test { + extern crate hyper; + extern crate url; extern crate parking_lot; use self::parking_lot::Mutex; @@ -137,43 +142,45 @@ mod test { use std::sync::atomic::{AtomicBool, Ordering}; use fetch; use fetch::Fetch; - use futures; - use futures::future::{Future, FutureResult}; + use futures_cpupool::CpuPool; + use futures::future::{self, FutureResult}; use Client; + use self::hyper::StatusCode; + use self::url::Url; #[derive(Clone)] struct FakeFetch(Option, Arc>); + + impl FakeFetch { + fn new() -> Result { + Ok(FakeFetch(None, Default::default())) + } + } + impl Fetch for FakeFetch { type Result = FutureResult; - fn new() -> Result where Self: Sized { Ok(FakeFetch(None, Default::default())) } - fn fetch_with_abort(&self, url: &str, _abort: fetch::Abort) -> Self::Result { + + fn fetch(&self, url: &str, abort: fetch::Abort) -> Self::Result { assert_eq!(url, "https://api.etherscan.io/api?module=stats&action=ethprice"); + let u = Url::parse(url).unwrap(); let mut val = self.1.lock(); *val = *val + 1; if let Some(ref response) = self.0 { - let data = ::std::io::Cursor::new(response.clone()); - futures::future::ok(fetch::Response::from_reader(data)) + let r = hyper::Response::new().with_body(response.clone()); + future::ok(fetch::client::Response::new(u, r, abort)) } else { - futures::future::ok(fetch::Response::not_found()) + let r = hyper::Response::new().with_status(StatusCode::NotFound); + future::ok(fetch::client::Response::new(u, r, abort)) } } - - // this guarantees that the calls to price_info::Client::get will block for execution - fn process_and_forget(&self, f: F) where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, - { - let _ = f.wait(); - } } fn price_info_ok(response: &str) -> Client { - Client::new(FakeFetch(Some(response.to_owned()), Default::default())) + Client::new(FakeFetch(Some(response.to_owned()), Default::default()), CpuPool::new(1)) } fn price_info_not_found() -> Client { - Client::new(FakeFetch::new().unwrap()) + Client::new(FakeFetch::new().unwrap(), CpuPool::new(1)) } #[test] diff --git a/rpc/src/v1/impls/light/parity_set.rs b/rpc/src/v1/impls/light/parity_set.rs index 25df2962517..a1344fa69b9 100644 --- a/rpc/src/v1/impls/light/parity_set.rs +++ b/rpc/src/v1/impls/light/parity_set.rs @@ -21,7 +21,7 @@ use std::io; use std::sync::Arc; use ethsync::ManageNetwork; -use fetch::Fetch; +use fetch::{self, Fetch}; use futures_cpupool::CpuPool; use hash::keccak_buffer; @@ -37,7 +37,7 @@ pub struct ParitySetClient { net: Arc, dapps: Option>, fetch: F, - pool: CpuPool + pool: CpuPool, } impl ParitySetClient { @@ -47,7 +47,7 @@ impl ParitySetClient { net: net, dapps: dapps, fetch: fetch, - pool: p + pool: p, } } } @@ -128,14 +128,16 @@ impl ParitySet for ParitySetClient { } fn hash_content(&self, url: String) -> BoxFuture { - Box::new(self.pool.spawn(self.fetch.fetch(&url).then(move |result| { + let future = self.fetch.fetch(&url, Default::default()).then(move |result| { result .map_err(errors::fetch) - .and_then(|response| { - keccak_buffer(&mut io::BufReader::new(response)).map_err(errors::fetch) + .and_then(move |response| { + let mut reader = io::BufReader::new(fetch::BodyReader::new(response)); + keccak_buffer(&mut reader).map_err(errors::fetch) }) .map(Into::into) - }))) + }); + Box::new(self.pool.spawn(future)) } fn dapps_refresh(&self) -> Result { diff --git a/rpc/src/v1/impls/parity_set.rs b/rpc/src/v1/impls/parity_set.rs index 1e6a84c1b74..9d071187230 100644 --- a/rpc/src/v1/impls/parity_set.rs +++ b/rpc/src/v1/impls/parity_set.rs @@ -57,7 +57,7 @@ impl ParitySetClient net: &Arc, dapps: Option>, fetch: F, - pool: CpuPool + pool: CpuPool, ) -> Self { ParitySetClient { client: client.clone(), @@ -170,14 +170,16 @@ impl ParitySet for ParitySetClient where } fn hash_content(&self, url: String) -> BoxFuture { - Box::new(self.pool.spawn(self.fetch.fetch(&url).then(move |result| { + let future = self.fetch.fetch(&url, Default::default()).then(move |result| { result .map_err(errors::fetch) - .and_then(|response| { - keccak_buffer(&mut io::BufReader::new(response)).map_err(errors::fetch) + .and_then(move |response| { + let mut reader = io::BufReader::new(fetch::BodyReader::new(response)); + keccak_buffer(&mut reader).map_err(errors::fetch) }) .map(Into::into) - }))) + }); + Box::new(self.pool.spawn(future)) } fn dapps_refresh(&self) -> Result { diff --git a/updater/src/updater.rs b/updater/src/updater.rs index 24096704d70..ba7a42c875b 100644 --- a/updater/src/updater.rs +++ b/updater/src/updater.rs @@ -58,6 +58,8 @@ pub struct UpdatePolicy { pub track: ReleaseTrack, /// Path for the updates to go. pub path: String, + /// Maximum download size. + pub max_size: usize, } impl Default for UpdatePolicy { @@ -68,6 +70,7 @@ impl Default for UpdatePolicy { filter: UpdateFilter::None, track: ReleaseTrack::Unknown, path: Default::default(), + max_size: 128 * 1024 * 1024, } } } @@ -341,7 +344,8 @@ impl Updater { drop(s); let weak_self = self.weak_self.lock().clone(); let f = move |r: Result| if let Some(this) = weak_self.upgrade() { this.fetch_done(r) }; - self.fetcher.fetch(b, Box::new(f)); + let a = fetch::Abort::default().with_max_size(self.update_policy.max_size); + self.fetcher.fetch(b, a, Box::new(f)); } } } diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index 57d745d19c5..f644f0d1906 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Parity Technologies "] [dependencies] futures = "0.1" +futures-timer = "0.1" hyper = "0.11" hyper-rustls = "0.11" log = "0.4" diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 049adfa1fc2..22016c9010d 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -14,120 +14,153 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Fetching - -use std::cmp::min; -use std::{io, error, fmt, mem}; -use std::sync::Arc; -use std::sync::atomic::{self, AtomicBool}; -use std::thread; -use std::time::Duration; - -use futures::{self, Future, Async, Sink, Stream}; -use futures::future::{self, Either}; +use futures::future::{self, Loop}; use futures::sync::{mpsc, oneshot}; -use parking_lot::{Condvar, Mutex}; - -use hyper::{self, Request, Method, StatusCode}; -use hyper::header::{UserAgent, Location, ContentType}; +use futures::{self, Future, Async, Sink, Stream}; +use futures_timer::FutureExt; +use hyper::header::{UserAgent, Location, ContentLength, ContentType}; use hyper::mime::Mime; - +use hyper::{self, Request, Method, StatusCode}; use hyper_rustls; +use parking_lot::{Condvar, Mutex}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::Duration; +use std::{cmp, io, fmt, mem}; use tokio_core::reactor; use url::{self, Url}; -type BoxFuture = Box + Send>; - -/// Fetch abort control -#[derive(Default, Debug, Clone)] -pub struct Abort(Arc); +const MAX_SIZE: usize = 64 * 1024 * 1024; +const MAX_SECS: u64 = 5; +const MAX_REDR: usize = 5; + +/// A handle to abort requests. +/// +/// Requests are either aborted based on reaching thresholds such as +/// maximum response size, timeouts or too many redirects, or else +/// they can be aborted explicitly by the calling code. +#[derive(Clone, Debug)] +pub struct Abort { + abort: Arc, + size: usize, + time: Duration, + redir: usize, +} -impl Abort { - /// Returns `true` if request is aborted. - pub fn is_aborted(&self) -> bool { - self.0.load(atomic::Ordering::SeqCst) +impl Default for Abort { + fn default() -> Abort { + Abort { + abort: Arc::new(AtomicBool::new(false)), + size: MAX_SIZE, + time: Duration::from_secs(MAX_SECS), + redir: MAX_REDR + } } } impl From> for Abort { - fn from(a: Arc) -> Self { - Abort(a) + fn from(a: Arc) -> Abort { + Abort { + abort: a, + size: MAX_SIZE, + time: Duration::from_secs(MAX_SECS), + redir: MAX_REDR + } } } -/// Fetch -pub trait Fetch: Clone + Send + Sync + 'static { - /// Result type - type Result: Future + Send + 'static; +impl Abort { + /// True if `abort` has been invoked. + pub fn is_aborted(&self) -> bool { + self.abort.load(Ordering::SeqCst) + } - /// Fetch URL and get a future for the result. - /// Supports aborting the request in the middle of execution. - fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result; + /// The maximum response body size. + pub fn max_size(&self) -> usize { + self.size + } - /// Fetch URL and get a future for the result. - fn fetch(&self, url: &str) -> Self::Result { - self.fetch_with_abort(url, Default::default()) + /// The maximum total time, including redirects. + pub fn max_duration(&self) -> Duration { + self.time } -} -const THREAD_NAME: &str = "fetch"; -const CLIENT_TIMEOUT_SECONDS: u64 = 5; -const MAX_REDIRECTS: usize = 5; + /// The maximum number of redirects to allow. + pub fn max_redirects(&self) -> usize { + self.redir + } -type TxResponse = oneshot::Sender>; -type RxResponse = oneshot::Receiver>; -type StartupCond = Arc<(Mutex>, Condvar)>; + /// Mark as aborted. + pub fn abort(&self) { + self.abort.store(true, Ordering::SeqCst) + } -// `Proto`col values are sent over an mpsc channel from clients to -// their shared background thread with a tokio core and hyper cient inside. -enum Proto { - Request(Url, hyper::Request, TxResponse, usize), - Quit // terminates background thread -} + /// Set the maximum reponse body size. + pub fn with_max_size(self, n: usize) -> Abort { + Abort { size: n, .. self } + } + + /// Set the maximum duration (including redirects). + pub fn with_max_duration(self, d: Duration) -> Abort { + Abort { time: d, .. self } + } -impl Proto { - fn is_quit(&self) -> bool { - if let Proto::Quit = *self { true } else { false } + /// Set the maximum number of redirects to follow. + pub fn with_max_redirects(self, n: usize) -> Abort { + Abort { redir: n, .. self } } } -/// Fetch client -#[derive(Clone)] +/// Types which retrieve content from some URL. +pub trait Fetch: Clone + Send + Sync + 'static { + /// The result future. + type Result: Future + Send + 'static; + + /// Get content from some URL. + fn fetch(&self, url: &str, abort: Abort) -> Self::Result; +} + +type TxResponse = oneshot::Sender>; +type StartupCond = Arc<(Mutex>, Condvar)>; +type ChanItem = (Url, Abort, TxResponse); + +/// An implementation of `Fetch` using a `hyper` client. +// Due to the `Send` bound of `Fetch` we spawn a background thread for +// actual request/response processing as `hyper::Client` itself does +// not implement `Send` currently. +#[derive(Debug, Clone)] pub struct Client { - tx_proto: mpsc::Sender, - limit: Option + core: mpsc::Sender, } impl Client { - /// Create a new client which spins up a separate thread running a - /// tokio `Core` and a `hyper::Client`. - /// Clones of this client share the same background thread. + /// Create a new fetch client. + /// + /// This spawns off a background thread for request/response processing. pub fn new() -> Result { let startup_done = Arc::new((Mutex::new(Ok(())), Condvar::new())); let (tx_proto, rx_proto) = mpsc::channel(64); - Client::background_thread(startup_done.clone(), tx_proto.clone(), rx_proto)?; + Client::background_thread(startup_done.clone(), rx_proto)?; let mut guard = startup_done.0.lock(); let startup_result = startup_done.1.wait_for(&mut guard, Duration::from_secs(3)); if startup_result.timed_out() { - error!(target: "fetch", "timeout starting {}", THREAD_NAME); - return Err(Error::Other("timeout starting background thread".into())) + error!(target: "fetch", "timeout starting background thread"); + return Err(Error::BackgroundThreadDead) } if let Err(e) = mem::replace(&mut *guard, Ok(())) { error!(target: "fetch", "error starting background thread: {}", e); return Err(e.into()) } - Ok(Client { tx_proto: tx_proto, limit: Some(64 * 1024 * 1024) }) + Ok(Client { core: tx_proto }) } - fn background_thread(start: StartupCond, - tx_proto: mpsc::Sender, - rx_proto: mpsc::Receiver) -> io::Result> - { - thread::Builder::new().name(THREAD_NAME.into()).spawn(move || { + fn background_thread(start: StartupCond, rx_proto: mpsc::Receiver) -> io::Result> { + thread::Builder::new().name("fetch".into()).spawn(move || { let mut core = match reactor::Core::new() { Ok(c) => c, Err(e) => { @@ -137,124 +170,96 @@ impl Client { } }; let handle = core.handle(); - let client = hyper::Client::configure() + let hyper = hyper::Client::configure() .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())) .build(&core.handle()); start.1.notify_one(); debug!(target: "fetch", "processing requests ..."); - let maxdur = Duration::from_secs(CLIENT_TIMEOUT_SECONDS); - let work = rx_proto.take_while(|item| Ok(!item.is_quit())).for_each(|item| { - if let Proto::Request(url, rq, sender, redir) = item { - trace!(target: "fetch", "new request to {}", url); - let timeout = match reactor::Timeout::new(maxdur, &handle) { - Ok(t) => t, - Err(e) => { - error!(target: "fetch", "failed to create timeout: {}.", e); - return future::err(()) - } - }; - let reschedule = tx_proto.clone(); - let future = client.request(rq).select2(timeout).then(move |rs| { - trace!(target: "fetch", "response received from {}", url); - // When sending responses back over the oneshot channels, we treat - // the possibility that the other end is gone as normal, hence we - // use `unwrap_or(())` and do not error. - match rs { - Ok(Either::A((rs, _))) => { - if let Some(next_url) = redirect_location(url, &rs) { - if redir == 0 { - Either::A(future::ok(sender.send(Err(Error::TooManyRedirects)).unwrap_or(()))) - } else { - let next_req = get(&next_url); - Either::B(reschedule.send(Proto::Request(next_url, next_req, sender, redir - 1)).then(|result| { - if let Err(e) = result { - error!(target: "fetch", "failed to reschedule request: {}", e); - } - // We can not recover from this error. Client code will - // get a `oneshot::Canceled` error since we dropped the - // `oneshot::Sender`. This should not happen as long as - // this thread runs, as with `reschedule` we are - // sending the `Proto` value back to ourselves. - future::ok(()) - })) - } - } else { - Either::A(future::ok(sender.send(Ok(rs)).unwrap_or(()))) + let work = rx_proto.for_each(|(url, abort, sender)| { + trace!(target: "fetch", "new request to {}", url); + if abort.is_aborted() { + return future::ok(sender.send(Err(Error::Aborted)).unwrap_or(())) + } + let ini = (hyper.clone(), url, abort, 0); + let fut = future::loop_fn(ini, |(client, url, abort, redirects)| { + let url2 = url.clone(); + let abort2 = abort.clone(); + client.request(get(&url)) + .map(move |resp| Response::new(url2, resp, abort2)) + .from_err() + .and_then(move |resp| { + if abort.is_aborted() { + debug!(target: "fetch", "fetch of {} aborted", url); + return Err(Error::Aborted) + } + if let Some(next_url) = redirect_location(url, &resp) { + if redirects >= abort.max_redirects() { + return Err(Error::TooManyRedirects) } + Ok(Loop::Continue((client, next_url, abort, redirects + 1))) + } else { + let content_len = resp.headers.get::().cloned(); + if content_len.map(|n| *n > abort.max_size() as u64).unwrap_or(false) { + return Err(Error::SizeLimit) + } + Ok(Loop::Break(resp)) } - Ok(Either::B((_, _))) => Either::A(future::ok(sender.send(Err(Error::Timeout)).unwrap_or(()))), - Err(Either::A((err, _))) => Either::A(future::ok(sender.send(Err(err.into())).unwrap_or(()))), - Err(Either::B((err, _))) => Either::A(future::ok(sender.send(Err(err.into())).unwrap_or(()))), - } + }) + }) + .then(|result| { + future::ok(sender.send(result).unwrap_or(())) }); - handle.spawn(future); - trace!(target: "fetch", "waiting for next request...") - } + handle.spawn(fut); + trace!(target: "fetch", "waiting for next request ..."); future::ok(()) }); if let Err(()) = core.run(work) { error!(target: "fetch", "error while executing future") } - debug!(target: "fetch", "{} background thread finished", THREAD_NAME) + debug!(target: "fetch", "fetch background thread finished") }) } - - /// Close this client by shutting down the background thread. - /// - /// Please note that this will affect all clones of this `Client` as they all - /// share the same background thread. - pub fn close(self) -> Result<(), Error> { - self.tx_proto.clone().send(Proto::Quit).wait() - .map_err(|e| { - error!(target: "fetch", "failed to send quit to background thread: {}", e); - // We can not put `e: SendError` into `Other` as it is not `Send`. - Error::Other("failed to terminate background thread".into()) - })?; - Ok(()) - } - - /// (Un-)set size limit on response body. - pub fn set_limit(&mut self, limit: Option) { - self.limit = limit; - } } impl Fetch for Client { - type Result = BoxFuture; + type Result = Box + Send>; - fn fetch_with_abort(&self, url: &str, abort: Abort) -> Self::Result { + fn fetch(&self, url: &str, abort: Abort) -> Self::Result { debug!(target: "fetch", "fetching: {:?}", url); - + if abort.is_aborted() { + return Box::new(future::err(Error::Aborted)) + } let url: Url = match url.parse() { - Ok(u) => u, - Err(e) => return Box::new(futures::future::err(e.into())) + Ok(u) => u, + Err(e) => return Box::new(future::err(e.into())) }; - - let req = get(&url); - let sender = self.tx_proto.clone(); - let limit = self.limit.clone(); let (tx_res, rx_res) = oneshot::channel(); - let future = sender.send(Proto::Request(url.clone(), req, tx_res, MAX_REDIRECTS)) - .map(|_| rx_res) + let maxdur = abort.max_duration(); + let sender = self.core.clone(); + let future = sender.send((url.clone(), abort, tx_res)) .map_err(|e| { error!(target: "fetch", "failed to schedule request: {}", e); - Error::Other("failed to schedule request".into()) + Error::BackgroundThreadDead }) - .and_then(move |rx_res| { - FetchTask { - url: url, - rx_res: rx_res, - limit: limit, - abort: abort + .and_then(|_| rx_res.map_err(|oneshot::Canceled| Error::BackgroundThreadDead)) + .and_then(future::result) + .timeout(maxdur) + .map_err(|err| { + if let Error::Io(ref e) = err { + if let io::ErrorKind::TimedOut = e.kind() { + return Error::Timeout + } } + err.into() }); Box::new(future) } } -fn redirect_location(u: Url, r: &hyper::Response) -> Option { +// Extract redirect location from response. +fn redirect_location(u: Url, r: &Response) -> Option { use hyper::StatusCode::*; match r.status() { MovedPermanently @@ -262,7 +267,7 @@ fn redirect_location(u: Url, r: &hyper::Response) -> Option { | TemporaryRedirect | Found | SeeOther => { - if let Some(loc) = r.headers().get::() { + if let Some(loc) = r.headers.get::() { u.join(loc).ok() } else { None @@ -272,166 +277,49 @@ fn redirect_location(u: Url, r: &hyper::Response) -> Option { } } +// Build a simple GET request for the given Url. fn get(u: &Url) -> hyper::Request { - let uri = u.as_ref().parse().expect("Every valid URL is aso a URI"); + let uri = u.as_ref().parse().expect("Every valid URL is aso a URI."); let mut rq = Request::new(Method::Get, uri); rq.headers_mut().set(UserAgent::new("Parity Fetch Neo")); rq } -struct FetchTask { - url: Url, - rx_res: RxResponse, - limit: Option, - abort: Abort -} - -impl Future for FetchTask { - type Item = Response; - type Error = Error; - - fn poll(&mut self) -> futures::Poll { - if self.abort.is_aborted() { - debug!(target: "fetch", "Fetch of {:?} aborted.", self.url); - return Err(Error::Aborted); - } - match self.rx_res.poll()? { - Async::Ready(Err(e)) => Err(e.into()), - Async::Ready(Ok(r)) => { - let ctype = r.headers().get::().cloned(); - Ok(Async::Ready(Response { - inner: ResponseInner::Response(r.status(), ctype, BodyReader::new(r.body())), - abort: self.abort.clone(), - limit: self.limit.clone(), - read: 0 - })) - } - Async::NotReady => Ok(Async::NotReady) - } - } -} - -/// Fetch related error cases. -#[derive(Debug)] -pub enum Error { - /// Error produced by hyper. - Hyper(hyper::Error), - /// I/O error - Io(io::Error), - /// URL parse error - Url(url::ParseError), - /// Request aborted - Aborted, - /// Followed too many redirects - TooManyRedirects, - /// Request took too long - Timeout, - /// The background request procesing was canceled - Canceled, - /// Some other error - Other(Box) -} - -impl fmt::Display for Error { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - match *self { - Error::Aborted => write!(fmt, "The request has been aborted."), - Error::Hyper(ref e) => write!(fmt, "{}", e), - Error::Url(ref e) => write!(fmt, "{}", e), - Error::Io(ref e) => write!(fmt, "{}", e), - Error::Other(ref e) => write!(fmt, "{}", e), - Error::TooManyRedirects => write!(fmt, "too many redirects"), - Error::Timeout => write!(fmt, "request timed out"), - Error::Canceled => write!(fmt, "background thread canceled request processing"), - } - } -} - -impl From for Error { - fn from(_: oneshot::Canceled) -> Self { - Error::Canceled - } -} - -impl From for Error { - fn from(e: hyper::Error) -> Self { - Error::Hyper(e) - } -} - -impl From for Error { - fn from(e: io::Error) -> Self { - Error::Io(e) - } -} - -impl From for Error { - fn from(e: url::ParseError) -> Self { - Error::Url(e) - } -} - -enum ResponseInner { - Response(StatusCode, Option, BodyReader), - Reader(Box), - NotFound -} - -impl fmt::Debug for ResponseInner { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ResponseInner::Response(s, ..) => write!(f, "hyper response (status={})", s), - ResponseInner::NotFound => write!(f, "not found"), - ResponseInner::Reader(_) => write!(f, "io reader"), - } - } -} - -/// A fetch response type. +/// An HTTP response. #[derive(Debug)] pub struct Response { - inner: ResponseInner, + url: Url, + status: StatusCode, + headers: hyper::Headers, + body: hyper::Body, abort: Abort, - limit: Option, - read: usize + nread: usize } impl Response { - /// Creates new successfuly response reading from a file. - pub fn from_reader(reader: R) -> Self { - Response { - inner: ResponseInner::Reader(Box::new(reader)), - abort: Abort::default(), - limit: None, - read: 0 - } - } - - /// Creates 404 response (useful for tests) - pub fn not_found() -> Self { + /// Create a new response, wrapping a hyper response. + pub fn new(u: Url, r: hyper::Response, a: Abort) -> Response { Response { - inner: ResponseInner::NotFound, - abort: Abort::default(), - limit: None, - read: 0 + url: u, + status: r.status(), + headers: r.headers().clone(), + body: r.body(), + abort: a, + nread: 0 } } - /// Returns status code of this response. + /// The response status. pub fn status(&self) -> StatusCode { - match self.inner { - ResponseInner::Response(s, ..) => s, - ResponseInner::NotFound => StatusCode::NotFound, - _ => StatusCode::Ok - } + self.status } - /// Returns `true` if response status code is successful. + /// Status code == OK (200)? pub fn is_success(&self) -> bool { self.status() == StatusCode::Ok } - /// Returns `true` if content type of this response is `text/html` + /// Is the content-type text/html? pub fn is_html(&self) -> bool { if let Some(ref mime) = self.content_type() { mime.type_() == "text" && mime.subtype() == "html" @@ -440,56 +328,53 @@ impl Response { } } - /// Returns content type of this response (if present) + /// The conten-type header value. pub fn content_type(&self) -> Option { - if let ResponseInner::Response(_, ref c, _) = self.inner { - c.as_ref().map(|mime| mime.0.clone()) - } else { - None - } + self.headers.get::().map(|ct| ct.0.clone()) } } -impl io::Read for Response { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.abort.is_aborted() { - return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "Fetch aborted.")); - } - - let res = match self.inner { - ResponseInner::Response(_, _, ref mut r) => r.read(buf), - ResponseInner::NotFound => return Ok(0), - ResponseInner::Reader(ref mut r) => r.read(buf) - }; +impl Stream for Response { + type Item = hyper::Chunk; + type Error = Error; - // increase bytes read - if let Ok(read) = res { - self.read += read + fn poll(&mut self) -> futures::Poll, Self::Error> { + if self.abort.is_aborted() { + debug!(target: "fetch", "fetch of {} aborted", self.url); + return Err(Error::Aborted) } - - // check limit - match self.limit { - Some(limit) if limit < self.read => { - return Err(io::Error::new(io::ErrorKind::PermissionDenied, "Size limit reached.")); + match try_ready!(self.body.poll()) { + None => Ok(Async::Ready(None)), + Some(c) => { + if self.nread + c.len() > self.abort.max_size() { + debug!(target: "fetch", "size limit {:?} for {} exceeded", self.abort.max_size(), self.url); + return Err(Error::SizeLimit) + } + self.nread += c.len(); + Ok(Async::Ready(Some(c))) } - _ => {} } - - res } } -// `BodyReader` serves as a bridge from async to sync I/O. It implements -// `io::Read` by repedately waiting for the next `Chunk` of hyper's response `Body`. -struct BodyReader { +/// `BodyReader` serves as an adapter from async to sync I/O. +/// +/// It implements `io::Read` by repedately waiting for the next `Chunk` +/// of hyper's response `Body` which blocks the current thread. +pub struct BodyReader { chunk: hyper::Chunk, body: Option, - offset: usize + offset: usize, } impl BodyReader { - fn new(b: hyper::Body) -> BodyReader { - BodyReader { body: Some(b), chunk: Default::default(), offset: 0 } + /// Create a new body reader for the given response. + pub fn new(r: Response) -> BodyReader { + BodyReader { + body: Some(r.body), + chunk: Default::default(), + offset: 0, + } } } @@ -499,7 +384,7 @@ impl io::Read for BodyReader { while self.body.is_some() { // Can we still read from the current chunk? if self.offset < self.chunk.len() { - let k = min(self.chunk.len() - self.offset, buf.len() - m); + let k = cmp::min(self.chunk.len() - self.offset, buf.len() - m); let c = &self.chunk[self.offset .. self.offset + k]; (&mut buf[m .. m + k]).copy_from_slice(c); self.offset += k; @@ -527,27 +412,80 @@ impl io::Read for BodyReader { } } +/// Fetch error cases. +#[derive(Debug)] +pub enum Error { + /// Hyper gave us an error. + Hyper(hyper::Error), + /// Some I/O error occured. + Io(io::Error), + /// Invalid URLs where attempted to parse. + Url(url::ParseError), + /// Calling code invoked `Abort::abort`. + Aborted, + /// Too many redirects have been encountered. + TooManyRedirects, + /// The maximum duration was reached. + Timeout, + /// The response body is too large. + SizeLimit, + /// The background processing thread does not run. + BackgroundThreadDead, +} + +impl fmt::Display for Error { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match *self { + Error::Aborted => write!(fmt, "The request has been aborted."), + Error::Hyper(ref e) => write!(fmt, "{}", e), + Error::Url(ref e) => write!(fmt, "{}", e), + Error::Io(ref e) => write!(fmt, "{}", e), + Error::BackgroundThreadDead => write!(fmt, "background thread gond"), + Error::TooManyRedirects => write!(fmt, "too many redirects"), + Error::Timeout => write!(fmt, "request timed out"), + Error::SizeLimit => write!(fmt, "size limit reached"), + } + } +} + +impl From for Error { + fn from(e: hyper::Error) -> Self { + Error::Hyper(e) + } +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Error::Io(e) + } +} + +impl From for Error { + fn from(e: url::ParseError) -> Self { + Error::Url(e) + } +} + #[cfg(test)] mod test { use super::*; - use std::io::Read; #[test] fn it_should_fetch() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/drip?numbytes=3&duration=3&delay=1&code=200"); - let mut resp = future.wait().unwrap(); + let abort = Abort::default().with_max_duration(Duration::from_secs(10)); + let future = client.fetch("https://httpbin.org/drip?numbytes=3&duration=3&delay=1&code=200", abort); + let resp = future.wait().unwrap(); assert!(resp.is_success()); - let mut body = Vec::new(); - resp.read_to_end(&mut body).unwrap(); + let body = resp.concat2().wait().unwrap(); assert_eq!(body.len(), 3) } #[test] fn it_should_timeout() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/delay/7"); - match future.wait() { + let abort = Abort::default().with_max_duration(Duration::from_secs(5)); + match client.fetch("https://httpbin.org/delay/7", abort).wait() { Err(Error::Timeout) => {} other => panic!("expected timeout, got {:?}", other) } @@ -556,22 +494,24 @@ mod test { #[test] fn it_should_follow_redirects() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/absolute-redirect/3"); + let abort = Abort::default().with_max_redirects(4).with_max_duration(Duration::from_secs(15)); + let future = client.fetch("https://httpbin.org/absolute-redirect/3", abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_follow_relative_redirects() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/relative-redirect/3"); + let abort = Abort::default().with_max_redirects(4).with_max_duration(Duration::from_secs(15)); + let future = client.fetch("https://httpbin.org/relative-redirect/3", abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_not_follow_too_many_redirects() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/absolute-redirect/100"); - match future.wait() { + let abort = Abort::default().with_max_redirects(3); + match client.fetch("https://httpbin.org/absolute-redirect/4", abort).wait() { Err(Error::TooManyRedirects) => {} other => panic!("expected too many redirects error, got {:?}", other) } @@ -580,22 +520,20 @@ mod test { #[test] fn it_should_read_data() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/bytes/1024"); - let mut resp = future.wait().unwrap(); + let abort = Abort::default(); + let future = client.fetch("https://httpbin.org/bytes/1024", abort); + let resp = future.wait().unwrap(); assert!(resp.is_success()); - let mut body = Vec::new(); - resp.read_to_end(&mut body).unwrap(); - assert_eq!(body.len(), 1024) + assert_eq!(resp.concat2().wait().unwrap().len(), 1024) } #[test] - fn it_should_read_chunked_data() { + fn it_should_not_read_too_much_data() { let client = Client::new().unwrap(); - let future = client.fetch("https://httpbin.org/stream-bytes/1024?chunk_size=19"); - let mut resp = future.wait().unwrap(); - assert!(resp.is_success()); - let mut body = Vec::new(); - resp.read_to_end(&mut body).unwrap(); - assert_eq!(body.len(), 1024) + let abort = Abort::default().with_max_size(3); + match client.fetch("https://httpbin.org/bytes/4", abort).wait() { + Err(Error::SizeLimit) => {} + other => panic!("expected size limit error, got {:?}", other) + } } } diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index a51352bb70a..347e4d94b96 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -21,7 +21,9 @@ #[macro_use] extern crate log; +#[macro_use] extern crate futures; +extern crate futures_timer; extern crate parking_lot; extern crate hyper; @@ -30,7 +32,8 @@ extern crate hyper_rustls; extern crate tokio_core; extern crate url; +/// Fetch client implementation. pub mod client; -pub use self::client::{Client, Fetch, Error, Response, Abort}; +pub use self::client::{Client, Fetch, Error, Response, Abort, BodyReader}; From da1794e40ba90254d3f26cc5ffd7849400d350c1 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Tue, 13 Mar 2018 11:20:12 +0100 Subject: [PATCH 06/15] Use reference counter to stop background thread. On `clone` the counter is incremented, on `drop` decremented. Once 0 we send `None` over the channel, expecting the background thread to end. --- Cargo.lock | 2 +- hash-fetch/src/client.rs | 7 ----- util/fetch/src/client.rs | 64 +++++++++++++++++++++++++++++----------- 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd04365888e..648e20ee7d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ dependencies = [ "ethcore-io 1.11.0", "kvdb 0.1.0", "kvdb-rocksdb 0.1.0", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "stop-guard 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/hash-fetch/src/client.rs b/hash-fetch/src/client.rs index 751beb3abf7..88a9945945f 100644 --- a/hash-fetch/src/client.rs +++ b/hash-fetch/src/client.rs @@ -116,13 +116,6 @@ pub struct Client { random_path: Arc PathBuf + Sync + Send>, } -impl Client { - /// Creates new instance of the `Client` given on-chain contract client and task runner. - pub fn new(contract: Arc>, pool: CpuPool, remote: Remote) -> Self { - Client::with_fetch(contract, pool, fetch::Client::new().unwrap(), remote) - } -} - impl Client { /// Creates new instance of the `Client` given on-chain contract client, fetch service and task runner. pub fn with_fetch(contract: Arc>, pool: CpuPool, fetch: F, remote: Remote) -> Self { diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 22016c9010d..8a4d48579a4 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -24,7 +24,7 @@ use hyper::{self, Request, Method, StatusCode}; use hyper_rustls; use parking_lot::{Condvar, Mutex}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::thread; use std::time::Duration; use std::{cmp, io, fmt, mem}; @@ -54,7 +54,7 @@ impl Default for Abort { abort: Arc::new(AtomicBool::new(false)), size: MAX_SIZE, time: Duration::from_secs(MAX_SECS), - redir: MAX_REDR + redir: MAX_REDR, } } } @@ -65,7 +65,7 @@ impl From> for Abort { abort: a, size: MAX_SIZE, time: Duration::from_secs(MAX_SECS), - redir: MAX_REDR + redir: MAX_REDR, } } } @@ -123,15 +123,38 @@ pub trait Fetch: Clone + Send + Sync + 'static { type TxResponse = oneshot::Sender>; type StartupCond = Arc<(Mutex>, Condvar)>; -type ChanItem = (Url, Abort, TxResponse); +type ChanItem = Option<(Url, Abort, TxResponse)>; /// An implementation of `Fetch` using a `hyper` client. // Due to the `Send` bound of `Fetch` we spawn a background thread for // actual request/response processing as `hyper::Client` itself does // not implement `Send` currently. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Client { core: mpsc::Sender, + refs: Arc, +} + +// When cloning a client we increment the internal reference counter. +impl Clone for Client { + fn clone(&self) -> Client { + self.refs.fetch_add(1, Ordering::SeqCst); + Client { + core: self.core.clone(), + refs: self.refs.clone(), + } + } +} + +// When dropping a client, we decrement the reference counter. +// Once it reaches 0 we terminate the background thread. +impl Drop for Client { + fn drop(&mut self) { + if self.refs.fetch_sub(1, Ordering::SeqCst) == 1 { + // ignore send error as it means the background thread is gone already + let _ = self.core.clone().send(None).wait(); + } + } } impl Client { @@ -156,7 +179,10 @@ impl Client { return Err(e.into()) } - Ok(Client { core: tx_proto }) + Ok(Client { + core: tx_proto, + refs: Arc::new(AtomicUsize::new(1)), + }) } fn background_thread(start: StartupCond, rx_proto: mpsc::Receiver) -> io::Result> { @@ -174,10 +200,10 @@ impl Client { .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())) .build(&core.handle()); - start.1.notify_one(); - debug!(target: "fetch", "processing requests ..."); - - let work = rx_proto.for_each(|(url, abort, sender)| { + let future = rx_proto.take_while(|item| Ok(item.is_some())) + .map(|item| item.expect("`take_while` is only passing on channel items != None")) + .for_each(|(url, abort, sender)| + { trace!(target: "fetch", "new request to {}", url); if abort.is_aborted() { return future::ok(sender.send(Err(Error::Aborted)).unwrap_or(())) @@ -215,7 +241,11 @@ impl Client { trace!(target: "fetch", "waiting for next request ..."); future::ok(()) }); - if let Err(()) = core.run(work) { + + start.1.notify_one(); + + debug!(target: "fetch", "processing requests ..."); + if let Err(()) = core.run(future) { error!(target: "fetch", "error while executing future") } debug!(target: "fetch", "fetch background thread finished") @@ -238,7 +268,7 @@ impl Fetch for Client { let (tx_res, rx_res) = oneshot::channel(); let maxdur = abort.max_duration(); let sender = self.core.clone(); - let future = sender.send((url.clone(), abort, tx_res)) + let future = sender.send(Some((url.clone(), abort, tx_res))) .map_err(|e| { error!(target: "fetch", "failed to schedule request: {}", e); Error::BackgroundThreadDead @@ -279,7 +309,7 @@ fn redirect_location(u: Url, r: &Response) -> Option { // Build a simple GET request for the given Url. fn get(u: &Url) -> hyper::Request { - let uri = u.as_ref().parse().expect("Every valid URL is aso a URI."); + let uri = u.as_ref().parse().expect("Every valid URL is aso a URI."); let mut rq = Request::new(Method::Get, uri); rq.headers_mut().set(UserAgent::new("Parity Fetch Neo")); rq @@ -293,7 +323,7 @@ pub struct Response { headers: hyper::Headers, body: hyper::Body, abort: Abort, - nread: usize + nread: usize, } impl Response { @@ -305,7 +335,7 @@ impl Response { headers: r.headers().clone(), body: r.body(), abort: a, - nread: 0 + nread: 0, } } @@ -393,8 +423,8 @@ impl io::Read for BodyReader { break } } else { - // While in this loop, `self.body` is always defined => wait for the next chunk. - match self.body.take().unwrap().into_future().wait() { + let body = self.body.take().expect("within this loop `self.body` is always defined"); + match body.into_future().wait() { // wait for next chunk Err((e, _)) => { error!(target: "fetch", "failed to read chunk: {}", e); return Err(io::Error::new(io::ErrorKind::Other, "failed to read body chunk")) From dc8ed22ea3a4de48bbe7fb9e3704300c56e876b5 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Tue, 13 Mar 2018 15:01:07 +0100 Subject: [PATCH 07/15] Fix tests. --- Cargo.lock | 2 ++ dapps/Cargo.toml | 1 + dapps/src/apps/fetcher/mod.rs | 2 +- dapps/src/lib.rs | 2 ++ dapps/src/tests/helpers/fetch.rs | 26 ++++++-------------------- dapps/src/tests/helpers/mod.rs | 20 ++++++-------------- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 3 +++ rpc/src/v1/tests/helpers/fetch.rs | 15 +++++++-------- rpc/src/v1/tests/mocked/parity_set.rs | 4 +++- 10 files changed, 32 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 648e20ee7d2..db3784e5bf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2004,6 +2004,7 @@ dependencies = [ "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "zip 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2167,6 +2168,7 @@ dependencies = [ "tiny-keccak 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", ] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 081867fbdd8..8a9725db88f 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -42,6 +42,7 @@ registrar = { path = "../registrar" } [dev-dependencies] env_logger = "0.4" ethcore-devtools = { path = "../devtools" } +url = "1" [features] ui = ["parity-ui/no-precompiled-js"] diff --git a/dapps/src/apps/fetcher/mod.rs b/dapps/src/apps/fetcher/mod.rs index 7362830125e..8ed3024fdf2 100644 --- a/dapps/src/apps/fetcher/mod.rs +++ b/dapps/src/apps/fetcher/mod.rs @@ -283,7 +283,7 @@ impl Endpoint for ContentFetcher { mod tests { use std::env; use std::sync::Arc; - use fetch::{Fetch, Client}; + use fetch::Client; use futures::{future, Future}; use hash_fetch::urlhint::{URLHint, URLHintResult}; use ethereum_types::H256; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index 482ee395958..e616a2444d1 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -58,6 +58,8 @@ extern crate ethcore_devtools as devtools; extern crate jsonrpc_core; #[cfg(test)] extern crate parity_reactor; +#[cfg(test)] +extern crate url; mod endpoint; mod apps; diff --git a/dapps/src/tests/helpers/fetch.rs b/dapps/src/tests/helpers/fetch.rs index 21d5bf2a7f5..3ef6f431380 100644 --- a/dapps/src/tests/helpers/fetch.rs +++ b/dapps/src/tests/helpers/fetch.rs @@ -14,9 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::{io, thread, time}; +use std::{thread, time}; use std::sync::{atomic, mpsc, Arc}; use parking_lot::Mutex; +use hyper; +use url::Url; use futures::{self, Future}; use fetch::{self, Fetch}; @@ -96,11 +98,8 @@ impl FakeFetch { impl Fetch for FakeFetch { type Result = Box + Send>; - fn new() -> Result where Self: Sized { - Ok(FakeFetch::default()) - } - - fn fetch_with_abort(&self, url: &str, _abort: fetch::Abort) -> Self::Result { + fn fetch(&self, url: &str, abort: fetch::Abort) -> Self::Result { + let u = Url::parse(url).unwrap(); self.requested.lock().push(url.into()); let manual = self.manual.clone(); let response = self.response.clone(); @@ -111,23 +110,10 @@ impl Fetch for FakeFetch { // wait for manual resume let _ = rx.recv(); } - let data = response.lock().take().unwrap_or(b"Some content"); - let cursor = io::Cursor::new(data); - tx.send(fetch::Response::from_reader(cursor)).unwrap(); + tx.send(fetch::Response::new(u, hyper::Response::new().with_body(data), abort)).unwrap(); }); Box::new(rx.map_err(|_| fetch::Error::Aborted)) } - - fn process_and_forget(&self, f: F) where - F: Future + Send + 'static, - I: Send + 'static, - E: Send + 'static, - { - // Spawn the task in a separate thread. - thread::spawn(|| { - let _ = f.wait(); - }); - } } diff --git a/dapps/src/tests/helpers/mod.rs b/dapps/src/tests/helpers/mod.rs index 30017a718b8..3a1311578c7 100644 --- a/dapps/src/tests/helpers/mod.rs +++ b/dapps/src/tests/helpers/mod.rs @@ -63,7 +63,7 @@ pub fn init_server(process: F, io: IoHandler) -> (Server, Arc { web_proxy_tokens: Arc, signer_address: Option<(String, u16)>, allowed_hosts: DomainsValidation, - fetch: Option, + fetch: T, serve_ui: bool, } impl ServerBuilder { /// Construct new dapps server - pub fn new>(dapps_path: P, registrar: Arc>) -> Self { + pub fn new>(fetch: FetchClient, dapps_path: P, registrar: Arc>) -> Self { ServerBuilder { dapps_path: dapps_path.as_ref().to_owned(), registrar: registrar, @@ -163,7 +163,7 @@ impl ServerBuilder { web_proxy_tokens: Arc::new(|_| None), signer_address: None, allowed_hosts: DomainsValidation::Disabled, - fetch: None, + fetch: fetch, serve_ui: false, } } @@ -179,7 +179,7 @@ impl ServerBuilder { web_proxy_tokens: self.web_proxy_tokens, signer_address: self.signer_address, allowed_hosts: self.allowed_hosts, - fetch: Some(fetch), + fetch: fetch, serve_ui: self.serve_ui, } } @@ -187,7 +187,6 @@ impl ServerBuilder { /// Asynchronously start server with no authentication, /// returns result with `Server` handle on success or an error. pub fn start_unsecured_http(self, addr: &SocketAddr, io: IoHandler) -> io::Result { - let fetch = self.fetch_client(); Server::start_http( addr, io, @@ -199,17 +198,10 @@ impl ServerBuilder { self.sync_status, self.web_proxy_tokens, Remote::new_sync(), - fetch, + self.fetch, self.serve_ui, ) } - - fn fetch_client(&self) -> T { - match self.fetch.clone() { - Some(fetch) => fetch, - None => T::new().unwrap(), - } - } } const DAPPS_DOMAIN: &'static str = "web3.site"; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c0158658b63..e93c4f844e4 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -70,3 +70,4 @@ pretty_assertions = "0.1" macros = { path = "../util/macros" } ethcore-network = { path = "../util/network" } kvdb-memorydb = { path = "../util/kvdb-memorydb" } +url = "1" diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 4a2003d652b..4e70ba3094c 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -91,6 +91,9 @@ extern crate macros; #[cfg(test)] extern crate kvdb_memorydb; +#[cfg(test)] +extern crate url; + extern crate tempdir; pub extern crate jsonrpc_ws_server as ws; diff --git a/rpc/src/v1/tests/helpers/fetch.rs b/rpc/src/v1/tests/helpers/fetch.rs index fba2aaf5d93..abf59cd6cf2 100644 --- a/rpc/src/v1/tests/helpers/fetch.rs +++ b/rpc/src/v1/tests/helpers/fetch.rs @@ -16,9 +16,11 @@ //! Test implementation of fetch client. -use std::{io, thread}; +use std::thread; use jsonrpc_core::futures::{self, Future}; use fetch::{self, Fetch}; +use hyper; +use url::Url; /// Test implementation of fetcher. Will always return the same file. #[derive(Default, Clone)] @@ -27,15 +29,12 @@ pub struct TestFetch; impl Fetch for TestFetch { type Result = Box + Send + 'static>; - fn new() -> Result where Self: Sized { - Ok(TestFetch) - } - - fn fetch_with_abort(&self, _url: &str, _abort: fetch::Abort) -> Self::Result { + fn fetch(&self, url: &str, abort: fetch::Abort) -> Self::Result { + let u = Url::parse(url).unwrap(); let (tx, rx) = futures::oneshot(); thread::spawn(move || { - let cursor = io::Cursor::new(b"Some content"); - tx.send(fetch::Response::from_reader(cursor)).unwrap(); + let r = hyper::Response::new().with_body(&b"Some content"[..]); + tx.send(fetch::Response::new(u, r, abort)).unwrap(); }); Box::new(rx.map_err(|_| fetch::Error::Aborted)) diff --git a/rpc/src/v1/tests/mocked/parity_set.rs b/rpc/src/v1/tests/mocked/parity_set.rs index 0666d4f42cf..f92841ee8b5 100644 --- a/rpc/src/v1/tests/mocked/parity_set.rs +++ b/rpc/src/v1/tests/mocked/parity_set.rs @@ -22,6 +22,7 @@ use ethereum_types::{U256, Address}; use ethcore::miner::MinerService; use ethcore::client::TestBlockChainClient; use ethsync::ManageNetwork; +use futures_cpupool::CpuPool; use jsonrpc_core::IoHandler; use v1::{ParitySet, ParitySetClient}; @@ -53,7 +54,8 @@ fn parity_set_client( net: &Arc, ) -> TestParitySetClient { let dapps_service = Arc::new(TestDappsService); - ParitySetClient::new(client, miner, updater, &(net.clone() as Arc), Some(dapps_service), TestFetch::default()) + let pool = CpuPool::new(1); + ParitySetClient::new(client, miner, updater, &(net.clone() as Arc), Some(dapps_service), TestFetch::default(), pool) } #[test] From a3ef61cd4e8ad5d6cc4746549d2c5f379a3835cd Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Tue, 13 Mar 2018 16:28:37 +0100 Subject: [PATCH 08/15] Comment. --- util/fetch/src/client.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 8a4d48579a4..a27293abbca 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -159,8 +159,6 @@ impl Drop for Client { impl Client { /// Create a new fetch client. - /// - /// This spawns off a background thread for request/response processing. pub fn new() -> Result { let startup_done = Arc::new((Mutex::new(Ok(())), Condvar::new())); let (tx_proto, rx_proto) = mpsc::channel(64); From 6e49a0948d441e805d5e375dc3b2a606fcf6a454 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Tue, 13 Mar 2018 18:28:59 +0100 Subject: [PATCH 09/15] Change expect messages. --- util/fetch/src/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index a27293abbca..04cc04c9f0e 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -166,7 +166,7 @@ impl Client { Client::background_thread(startup_done.clone(), rx_proto)?; let mut guard = startup_done.0.lock(); - let startup_result = startup_done.1.wait_for(&mut guard, Duration::from_secs(3)); + let startup_result = startup_done.1.wait_for(&mut guard, Duration::from_secs(10)); if startup_result.timed_out() { error!(target: "fetch", "timeout starting background thread"); @@ -199,7 +199,7 @@ impl Client { .build(&core.handle()); let future = rx_proto.take_while(|item| Ok(item.is_some())) - .map(|item| item.expect("`take_while` is only passing on channel items != None")) + .map(|item| item.expect("`take_while` is only passing on channel items != None; qed")) .for_each(|(url, abort, sender)| { trace!(target: "fetch", "new request to {}", url); @@ -421,7 +421,7 @@ impl io::Read for BodyReader { break } } else { - let body = self.body.take().expect("within this loop `self.body` is always defined"); + let body = self.body.take().expect("loop condition ensures `self.body` is always defined; qed"); match body.into_future().wait() { // wait for next chunk Err((e, _)) => { error!(target: "fetch", "failed to read chunk: {}", e); From 4e4519b060fbe40ebddd2282b48fc13be8226b1a Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 10:32:05 +0100 Subject: [PATCH 10/15] Use local test server for testing fetch client. --- util/fetch/src/client.rs | 99 ++++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 13 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 04cc04c9f0e..0fab704314d 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -497,23 +497,29 @@ impl From for Error { #[cfg(test)] mod test { use super::*; + use futures::future; + use futures::sync::mpsc; + use futures_timer::Delay; + use hyper::StatusCode; + use hyper::server::{Http, Request, Response, Service}; #[test] fn it_should_fetch() { + let _server = TestServer::run("127.0.0.1:11111"); let client = Client::new().unwrap(); - let abort = Abort::default().with_max_duration(Duration::from_secs(10)); - let future = client.fetch("https://httpbin.org/drip?numbytes=3&duration=3&delay=1&code=200", abort); + let future = client.fetch("http://127.0.0.1:11111?123", Default::default()); let resp = future.wait().unwrap(); assert!(resp.is_success()); let body = resp.concat2().wait().unwrap(); - assert_eq!(body.len(), 3) + assert_eq!(&body[..], b"123") } #[test] fn it_should_timeout() { + let _server = TestServer::run("127.0.0.1:11112"); let client = Client::new().unwrap(); - let abort = Abort::default().with_max_duration(Duration::from_secs(5)); - match client.fetch("https://httpbin.org/delay/7", abort).wait() { + let abort = Abort::default().with_max_duration(Duration::from_secs(3)); + match client.fetch("http://127.0.0.1:11112/delay?4", abort).wait() { Err(Error::Timeout) => {} other => panic!("expected timeout, got {:?}", other) } @@ -521,25 +527,28 @@ mod test { #[test] fn it_should_follow_redirects() { + let _server = TestServer::run("127.0.0.1:11113"); let client = Client::new().unwrap(); - let abort = Abort::default().with_max_redirects(4).with_max_duration(Duration::from_secs(15)); - let future = client.fetch("https://httpbin.org/absolute-redirect/3", abort); + let abort = Abort::default(); + let future = client.fetch("http://127.0.0.1:11113/redirect?http://127.0.0.1:11113/", abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_follow_relative_redirects() { + let _server = TestServer::run("127.0.0.1:11114"); let client = Client::new().unwrap(); - let abort = Abort::default().with_max_redirects(4).with_max_duration(Duration::from_secs(15)); - let future = client.fetch("https://httpbin.org/relative-redirect/3", abort); + let abort = Abort::default().with_max_redirects(4); + let future = client.fetch("http://127.0.0.1:11114/redirect?/", abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_not_follow_too_many_redirects() { + let _server = TestServer::run("127.0.0.1:11115"); let client = Client::new().unwrap(); let abort = Abort::default().with_max_redirects(3); - match client.fetch("https://httpbin.org/absolute-redirect/4", abort).wait() { + match client.fetch("http://127.0.0.1:11115/loop", abort).wait() { Err(Error::TooManyRedirects) => {} other => panic!("expected too many redirects error, got {:?}", other) } @@ -547,21 +556,85 @@ mod test { #[test] fn it_should_read_data() { + let _server = TestServer::run("127.0.0.1:11116"); let client = Client::new().unwrap(); let abort = Abort::default(); - let future = client.fetch("https://httpbin.org/bytes/1024", abort); + let future = client.fetch("http://127.0.0.1:11116?abcdefghijklmnopqrstuvwxyz", abort); let resp = future.wait().unwrap(); assert!(resp.is_success()); - assert_eq!(resp.concat2().wait().unwrap().len(), 1024) + assert_eq!(&resp.concat2().wait().unwrap()[..], b"abcdefghijklmnopqrstuvwxyz") } #[test] fn it_should_not_read_too_much_data() { + let _server = TestServer::run("127.0.0.1:11117"); let client = Client::new().unwrap(); let abort = Abort::default().with_max_size(3); - match client.fetch("https://httpbin.org/bytes/4", abort).wait() { + let resp = client.fetch("http://127.0.0.1:11117/?1234", abort).wait().unwrap(); + assert!(resp.is_success()); + match resp.concat2().wait() { Err(Error::SizeLimit) => {} other => panic!("expected size limit error, got {:?}", other) } } + + struct TestServer; + + impl Service for TestServer { + type Request = Request; + type Response = Response; + type Error = hyper::Error; + type Future = Box>; + + fn call(&self, req: Request) -> Self::Future { + match req.uri().path() { + "/" => { + let body = req.uri().query().unwrap_or("").to_string(); + let req = Response::new().with_body(body); + Box::new(future::ok(req)) + } + "/redirect" => { + let loc = Location::new(req.uri().query().unwrap_or("/").to_string()); + let req = Response::new() + .with_status(StatusCode::MovedPermanently) + .with_header(loc); + Box::new(future::ok(req)) + } + "/loop" => { + let req = Response::new() + .with_status(StatusCode::MovedPermanently) + .with_header(Location::new("/loop".to_string())); + Box::new(future::ok(req)) + } + "/delay" => { + let d = Duration::from_secs(req.uri().query().unwrap_or("0").parse().unwrap()); + Box::new(Delay::new(d).from_err().map(|_| Response::new())) + } + _ => Box::new(future::ok(Response::new().with_status(StatusCode::NotFound))) + } + } + } + + impl TestServer { + fn run(addr: &'static str) -> Handle { + let (tx, rx) = mpsc::channel(0); + let rx_fut = rx.into_future() + .map(|_| ()) + .map_err(|_| ()); + thread::spawn(move || { + let addr = addr.parse().unwrap(); + let server = Http::new().bind(&addr, || Ok(TestServer)).unwrap(); + server.run_until(rx_fut).unwrap(); + }); + Handle(tx) + } + } + + struct Handle(mpsc::Sender<()>); + + impl Drop for Handle { + fn drop(&mut self) { + self.0.clone().send(()).wait().unwrap(); + } + } } From 8bd9bd9231ebb13a9201476e235a581d4e3636e0 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 10:42:13 +0100 Subject: [PATCH 11/15] Ensure max_size also in BodyReader. --- util/fetch/src/client.rs | 62 +++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 0fab704314d..fc3fa77d272 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -23,11 +23,12 @@ use hyper::mime::Mime; use hyper::{self, Request, Method, StatusCode}; use hyper_rustls; use parking_lot::{Condvar, Mutex}; +use std::cmp::min; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::thread; use std::time::Duration; -use std::{cmp, io, fmt, mem}; +use std::{io, fmt, mem}; use tokio_core::reactor; use url::{self, Url}; @@ -390,9 +391,11 @@ impl Stream for Response { /// It implements `io::Read` by repedately waiting for the next `Chunk` /// of hyper's response `Body` which blocks the current thread. pub struct BodyReader { - chunk: hyper::Chunk, - body: Option, + chunk: hyper::Chunk, + body: Option, + abort: Abort, offset: usize, + count: usize, } impl BodyReader { @@ -401,23 +404,30 @@ impl BodyReader { BodyReader { body: Some(r.body), chunk: Default::default(), + abort: r.abort, offset: 0, + count: 0, } } } impl io::Read for BodyReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { - let mut m = 0; + let mut n = 0; while self.body.is_some() { // Can we still read from the current chunk? if self.offset < self.chunk.len() { - let k = cmp::min(self.chunk.len() - self.offset, buf.len() - m); + let k = min(self.chunk.len() - self.offset, buf.len() - n); + if self.count + k > self.abort.max_size() { + debug!(target: "fetch", "size limit {:?} exceeded", self.abort.max_size()); + return Err(io::Error::new(io::ErrorKind::PermissionDenied, "size limit exceeded")) + } let c = &self.chunk[self.offset .. self.offset + k]; - (&mut buf[m .. m + k]).copy_from_slice(c); + (&mut buf[n .. n + k]).copy_from_slice(c); self.offset += k; - m += k; - if m == buf.len() { + self.count += k; + n += k; + if n == buf.len() { break } } else { @@ -436,7 +446,7 @@ impl io::Read for BodyReader { } } } - Ok(m) + Ok(n) } } @@ -502,6 +512,8 @@ mod test { use futures_timer::Delay; use hyper::StatusCode; use hyper::server::{Http, Request, Response, Service}; + use std; + use std::io::Read; #[test] fn it_should_fetch() { @@ -518,8 +530,8 @@ mod test { fn it_should_timeout() { let _server = TestServer::run("127.0.0.1:11112"); let client = Client::new().unwrap(); - let abort = Abort::default().with_max_duration(Duration::from_secs(3)); - match client.fetch("http://127.0.0.1:11112/delay?4", abort).wait() { + let abort = Abort::default().with_max_duration(Duration::from_secs(1)); + match client.fetch("http://127.0.0.1:11112/delay?3", abort).wait() { Err(Error::Timeout) => {} other => panic!("expected timeout, got {:?}", other) } @@ -578,6 +590,21 @@ mod test { } } + #[test] + fn it_should_not_read_too_much_data_sync() { + let _server = TestServer::run("127.0.0.1:11118"); + let client = Client::new().unwrap(); + let abort = Abort::default().with_max_size(3); + let resp = client.fetch("http://127.0.0.1:11118/?1234", abort).wait().unwrap(); + assert!(resp.is_success()); + let mut buffer = Vec::new(); + let mut reader = BodyReader::new(resp); + match reader.read_to_end(&mut buffer) { + Err(ref e) if e.kind() == io::ErrorKind::PermissionDenied => {} + other => panic!("expected size limit error, got {:?}", other) + } + } + struct TestServer; impl Service for TestServer { @@ -617,16 +644,17 @@ mod test { impl TestServer { fn run(addr: &'static str) -> Handle { - let (tx, rx) = mpsc::channel(0); - let rx_fut = rx.into_future() - .map(|_| ()) - .map_err(|_| ()); + let (tx_start, rx_start) = std::sync::mpsc::sync_channel(1); + let (tx_end, rx_end) = mpsc::channel(0); + let rx_end_fut = rx_end.into_future().map(|_| ()).map_err(|_| ()); thread::spawn(move || { let addr = addr.parse().unwrap(); let server = Http::new().bind(&addr, || Ok(TestServer)).unwrap(); - server.run_until(rx_fut).unwrap(); + tx_start.send(()).unwrap_or(()); + server.run_until(rx_end_fut).unwrap(); }); - Handle(tx) + rx_start.recv().unwrap(); + Handle(tx_end) } } From 88cd685c19e5f7672d54def176d138fb6d605edc Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 11:01:43 +0100 Subject: [PATCH 12/15] Replace `Condvar` with `sync_channel`. --- Cargo.lock | 1 - util/fetch/Cargo.toml | 1 - util/fetch/src/client.rs | 46 +++++++++++++++++++++------------------- util/fetch/src/lib.rs | 1 - 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db3784e5bf2..b5d8d7eb039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -956,7 +956,6 @@ dependencies = [ "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/util/fetch/Cargo.toml b/util/fetch/Cargo.toml index f644f0d1906..61925e189d1 100644 --- a/util/fetch/Cargo.toml +++ b/util/fetch/Cargo.toml @@ -12,7 +12,6 @@ futures-timer = "0.1" hyper = "0.11" hyper-rustls = "0.11" log = "0.4" -parking_lot = "0.5" tokio-core = "0.1" url = "1" diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index fc3fa77d272..3e3917c355a 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -22,13 +22,14 @@ use hyper::header::{UserAgent, Location, ContentLength, ContentType}; use hyper::mime::Mime; use hyper::{self, Request, Method, StatusCode}; use hyper_rustls; -use parking_lot::{Condvar, Mutex}; +use std; use std::cmp::min; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::RecvTimeoutError; use std::thread; use std::time::Duration; -use std::{io, fmt, mem}; +use std::{io, fmt}; use tokio_core::reactor; use url::{self, Url}; @@ -123,7 +124,7 @@ pub trait Fetch: Clone + Send + Sync + 'static { } type TxResponse = oneshot::Sender>; -type StartupCond = Arc<(Mutex>, Condvar)>; +type TxStartup = std::sync::mpsc::SyncSender>; type ChanItem = Option<(Url, Abort, TxResponse)>; /// An implementation of `Fetch` using a `hyper` client. @@ -161,21 +162,25 @@ impl Drop for Client { impl Client { /// Create a new fetch client. pub fn new() -> Result { - let startup_done = Arc::new((Mutex::new(Ok(())), Condvar::new())); + let (tx_start, rx_start) = std::sync::mpsc::sync_channel(1); let (tx_proto, rx_proto) = mpsc::channel(64); - Client::background_thread(startup_done.clone(), rx_proto)?; + Client::background_thread(tx_start, rx_proto)?; - let mut guard = startup_done.0.lock(); - let startup_result = startup_done.1.wait_for(&mut guard, Duration::from_secs(10)); - - if startup_result.timed_out() { - error!(target: "fetch", "timeout starting background thread"); - return Err(Error::BackgroundThreadDead) - } - if let Err(e) = mem::replace(&mut *guard, Ok(())) { - error!(target: "fetch", "error starting background thread: {}", e); - return Err(e.into()) + match rx_start.recv_timeout(Duration::from_secs(10)) { + Err(RecvTimeoutError::Timeout) => { + error!(target: "fetch", "timeout starting background thread"); + return Err(Error::BackgroundThreadDead) + } + Err(RecvTimeoutError::Disconnected) => { + error!(target: "fetch", "background thread gone"); + return Err(Error::BackgroundThreadDead) + } + Ok(Err(e)) => { + error!(target: "fetch", "error starting background thread: {}", e); + return Err(e.into()) + } + Ok(Ok(())) => {} } Ok(Client { @@ -184,16 +189,13 @@ impl Client { }) } - fn background_thread(start: StartupCond, rx_proto: mpsc::Receiver) -> io::Result> { + fn background_thread(tx_start: TxStartup, rx_proto: mpsc::Receiver) -> io::Result> { thread::Builder::new().name("fetch".into()).spawn(move || { let mut core = match reactor::Core::new() { Ok(c) => c, - Err(e) => { - *start.0.lock() = Err(e); - start.1.notify_one(); - return () - } + Err(e) => return tx_start.send(Err(e)).unwrap_or(()) }; + let handle = core.handle(); let hyper = hyper::Client::configure() .connector(hyper_rustls::HttpsConnector::new(4, &core.handle())) @@ -241,7 +243,7 @@ impl Client { future::ok(()) }); - start.1.notify_one(); + tx_start.send(Ok(())).unwrap_or(()); debug!(target: "fetch", "processing requests ..."); if let Err(()) = core.run(future) { diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index 347e4d94b96..886297a0340 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -24,7 +24,6 @@ extern crate log; #[macro_use] extern crate futures; extern crate futures_timer; -extern crate parking_lot; extern crate hyper; extern crate hyper_rustls; From 43539ac5636889a381c9c312074648ff29208716 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 11:14:31 +0100 Subject: [PATCH 13/15] Re-export `url::Url` from `fetch` crate. --- Cargo.lock | 4 ---- dapps/Cargo.toml | 1 - dapps/src/lib.rs | 2 -- dapps/src/tests/helpers/fetch.rs | 3 +-- hash-fetch/Cargo.toml | 1 - hash-fetch/src/client.rs | 4 +--- price-info/Cargo.toml | 1 - price-info/src/lib.rs | 4 +--- rpc/Cargo.toml | 1 - rpc/src/lib.rs | 3 --- rpc/src/v1/tests/helpers/fetch.rs | 3 +-- util/fetch/src/client.rs | 4 ++-- util/fetch/src/lib.rs | 1 + 13 files changed, 7 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5d8d7eb039..df46accfbc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2003,7 +2003,6 @@ dependencies = [ "serde_derive 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "zip 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2057,7 +2056,6 @@ dependencies = [ "rand 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2167,7 +2165,6 @@ dependencies = [ "tiny-keccak 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "transient-hashmap 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", ] @@ -2444,7 +2441,6 @@ dependencies = [ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", - "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 8a9725db88f..081867fbdd8 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -42,7 +42,6 @@ registrar = { path = "../registrar" } [dev-dependencies] env_logger = "0.4" ethcore-devtools = { path = "../devtools" } -url = "1" [features] ui = ["parity-ui/no-precompiled-js"] diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index e616a2444d1..482ee395958 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -58,8 +58,6 @@ extern crate ethcore_devtools as devtools; extern crate jsonrpc_core; #[cfg(test)] extern crate parity_reactor; -#[cfg(test)] -extern crate url; mod endpoint; mod apps; diff --git a/dapps/src/tests/helpers/fetch.rs b/dapps/src/tests/helpers/fetch.rs index 3ef6f431380..dfe523200e4 100644 --- a/dapps/src/tests/helpers/fetch.rs +++ b/dapps/src/tests/helpers/fetch.rs @@ -18,10 +18,9 @@ use std::{thread, time}; use std::sync::{atomic, mpsc, Arc}; use parking_lot::Mutex; use hyper; -use url::Url; use futures::{self, Future}; -use fetch::{self, Fetch}; +use fetch::{self, Fetch, Url}; pub struct FetchControl { sender: mpsc::Sender<()>, diff --git a/hash-fetch/Cargo.toml b/hash-fetch/Cargo.toml index 4aa16312603..d5630993670 100644 --- a/hash-fetch/Cargo.toml +++ b/hash-fetch/Cargo.toml @@ -28,4 +28,3 @@ ethabi-contract = "5.0" [dev-dependencies] hyper = "0.11" parking_lot = "0.5" -url = "1" diff --git a/hash-fetch/src/client.rs b/hash-fetch/src/client.rs index 88a9945945f..b3a9356ea2c 100644 --- a/hash-fetch/src/client.rs +++ b/hash-fetch/src/client.rs @@ -194,18 +194,16 @@ fn random_temp_path() -> PathBuf { #[cfg(test)] mod tests { extern crate hyper; - extern crate url; use rustc_hex::FromHex; use std::sync::{Arc, mpsc}; use parking_lot::Mutex; use futures::future; use futures_cpupool::CpuPool; - use fetch::{self, Fetch}; + use fetch::{self, Fetch, Url}; use parity_reactor::Remote; use urlhint::tests::{FakeRegistrar, URLHINT}; use super::{Error, Client, HashFetch, random_temp_path}; use self::hyper::StatusCode; - use self::url::Url; #[derive(Clone)] diff --git a/price-info/Cargo.toml b/price-info/Cargo.toml index 198cccb98bc..5122ebb9dd7 100644 --- a/price-info/Cargo.toml +++ b/price-info/Cargo.toml @@ -16,4 +16,3 @@ serde_json = "1.0" [dev-dependencies] hyper = "0.11" parking_lot = "0.5" -url = "1" diff --git a/price-info/src/lib.rs b/price-info/src/lib.rs index c065aea9610..e233062b431 100644 --- a/price-info/src/lib.rs +++ b/price-info/src/lib.rs @@ -134,19 +134,17 @@ impl Client { #[cfg(test)] mod test { extern crate hyper; - extern crate url; extern crate parking_lot; use self::parking_lot::Mutex; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use fetch; - use fetch::Fetch; + use fetch::{Fetch, Url}; use futures_cpupool::CpuPool; use futures::future::{self, FutureResult}; use Client; use self::hyper::StatusCode; - use self::url::Url; #[derive(Clone)] struct FakeFetch(Option, Arc>); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index e93c4f844e4..c0158658b63 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -70,4 +70,3 @@ pretty_assertions = "0.1" macros = { path = "../util/macros" } ethcore-network = { path = "../util/network" } kvdb-memorydb = { path = "../util/kvdb-memorydb" } -url = "1" diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 4e70ba3094c..4a2003d652b 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -91,9 +91,6 @@ extern crate macros; #[cfg(test)] extern crate kvdb_memorydb; -#[cfg(test)] -extern crate url; - extern crate tempdir; pub extern crate jsonrpc_ws_server as ws; diff --git a/rpc/src/v1/tests/helpers/fetch.rs b/rpc/src/v1/tests/helpers/fetch.rs index abf59cd6cf2..316d0682554 100644 --- a/rpc/src/v1/tests/helpers/fetch.rs +++ b/rpc/src/v1/tests/helpers/fetch.rs @@ -18,9 +18,8 @@ use std::thread; use jsonrpc_core::futures::{self, Future}; -use fetch::{self, Fetch}; +use fetch::{self, Fetch, Url}; use hyper; -use url::Url; /// Test implementation of fetcher. Will always return the same file. #[derive(Default, Clone)] diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index 3e3917c355a..fde160b9607 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -435,11 +435,11 @@ impl io::Read for BodyReader { } else { let body = self.body.take().expect("loop condition ensures `self.body` is always defined; qed"); match body.into_future().wait() { // wait for next chunk - Err((e, _)) => { + Err((e, _)) => { error!(target: "fetch", "failed to read chunk: {}", e); return Err(io::Error::new(io::ErrorKind::Other, "failed to read body chunk")) } - Ok((None, _)) => break, // body is exhausted, break out of the loop + Ok((None, _)) => break, // body is exhausted, break out of the loop Ok((Some(c), b)) => { self.body = Some(b); self.chunk = c; diff --git a/util/fetch/src/lib.rs b/util/fetch/src/lib.rs index 886297a0340..a5722dfe26a 100644 --- a/util/fetch/src/lib.rs +++ b/util/fetch/src/lib.rs @@ -34,5 +34,6 @@ extern crate url; /// Fetch client implementation. pub mod client; +pub use url::Url; pub use self::client::{Client, Fetch, Error, Response, Abort, BodyReader}; From 065780a2cf518646db5883cabb29185e5f1d0194 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 11:40:16 +0100 Subject: [PATCH 14/15] Remove spaces. --- util/fetch/src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index fde160b9607..ae83169757b 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -192,7 +192,7 @@ impl Client { fn background_thread(tx_start: TxStartup, rx_proto: mpsc::Receiver) -> io::Result> { thread::Builder::new().name("fetch".into()).spawn(move || { let mut core = match reactor::Core::new() { - Ok(c) => c, + Ok(c) => c, Err(e) => return tx_start.send(Err(e)).unwrap_or(()) }; @@ -375,7 +375,7 @@ impl Stream for Response { return Err(Error::Aborted) } match try_ready!(self.body.poll()) { - None => Ok(Async::Ready(None)), + None => Ok(Async::Ready(None)), Some(c) => { if self.nread + c.len() > self.abort.max_size() { debug!(target: "fetch", "size limit {:?} for {} exceeded", self.abort.max_size(), self.url); From 208047a98685b64e4a0976655af45887535ae41a Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Wed, 14 Mar 2018 12:41:29 +0100 Subject: [PATCH 15/15] Use random ports in local test server. --- util/fetch/src/client.rs | 54 +++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/util/fetch/src/client.rs b/util/fetch/src/client.rs index ae83169757b..c57a362ca95 100644 --- a/util/fetch/src/client.rs +++ b/util/fetch/src/client.rs @@ -516,12 +516,15 @@ mod test { use hyper::server::{Http, Request, Response, Service}; use std; use std::io::Read; + use std::net::SocketAddr; + + const ADDRESS: &str = "127.0.0.1:0"; #[test] fn it_should_fetch() { - let _server = TestServer::run("127.0.0.1:11111"); + let server = TestServer::run(); let client = Client::new().unwrap(); - let future = client.fetch("http://127.0.0.1:11111?123", Default::default()); + let future = client.fetch(&format!("http://{}?123", server.addr()), Default::default()); let resp = future.wait().unwrap(); assert!(resp.is_success()); let body = resp.concat2().wait().unwrap(); @@ -530,10 +533,10 @@ mod test { #[test] fn it_should_timeout() { - let _server = TestServer::run("127.0.0.1:11112"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default().with_max_duration(Duration::from_secs(1)); - match client.fetch("http://127.0.0.1:11112/delay?3", abort).wait() { + match client.fetch(&format!("http://{}/delay?3", server.addr()), abort).wait() { Err(Error::Timeout) => {} other => panic!("expected timeout, got {:?}", other) } @@ -541,28 +544,28 @@ mod test { #[test] fn it_should_follow_redirects() { - let _server = TestServer::run("127.0.0.1:11113"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default(); - let future = client.fetch("http://127.0.0.1:11113/redirect?http://127.0.0.1:11113/", abort); + let future = client.fetch(&format!("http://{}/redirect?http://{}/", server.addr(), server.addr()), abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_follow_relative_redirects() { - let _server = TestServer::run("127.0.0.1:11114"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default().with_max_redirects(4); - let future = client.fetch("http://127.0.0.1:11114/redirect?/", abort); + let future = client.fetch(&format!("http://{}/redirect?/", server.addr()), abort); assert!(future.wait().unwrap().is_success()) } #[test] fn it_should_not_follow_too_many_redirects() { - let _server = TestServer::run("127.0.0.1:11115"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default().with_max_redirects(3); - match client.fetch("http://127.0.0.1:11115/loop", abort).wait() { + match client.fetch(&format!("http://{}/loop", server.addr()), abort).wait() { Err(Error::TooManyRedirects) => {} other => panic!("expected too many redirects error, got {:?}", other) } @@ -570,10 +573,10 @@ mod test { #[test] fn it_should_read_data() { - let _server = TestServer::run("127.0.0.1:11116"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default(); - let future = client.fetch("http://127.0.0.1:11116?abcdefghijklmnopqrstuvwxyz", abort); + let future = client.fetch(&format!("http://{}?abcdefghijklmnopqrstuvwxyz", server.addr()), abort); let resp = future.wait().unwrap(); assert!(resp.is_success()); assert_eq!(&resp.concat2().wait().unwrap()[..], b"abcdefghijklmnopqrstuvwxyz") @@ -581,10 +584,10 @@ mod test { #[test] fn it_should_not_read_too_much_data() { - let _server = TestServer::run("127.0.0.1:11117"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default().with_max_size(3); - let resp = client.fetch("http://127.0.0.1:11117/?1234", abort).wait().unwrap(); + let resp = client.fetch(&format!("http://{}/?1234", server.addr()), abort).wait().unwrap(); assert!(resp.is_success()); match resp.concat2().wait() { Err(Error::SizeLimit) => {} @@ -594,10 +597,10 @@ mod test { #[test] fn it_should_not_read_too_much_data_sync() { - let _server = TestServer::run("127.0.0.1:11118"); + let server = TestServer::run(); let client = Client::new().unwrap(); let abort = Abort::default().with_max_size(3); - let resp = client.fetch("http://127.0.0.1:11118/?1234", abort).wait().unwrap(); + let resp = client.fetch(&format!("http://{}/?1234", server.addr()), abort).wait().unwrap(); assert!(resp.is_success()); let mut buffer = Vec::new(); let mut reader = BodyReader::new(resp); @@ -645,26 +648,31 @@ mod test { } impl TestServer { - fn run(addr: &'static str) -> Handle { + fn run() -> Handle { let (tx_start, rx_start) = std::sync::mpsc::sync_channel(1); let (tx_end, rx_end) = mpsc::channel(0); let rx_end_fut = rx_end.into_future().map(|_| ()).map_err(|_| ()); thread::spawn(move || { - let addr = addr.parse().unwrap(); + let addr = ADDRESS.parse().unwrap(); let server = Http::new().bind(&addr, || Ok(TestServer)).unwrap(); - tx_start.send(()).unwrap_or(()); + tx_start.send(server.local_addr().unwrap()).unwrap_or(()); server.run_until(rx_end_fut).unwrap(); }); - rx_start.recv().unwrap(); - Handle(tx_end) + Handle(rx_start.recv().unwrap(), tx_end) } } - struct Handle(mpsc::Sender<()>); + struct Handle(SocketAddr, mpsc::Sender<()>); + + impl Handle { + fn addr(&self) -> SocketAddr { + self.0 + } + } impl Drop for Handle { fn drop(&mut self) { - self.0.clone().send(()).wait().unwrap(); + self.1.clone().send(()).wait().unwrap(); } } }