From 0121e1c816ba671ba6c3a77057419ddd08b8e75a Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 07:02:44 +1300 Subject: [PATCH 01/40] feat: Create new cargo project for block stream --- block-streamer/Cargo.lock | 7 +++++++ block-streamer/Cargo.toml | 8 ++++++++ block-streamer/src/main.rs | 3 +++ 3 files changed, 18 insertions(+) create mode 100644 block-streamer/Cargo.lock create mode 100644 block-streamer/Cargo.toml create mode 100644 block-streamer/src/main.rs diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock new file mode 100644 index 000000000..54e12ae9f --- /dev/null +++ b/block-streamer/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "block-streamer" +version = "0.1.0" diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml new file mode 100644 index 000000000..cbf63d4af --- /dev/null +++ b/block-streamer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "block-streamer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs new file mode 100644 index 000000000..e7a11a969 --- /dev/null +++ b/block-streamer/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 56fcd2a3b2766f0bdd2d790d270d456e78d0d88c Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 07:30:17 +1300 Subject: [PATCH 02/40] feat: Add `redis` module --- block-streamer/Cargo.toml | 2 + block-streamer/src/redis.rs | 150 ++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 block-streamer/src/redis.rs diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index cbf63d4af..a09c10de5 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -6,3 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } +tracing = "0.1.34" diff --git a/block-streamer/src/redis.rs b/block-streamer/src/redis.rs new file mode 100644 index 000000000..e426929ec --- /dev/null +++ b/block-streamer/src/redis.rs @@ -0,0 +1,150 @@ +pub use redis::{self, aio::ConnectionManager, FromRedisValue, ToRedisArgs}; + +const STORAGE: &str = "storage_alertexer"; + +pub const LAKE_BUCKET_PREFIX: &str = "near-lake-data-"; +pub const STREAMS_SET_KEY: &str = "streams"; + +pub async fn get_redis_client(redis_connection_str: &str) -> redis::Client { + redis::Client::open(redis_connection_str).expect("can create redis client") +} + +pub fn generate_real_time_stream_key(prefix: &str) -> String { + format!("{}:real_time:stream", prefix) +} + +pub fn generate_real_time_streamer_message_key(block_height: u64) -> String { + format!("streamer:message:{}", block_height) +} + +pub fn generate_real_time_storage_key(prefix: &str) -> String { + format!("{}:real_time:stream:storage", prefix) +} + +pub fn generate_historical_stream_key(prefix: &str) -> String { + format!("{}:historical:stream", prefix) +} + +pub fn generate_historical_storage_key(prefix: &str) -> String { + format!("{}:historical:stream:storage", prefix) +} + +pub async fn connect(redis_connection_str: &str) -> anyhow::Result { + Ok(get_redis_client(redis_connection_str) + .await + .get_tokio_connection_manager() + .await?) +} + +pub async fn del( + redis_connection_manager: &ConnectionManager, + key: impl ToRedisArgs + std::fmt::Debug, +) -> anyhow::Result<()> { + redis::cmd("DEL") + .arg(&key) + .query_async(&mut redis_connection_manager.clone()) + .await?; + tracing::debug!(target: STORAGE, "DEL: {:?}", key); + Ok(()) +} + +pub async fn set( + redis_connection_manager: &ConnectionManager, + key: impl ToRedisArgs + std::fmt::Debug, + value: impl ToRedisArgs + std::fmt::Debug, + expiration_seconds: Option, +) -> anyhow::Result<()> { + let mut cmd = redis::cmd("SET"); + cmd.arg(&key).arg(&value); + + // Add expiration arguments if present + if let Some(expiration_seconds) = expiration_seconds { + cmd.arg("EX").arg(expiration_seconds); + } + + cmd.query_async(&mut redis_connection_manager.clone()) + .await?; + tracing::debug!( + target: STORAGE, + "SET: {:?}: {:?} Ex: {:?}", + key, + value, + expiration_seconds + ); + Ok(()) +} + +pub async fn get( + redis_connection_manager: &ConnectionManager, + key: impl ToRedisArgs + std::fmt::Debug, +) -> anyhow::Result { + let value: V = redis::cmd("GET") + .arg(&key) + .query_async(&mut redis_connection_manager.clone()) + .await?; + tracing::debug!(target: STORAGE, "GET: {:?}: {:?}", &key, &value,); + Ok(value) +} + +pub async fn sadd( + redis_connection_manager: &ConnectionManager, + key: impl ToRedisArgs + std::fmt::Debug, + value: impl ToRedisArgs + std::fmt::Debug, +) -> anyhow::Result<()> { + tracing::debug!(target: STORAGE, "SADD: {:?}: {:?}", key, value); + + redis::cmd("SADD") + .arg(key) + .arg(value) + .query_async(&mut redis_connection_manager.clone()) + .await?; + + Ok(()) +} + +pub async fn xadd( + redis_connection_manager: &ConnectionManager, + stream_key: impl ToRedisArgs + std::fmt::Debug, + fields: &[(&str, impl ToRedisArgs + std::fmt::Debug)], +) -> anyhow::Result<()> { + tracing::debug!(target: STORAGE, "XADD: {:?}, {:?}", stream_key, fields); + + let mut cmd = redis::cmd("XADD"); + cmd.arg(stream_key).arg("*"); + + for (field, value) in fields { + cmd.arg(*field).arg(value); + } + + cmd.query_async(&mut redis_connection_manager.clone()) + .await?; + + Ok(()) +} + +pub async fn update_last_indexed_block( + redis_connection_manager: &ConnectionManager, + block_height: u64, +) -> anyhow::Result<()> { + set( + redis_connection_manager, + "last_indexed_block", + block_height, + None, + ) + .await?; + redis::cmd("INCR") + .arg("blocks_processed") + .query_async(&mut redis_connection_manager.clone()) + .await?; + Ok(()) +} + +pub async fn get_last_indexed_block( + redis_connection_manager: &ConnectionManager, +) -> anyhow::Result { + Ok(redis::cmd("GET") + .arg("last_indexed_block") + .query_async(&mut redis_connection_manager.clone()) + .await?) +} From 50b413e3faf97dcd5f73cb5020eba89032398aef Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 07:30:40 +1300 Subject: [PATCH 03/40] feat: Add existing historical processing logic --- block-streamer/Cargo.lock | 2917 +++++++++++++++++ block-streamer/Cargo.toml | 6 + .../src/historical_block_processing.rs | 430 +++ block-streamer/src/main.rs | 5 + 4 files changed, 3358 insertions(+) create mode 100644 block-streamer/src/historical_block_processing.rs diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index 54e12ae9f..e7586d9e3 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -2,6 +2,2923 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "aws-config" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "741327a7f70e6e639bdb5061964c66250460c70ad3f59c3fe2a3a64ac1484e33" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper", + "ring 0.16.20", + "time", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f99dd587a46af58f8cf37773687ecec19d0373a5954942d7e0f405751fe2369" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13fdfc00c57d95e10bcf83d2331c4ae9ca460ca84dc983b2cdd692de87640389" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74cdac70481d144bf7001c27884b95ee12c8f62e61db90320d59b673ae121cb8" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae411cb03ea6df0d4c4340a0d3c15cab7b19715d091f76c5629f31acd6403f3" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tower", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d2fb56182ac693a19364cc0bde22d95aef9be3663bf9b906ffbd0ab0a7c7d1" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tokio-stream", + "tower", + "url", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70adf3e9518c8d6d14f1239f6af04c019ffd260ab791e17deb11f1bce6a9f76" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "regex", + "tower", + "tracing", + "url", +] + +[[package]] +name = "aws-sig-auth" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22af7f6515f8b51dabef87df1d901c9734e4e367791c6d0e1082f9f31528120e" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14500f741fb73a3c6cb173f8d96b433319a0e27c370a4e783b9ad693fc86210e" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-http", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2 0.10.8", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9900be224962d65a626072d8777f847ae5406c07547f0dc14c60048978c4b" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e9e4d3c2296bcec2c03f9f769ac9b2424d972c2fe7afc0b59235447ac3a5c3" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1 0.10.6", + "sha2 0.10.8", + "tracing", +] + +[[package]] +name = "aws-smithy-client" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710ca0f8dacddda5fbcaf5c3cd9d02da7913fd463a2ee9555b617bf168bedacb" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1ff11ee22de3581114b60d4ae8e700638dacb5b5bbe6769726e251e6c3f20a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dcab29afbea7726f5c10c7be0c38666d7eb07db551580b3b26ed7cfb5d1935" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5856d2f1063c0f726a85f32dcd2a9f5a1d994eb27b156abccafc7260f3f471d" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb33659b68480495b5f906b946c8642928440118b1d7e26a25a067303ca01a5" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c4b21ee0e30ff046e87c7b7e017b99d445b42a81fe52c6e5139b23b795a98ae" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2013465a070decdeb3e85ceb3370ae85ba05f56f914abfd89858d7281c4f12c3" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d27bfaa164aa94aac721726a83aa78abe708a275e88a573e103b4961c5f0ede" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f00f4b0cdd345686e6389f3343a3020f93232d20040802b87673ddc2d02956" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "base64-simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "781dd20c3aff0bd194fe7d2a977dd92f21c173891f3a03b677359e5fa457e5d5" +dependencies = [ + "simd-abstraction", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blake2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +dependencies = [ + "crypto-mac", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-streamer" version = "0.1.0" +dependencies = [ + "anyhow", + "near-lake-framework", + "redis", + "tokio", + "tokio-util 0.6.10", + "tracing", +] + +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive", + "hashbrown 0.13.2", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate", + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +dependencies = [ + "serde", +] + +[[package]] +name = "c2-chacha" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27dae93fe7b1e0424dc57179ac396908c26b035a87234809f5c4dfd1b47dc80" +dependencies = [ + "cipher", + "ppv-lite86", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util 0.7.10", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.39", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core 0.20.3", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "derive_builder" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + +[[package]] +name = "easy-ext" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53aff6fdc1b181225acdcb5b14c47106726fd8e486707315b1b138baed68ee31" + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "zeroize", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "enum-map" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e6b4f374c071b18172e23134e01026653dc980636ee139e0dfe59c538c61e5" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdb3d73d1beaf47c8593a1364e577fde072677cbfd103600345c0f547408cc0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + +[[package]] +name = "h2" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.2", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json_comments" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbbfed4e59ba9750e15ba154fdfd9329cee16ff3df539c2666b70f58cc32105" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "near-account-id" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0cb40869cab7f5232f934f45db35bffe0f2d2a7cb0cd0346202fbe4ebf2dd7" +dependencies = [ + "borsh", + "serde", +] + +[[package]] +name = "near-config-utils" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5523e7dce493c45bc3241eb3100d943ec471852f9b1f84b46a34789eadf17031" +dependencies = [ + "anyhow", + "json_comments", + "thiserror", + "tracing", +] + +[[package]] +name = "near-crypto" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6b382b626e7e0cd372d027c6672ac97b4b6ee6114288c9e58d8180b935d315" +dependencies = [ + "blake2", + "borsh", + "bs58", + "c2-chacha", + "curve25519-dalek", + "derive_more", + "ed25519-dalek", + "hex", + "near-account-id", + "near-config-utils", + "near-stdx", + "once_cell", + "primitive-types", + "rand 0.7.3", + "secp256k1", + "serde", + "serde_json", + "subtle", + "thiserror", +] + +[[package]] +name = "near-fmt" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c44c842c6cfcd9b8c387cccd4cd0619a5f21920cde5d5c292af3cc5d40510672" +dependencies = [ + "near-primitives-core", +] + +[[package]] +name = "near-indexer-primitives" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b76c87827dcae78979748c3864d209d5906163958a01551afc2092a8ad56fa39" +dependencies = [ + "near-primitives", + "serde", + "serde_json", +] + +[[package]] +name = "near-lake-framework" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fbfc3d7c294aa144c3a1817452931e91ce045ee1cf2e34b7b439d4695073634" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", + "derive_builder", + "futures", + "near-indexer-primitives", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "near-primitives" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f7051aaf199adc4d068620fca6d5f70f906a1540d03a8bb3701271f8881835" +dependencies = [ + "arbitrary", + "borsh", + "bytesize", + "cfg-if", + "chrono", + "derive_more", + "easy-ext", + "enum-map", + "hex", + "near-crypto", + "near-fmt", + "near-primitives-core", + "near-rpc-error-macro", + "near-stdx", + "near-vm-errors", + "num-rational", + "once_cell", + "primitive-types", + "rand 0.8.5", + "reed-solomon-erasure", + "serde", + "serde_json", + "serde_with", + "serde_yaml", + "smart-default", + "strum", + "thiserror", + "time", + "tracing", +] + +[[package]] +name = "near-primitives-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775fec19ef51a341abdbf792a9dda5b4cb89f488f681b2fd689b9321d24db47b" +dependencies = [ + "arbitrary", + "base64", + "borsh", + "bs58", + "derive_more", + "enum-map", + "near-account-id", + "num-rational", + "serde", + "serde_repr", + "serde_with", + "sha2 0.10.8", + "strum", + "thiserror", +] + +[[package]] +name = "near-rpc-error-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c1eda300e2e78f4f945ae58117d49e806899f4a51ee2faa09eda5ebc2e6571" +dependencies = [ + "quote", + "serde", + "syn 2.0.39", +] + +[[package]] +name = "near-rpc-error-macro" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d2dadd765101c77e664029dd6fbec090e696877d4ae903c620d02ceda4969a" +dependencies = [ + "fs2", + "near-rpc-error-core", + "serde", + "syn 2.0.39", +] + +[[package]] +name = "near-stdx" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6540152fba5e96fe5d575b79e8cd244cf2add747bb01362426bdc069bc3a23bc" + +[[package]] +name = "near-vm-errors" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec545d1bede0579e7c15dd2dce9b998dc975c52f2165702ff40bec7ff69728bb" +dependencies = [ + "borsh", + "near-account-id", + "near-rpc-error-macro", + "serde", + "strum", + "thiserror", +] + +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "outref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4" + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "primitive-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" +dependencies = [ + "fixed-hash", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.11", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redis" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152f3863635cbb76b73bc247845781098302c6c9ad2060e1a9a7de56840346b6" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "combine", + "futures", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1 0.6.1", + "tokio", + "tokio-util 0.7.10", + "url", +] + +[[package]] +name = "reed-solomon-erasure" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a415a013dd7c5d4221382329a5a3482566da675737494935cbbbcdec04662f9d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom 0.2.11", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +dependencies = [ + "log", + "ring 0.16.20", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + +[[package]] +name = "secp256k1" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" +dependencies = [ + "rand 0.8.5", + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" +dependencies = [ + "cc", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_with" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +dependencies = [ + "darling 0.20.3", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_yaml" +version = "0.9.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +dependencies = [ + "indexmap 2.1.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + +[[package]] +name = "simd-abstraction" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadb29c57caadc51ff8346233b5cec1d240b68ce55cf1afc764818791876987" +dependencies = [ + "outref", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "smart-default" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2 0.5.5", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index a09c10de5..cbe334501 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -6,5 +6,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.57" + redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } tracing = "0.1.34" +tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } +tokio-util = "0.6.7" + +near-lake-framework = "0.7.1" diff --git a/block-streamer/src/historical_block_processing.rs b/block-streamer/src/historical_block_processing.rs new file mode 100644 index 000000000..17155d1e8 --- /dev/null +++ b/block-streamer/src/historical_block_processing.rs @@ -0,0 +1,430 @@ +use crate::indexer_types::IndexerFunction; +use crate::s3; +use anyhow::{bail, Context}; +use aws_sdk_s3::Client as S3Client; +use chrono::{DateTime, LocalResult, TimeZone, Utc}; +use indexer_rule_type::indexer_rule::MatchingRule; +use indexer_rules_engine::types::indexer_rule_match::ChainId; +use near_jsonrpc_client::JsonRpcClient; +use near_jsonrpc_primitives::types::blocks::RpcBlockRequest; +use near_lake_framework::near_indexer_primitives::types::{BlockHeight, BlockId, BlockReference}; +use serde_json::from_str; +use tokio::task::JoinHandle; + +pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; +pub const UNINDEXED_BLOCKS_SOFT_LIMIT: u64 = 7200; + +pub struct Task { + handle: JoinHandle<()>, + cancellation_token: tokio_util::sync::CancellationToken, +} + +/// Represents the async task used to process and push historical messages +pub struct Streamer { + task: Option, +} + +impl Streamer { + pub fn new() -> Self { + Streamer { task: None } + } + + pub fn start( + &mut self, + current_block_height: BlockHeight, + indexer: IndexerFunction, + redis_connection_manager: storage::ConnectionManager, + s3_client: S3Client, + chain_id: ChainId, + json_rpc_client: JsonRpcClient, + ) -> anyhow::Result<()> { + if self.task.is_some() { + return Err(anyhow::anyhow!("Streamer has already been started",)); + } + + let cancellation_token = tokio_util::sync::CancellationToken::new(); + let cancellation_token_clone = cancellation_token.clone(); + + let handle = tokio::spawn(async move { + tokio::select! { + _ = cancellation_token_clone.cancelled() => { + tracing::info!( + target: crate::INDEXER, + "Cancelling existing historical backfill for indexer: {}", + indexer.get_full_name(), + ); + }, + _ = process_historical_messages_or_handle_error( + current_block_height, + indexer.clone(), + &redis_connection_manager, + &s3_client, + &chain_id, + &json_rpc_client, + ) => { + tracing::info!( + target: crate::INDEXER, + "Finished historical backfill for indexer: {}", + indexer.get_full_name(), + ); + } + } + }); + + self.task = Some(Task { + handle, + cancellation_token, + }); + + Ok(()) + } + + pub async fn cancel(&mut self) -> anyhow::Result<()> { + if let Some(task) = self.task.take() { + task.cancellation_token.cancel(); + task.handle.await?; + + return Ok(()); + } + + Err(anyhow::anyhow!( + "Attempted to cancel already cancelled, or not started, Streamer" + )) + } +} + +pub(crate) async fn process_historical_messages_or_handle_error( + current_block_height: BlockHeight, + indexer_function: IndexerFunction, + redis_connection_manager: &storage::ConnectionManager, + s3_client: &S3Client, + chain_id: &ChainId, + json_rpc_client: &JsonRpcClient, +) -> i64 { + match process_historical_messages( + current_block_height, + indexer_function, + redis_connection_manager, + s3_client, + chain_id, + json_rpc_client, + ) + .await + { + Ok(block_difference) => block_difference, + Err(err) => { + // todo: when Coordinator can send log messages to Runner, send this error to Runner + tracing::error!( + target: crate::INDEXER, + "Error processing historical messages: {:?}", + err + ); + 0 + } + } +} +pub(crate) async fn process_historical_messages( + current_block_height: BlockHeight, + indexer_function: IndexerFunction, + redis_connection_manager: &storage::ConnectionManager, + s3_client: &S3Client, + chain_id: &ChainId, + json_rpc_client: &JsonRpcClient, +) -> anyhow::Result { + let start_block = indexer_function.start_block_height.unwrap(); + let block_difference: i64 = (current_block_height - start_block) as i64; + match block_difference { + i64::MIN..=-1 => { + bail!( + "Skipping back fill, start_block_height is greater than current block height: {}", + indexer_function.get_full_name(), + ); + } + 0 => { + bail!( + "Skipping back fill, start_block_height is equal to current block height: {}", + indexer_function.get_full_name(), + ); + } + 1..=i64::MAX => { + tracing::info!( + target: crate::INDEXER, + "Back filling {block_difference} blocks from {start_block} to current block height {current_block_height}: {}", + indexer_function.get_full_name(), + ); + + storage::del( + redis_connection_manager, + storage::generate_historical_stream_key(&indexer_function.get_full_name()), + ) + .await?; + storage::sadd( + redis_connection_manager, + storage::STREAMS_SET_KEY, + storage::generate_historical_stream_key(&indexer_function.get_full_name()), + ) + .await?; + storage::set( + redis_connection_manager, + storage::generate_historical_storage_key(&indexer_function.get_full_name()), + serde_json::to_string(&indexer_function)?, + None, + ) + .await?; + + let start_date = + lookup_block_date_or_next_block_date(start_block, json_rpc_client).await?; + + let last_indexed_block = last_indexed_block_from_metadata(s3_client).await?; + + let blocks_from_index = filter_matching_blocks_from_index_files( + start_block, + &indexer_function, + s3_client, + start_date, + ) + .await?; + + tracing::info!( + target: crate::INDEXER, + "Flushing {} block heights from index files to historical Stream for indexer: {}", + blocks_from_index.len(), + indexer_function.get_full_name(), + ); + + for block in &blocks_from_index { + storage::xadd( + redis_connection_manager, + storage::generate_historical_stream_key(&indexer_function.get_full_name()), + &[("block_height", block)], + ) + .await?; + } + + // Check for the case where an index file is written right after we get the last_indexed_block metadata + let last_block_in_data = blocks_from_index.last().unwrap_or(&start_block); + let last_indexed_block = if last_block_in_data > &last_indexed_block { + *last_block_in_data + } else { + last_indexed_block + }; + + let unindexed_block_difference = current_block_height - last_indexed_block; + + tracing::info!( + target: crate::INDEXER, + "Filtering {} unindexed blocks from lake: from block {last_indexed_block} to {current_block_height} for indexer: {}", + unindexed_block_difference, + indexer_function.get_full_name(), + ); + + if unindexed_block_difference > UNINDEXED_BLOCKS_SOFT_LIMIT { + tracing::warn!( + target: crate::INDEXER, + "Unindexed block difference exceeds soft limit of: {UNINDEXED_BLOCKS_SOFT_LIMIT} for indexer: {}", + indexer_function.get_full_name(), + ); + } + + let lake_config = match &chain_id { + ChainId::Mainnet => near_lake_framework::LakeConfigBuilder::default().mainnet(), + ChainId::Testnet => near_lake_framework::LakeConfigBuilder::default().testnet(), + } + .start_block_height(last_indexed_block) + .build() + .context("Failed to build lake config")?; + + let (sender, mut stream) = near_lake_framework::streamer(lake_config); + + let mut filtered_block_count = 0; + while let Some(streamer_message) = stream.recv().await { + let block_height = streamer_message.block.header.height; + if block_height == current_block_height { + break; + } + + let matches = indexer_rules_engine::reduce_indexer_rule_matches( + &indexer_function.indexer_rule, + &streamer_message, + chain_id.clone(), + ); + + if !matches.is_empty() { + filtered_block_count += 1; + + storage::xadd( + redis_connection_manager, + storage::generate_historical_stream_key(&indexer_function.get_full_name()), + &[("block_height", block_height)], + ) + .await?; + } + } + drop(sender); + + tracing::info!( + target: crate::INDEXER, + "Flushed {} unindexed block heights to historical Stream for indexer: {}", + filtered_block_count, + indexer_function.get_full_name(), + ); + } + } + Ok(block_difference) +} + +pub(crate) async fn last_indexed_block_from_metadata( + s3_client: &S3Client, +) -> anyhow::Result { + let key = format!( + "{}/{}", + s3::INDEXED_ACTIONS_FILES_FOLDER, + "latest_block.json" + ); + let metadata = + s3::fetch_text_file_from_s3(s3::INDEXED_DATA_FILES_BUCKET, key, s3_client).await?; + + let metadata: serde_json::Value = serde_json::from_str(&metadata).unwrap(); + let last_indexed_block = metadata["last_indexed_block"].clone(); + let last_indexed_block = last_indexed_block + .as_str() + .context("No last_indexed_block found in latest_block.json")?; + let last_indexed_block = + from_str(last_indexed_block).context("last_indexed_block couldn't be converted to u64")?; + tracing::info!( + target: crate::INDEXER, + "Last indexed block from latest_block.json: {:?}", + last_indexed_block + ); + Ok(last_indexed_block) +} + +pub(crate) async fn filter_matching_blocks_from_index_files( + start_block_height: BlockHeight, + indexer_function: &IndexerFunction, + s3_client: &S3Client, + start_date: DateTime, +) -> anyhow::Result> { + let s3_bucket = s3::INDEXED_DATA_FILES_BUCKET; + + let mut needs_dedupe_and_sort = false; + let indexer_rule = &indexer_function.indexer_rule; + + let index_files_content = match &indexer_rule.matching_rule { + MatchingRule::ActionAny { + affected_account_id, + .. + } => { + if affected_account_id.contains('*') || affected_account_id.contains(',') { + needs_dedupe_and_sort = true; + } + s3::fetch_contract_index_files( + s3_client, + s3_bucket, + s3::INDEXED_ACTIONS_FILES_FOLDER, + start_date, + affected_account_id, + ) + .await + } + MatchingRule::ActionFunctionCall { .. } => { + bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer_function.account_id, indexer_function.function_name); + } + MatchingRule::Event { .. } => { + bail!("Event matching rule not yet supported for historical processing, function {:?} {:?}", indexer_function.account_id, indexer_function.function_name); + } + }?; + + tracing::info!( + target: crate::INDEXER, + "Found {file_count} index files for function {:?} {:?} with matching rule {indexer_rule:?}", + indexer_function.account_id, + indexer_function.function_name, + file_count = index_files_content.len() + ); + let mut blocks_to_process: Vec = + parse_blocks_from_index_files(index_files_content, start_block_height); + if needs_dedupe_and_sort { + blocks_to_process.sort(); + blocks_to_process.dedup(); + } + tracing::info!( + target: crate::INDEXER, + "Found {block_count} indexed blocks to process for function {:?} {:?}", + indexer_function.account_id, + indexer_function.function_name, + block_count = blocks_to_process.len() + ); + + Ok(blocks_to_process) +} + +fn parse_blocks_from_index_files( + index_files_content: Vec, + start_block_height: u64, +) -> Vec { + index_files_content + .into_iter() + .flat_map(|file_content| { + if let Ok(file_json) = serde_json::from_str::(&file_content) { + if let Some(block_heights) = file_json["heights"].as_array() { + block_heights + .iter() + .map(|block_height| block_height.as_u64().unwrap()) + .collect::>() + .into_iter() + .filter(|block_height| block_height >= &start_block_height) + .collect() + } else { + tracing::error!( + target: crate::INDEXER, + "Unable to parse index file, no heights found: {:?}", + file_content + ); + vec![] + } + } else { + tracing::error!( + target: crate::INDEXER, + "Unable to parse index file: {:?}", + file_content + ); + vec![] + } + }) + .collect::>() +} + +// if block does not exist, try next block, up to MAX_RPC_BLOCKS_TO_PROCESS (20) blocks +pub async fn lookup_block_date_or_next_block_date( + block_height: u64, + client: &JsonRpcClient, +) -> anyhow::Result> { + let mut current_block_height = block_height; + let mut retry_count = 0; + loop { + let request = RpcBlockRequest { + block_reference: BlockReference::BlockId(BlockId::Height(current_block_height)), + }; + + match client.call(request).await { + Ok(response) => { + let header = response.header; + let timestamp_nanosec = header.timestamp_nanosec; + return match Utc.timestamp_opt((timestamp_nanosec / 1000000000) as i64, 0) { + LocalResult::Single(date) => Ok(date), + LocalResult::Ambiguous(date, _) => Ok(date), + LocalResult::None => Err(anyhow::anyhow!("Unable to get block timestamp")), + }; + } + Err(_) => { + tracing::debug!("RPC failed to get block: {:?}", current_block_height); + retry_count += 1; + if retry_count > MAX_RPC_BLOCKS_TO_PROCESS { + return Err(anyhow::anyhow!("Unable to get block")); + } + current_block_height += 1; + } + } + } +} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index e7a11a969..a3a18b9ac 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -1,3 +1,8 @@ +mod block_streamer; +mod redis; + +pub(crate) const LOG_TARGET: &str = "block_streamer"; + fn main() { println!("Hello, world!"); } From 998c823bfcd8cf0456825284c68d77e0437092bf Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 09:20:08 +1300 Subject: [PATCH 04/40] feat: Add supporting modules to make things compile --- block-streamer/Cargo.lock | 1246 ++++++++++++++--- block-streamer/Cargo.toml | 11 +- .../src/historical_block_processing.rs | 73 +- block-streamer/src/indexer_rule.rs | 54 + block-streamer/src/indexer_types.rs | 29 + block-streamer/src/main.rs | 3 + block-streamer/src/rules/matcher.rs | 141 ++ block-streamer/src/rules/mod.rs | 79 ++ block-streamer/src/rules/outcomes_reducer.rs | 262 ++++ block-streamer/src/rules/types/events.rs | 20 + .../src/rules/types/indexer_rule_match.rs | 146 ++ block-streamer/src/rules/types/mod.rs | 3 + .../src/rules/types/transactions.rs | 20 + block-streamer/src/s3.rs | 342 +++++ 14 files changed, 2210 insertions(+), 219 deletions(-) create mode 100644 block-streamer/src/indexer_rule.rs create mode 100644 block-streamer/src/indexer_types.rs create mode 100644 block-streamer/src/rules/matcher.rs create mode 100644 block-streamer/src/rules/mod.rs create mode 100644 block-streamer/src/rules/outcomes_reducer.rs create mode 100644 block-streamer/src/rules/types/events.rs create mode 100644 block-streamer/src/rules/types/indexer_rule_match.rs create mode 100644 block-streamer/src/rules/types/mod.rs create mode 100644 block-streamer/src/rules/types/transactions.rs create mode 100644 block-streamer/src/s3.rs diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index e7586d9e3..bf57e90c6 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -3,19 +3,60 @@ version = 3 [[package]] -name = "addr2line" -version = "0.21.0" +name = "actix" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "cba56612922b907719d4a01cf11c8d5b458e7d3dba946d0435f20f58d6795ed2" dependencies = [ - "gimli", + "actix-macros", + "actix-rt", + "actix_derive", + "bitflags 2.4.1", + "bytes", + "crossbeam-channel", + "futures-core", + "futures-sink", + "futures-task", + "futures-util", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util 0.7.3", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.39", +] + +[[package]] +name = "actix-rt" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +dependencies = [ + "futures-core", + "tokio", ] [[package]] -name = "adler" -version = "1.0.2" +name = "actix_derive" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] [[package]] name = "ahash" @@ -107,6 +148,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -211,7 +263,7 @@ dependencies = [ "aws-types", "bytes", "bytes-utils", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "once_cell", @@ -355,7 +407,7 @@ dependencies = [ "aws-smithy-http-tower", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "hyper", @@ -397,7 +449,7 @@ dependencies = [ "pin-project-lite", "pin-utils", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.3", "tracing", ] @@ -475,19 +527,10 @@ dependencies = [ ] [[package]] -name = "backtrace" -version = "0.3.69" +name = "base64" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" @@ -510,6 +553,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "blake2" version = "0.9.2" @@ -544,11 +593,21 @@ name = "block-streamer" version = "0.1.0" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-s3", + "borsh", + "chrono", + "futures", + "near-jsonrpc-client", + "near-jsonrpc-primitives", "near-lake-framework", "redis", + "serde", + "serde_json", "tokio", "tokio-util 0.6.10", "tracing", + "wildmatch", ] [[package]] @@ -688,6 +747,45 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "combine" version = "4.6.6" @@ -699,7 +797,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.3", ] [[package]] @@ -751,6 +849,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -796,18 +913,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" -dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", + "darling_core", + "darling_macro", ] [[package]] @@ -824,42 +931,17 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "darling_core" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.39", -] - [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core 0.14.4", + "darling_core", "quote", "syn 1.0.109", ] -[[package]] -name = "darling_macro" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" -dependencies = [ - "darling_core 0.20.3", - "quote", - "syn 2.0.39", -] - [[package]] name = "deranged" version = "0.3.9" @@ -867,7 +949,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ "powerfmt", - "serde", ] [[package]] @@ -896,7 +977,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.4", + "darling", "proc-macro2", "quote", "syn 1.0.109", @@ -980,6 +1061,15 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "enum-map" version = "2.7.2" @@ -1006,6 +1096,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -1015,6 +1115,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "fixed-hash" version = "0.7.0" @@ -1024,12 +1130,33 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -1170,12 +1297,6 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] -[[package]] -name = "gimli" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" - [[package]] name = "h2" version = "0.3.22" @@ -1191,7 +1312,7 @@ dependencies = [ "indexmap 2.1.0", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.3", "tracing", ] @@ -1216,12 +1337,30 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.3" @@ -1246,6 +1385,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.11" @@ -1297,7 +1445,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2", "tokio", "tower-service", "tracing", @@ -1319,6 +1467,31 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1366,7 +1539,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -1377,7 +1549,6 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", - "serde", ] [[package]] @@ -1389,6 +1560,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1422,12 +1608,37 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +[[package]] +name = "linux-raw-sys" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1445,13 +1656,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] -name = "miniz_oxide" -version = "0.7.1" +name = "mime" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" @@ -1460,25 +1668,73 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "near-account-id" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0cb40869cab7f5232f934f45db35bffe0f2d2a7cb0cd0346202fbe4ebf2dd7" +checksum = "12791d0f273e04609010d68deb6e1cd940659ad420edfa2e48238117154f1d5b" dependencies = [ + "arbitrary", "borsh", "serde", ] +[[package]] +name = "near-chain-configs" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84781ba3321e28640c5f0453d4d84305135db6edc399c51b7ee405ebb9381eaa" +dependencies = [ + "anyhow", + "chrono", + "derive_more", + "near-config-utils", + "near-crypto", + "near-o11y", + "near-primitives", + "num-rational", + "once_cell", + "serde", + "serde_json", + "sha2 0.10.8", + "smart-default", + "tracing", +] + [[package]] name = "near-config-utils" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5523e7dce493c45bc3241eb3100d943ec471852f9b1f84b46a34789eadf17031" +checksum = "137b6bbd477dc0ff1149e22c5e1f3f168a1f37d9d67f1519f058a5db81df3506" dependencies = [ "anyhow", "json_comments", @@ -1488,9 +1744,9 @@ dependencies = [ [[package]] name = "near-crypto" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6b382b626e7e0cd372d027c6672ac97b4b6ee6114288c9e58d8180b935d315" +checksum = "659a96750c4d933e4f00a50c66ba9948a32b862e5ecd6a952beee881b1cd2aaf" dependencies = [ "blake2", "borsh", @@ -1514,30 +1770,56 @@ dependencies = [ ] [[package]] -name = "near-fmt" -version = "0.17.0" +name = "near-indexer-primitives" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c44c842c6cfcd9b8c387cccd4cd0619a5f21920cde5d5c292af3cc5d40510672" +checksum = "56d10699cd998980a8965ee0d1922a77585b7ed1031934de29fb982ef4ae6ad2" dependencies = [ - "near-primitives-core", + "near-primitives", + "serde", + "serde_json", ] [[package]] -name = "near-indexer-primitives" -version = "0.17.0" +name = "near-jsonrpc-client" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf020bd5fb7952c8f005ab789ac6f7a2015557630571362bcbaf2ead27c5cb6" +dependencies = [ + "borsh", + "lazy_static", + "log", + "near-chain-configs", + "near-crypto", + "near-jsonrpc-primitives", + "near-primitives", + "reqwest", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "near-jsonrpc-primitives" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b76c87827dcae78979748c3864d209d5906163958a01551afc2092a8ad56fa39" +checksum = "f0968cc71825e020d75f5a1f3577ac0379859ec7b14351ed5754fef1cc89e0d7" dependencies = [ + "arbitrary", + "near-chain-configs", + "near-crypto", "near-primitives", + "near-rpc-error-macro", "serde", "serde_json", + "thiserror", ] [[package]] name = "near-lake-framework" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbfc3d7c294aa144c3a1817452931e91ce045ee1cf2e34b7b439d4695073634" +checksum = "31b733c11d21cb97f66721ebea525c1fd4fb6fdd4aeab5c7c9562363230f071e" dependencies = [ "anyhow", "async-stream", @@ -1557,11 +1839,37 @@ dependencies = [ "tracing", ] +[[package]] +name = "near-o11y" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445cbd86499dbabf68d34d95d9a45c680da5446e226f4982ce488fcf8556b23f" +dependencies = [ + "actix", + "atty", + "clap", + "near-crypto", + "near-primitives-core", + "once_cell", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "prometheus", + "serde", + "strum", + "thiserror", + "tokio", + "tracing", + "tracing-appender", + "tracing-opentelemetry", + "tracing-subscriber", +] + [[package]] name = "near-primitives" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f7051aaf199adc4d068620fca6d5f70f906a1540d03a8bb3701271f8881835" +checksum = "c4c030f28e8f988698145e7753b83bb54c05838c3afdd44835cc7c32c327ea1e" dependencies = [ "arbitrary", "borsh", @@ -1573,7 +1881,7 @@ dependencies = [ "enum-map", "hex", "near-crypto", - "near-fmt", + "near-o11y", "near-primitives-core", "near-rpc-error-macro", "near-stdx", @@ -1585,23 +1893,23 @@ dependencies = [ "reed-solomon-erasure", "serde", "serde_json", - "serde_with", "serde_yaml", "smart-default", "strum", "thiserror", "time", + "tokio", "tracing", ] [[package]] name = "near-primitives-core" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775fec19ef51a341abdbf792a9dda5b4cb89f488f681b2fd689b9321d24db47b" +checksum = "fe2059d16efc42ef7f9514f30910d32b67c01fee9b70c1fd28d50545ca145d88" dependencies = [ "arbitrary", - "base64", + "base64 0.13.1", "borsh", "bs58", "derive_more", @@ -1610,7 +1918,6 @@ dependencies = [ "num-rational", "serde", "serde_repr", - "serde_with", "sha2 0.10.8", "strum", "thiserror", @@ -1618,38 +1925,38 @@ dependencies = [ [[package]] name = "near-rpc-error-core" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c1eda300e2e78f4f945ae58117d49e806899f4a51ee2faa09eda5ebc2e6571" +checksum = "28f8f38dcfeba3d0d3bc60ce292ddb1f76a428a590e32de7fc3d5d431b9635ea" dependencies = [ "quote", "serde", - "syn 2.0.39", + "syn 1.0.109", ] [[package]] name = "near-rpc-error-macro" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d2dadd765101c77e664029dd6fbec090e696877d4ae903c620d02ceda4969a" +checksum = "78ca4e8d6887b344f3e2f8281c1ad2cc93dcbcb683b546726a4ce1ab1dfa623a" dependencies = [ "fs2", "near-rpc-error-core", "serde", - "syn 2.0.39", + "syn 1.0.109", ] [[package]] name = "near-stdx" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6540152fba5e96fe5d575b79e8cd244cf2add747bb01362426bdc069bc3a23bc" +checksum = "bc1279be274b9a49c2cb4b62541241a1ff6745cb77ca81ece7f949cfbc229bff" [[package]] name = "near-vm-errors" -version = "0.17.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec545d1bede0579e7c15dd2dce9b998dc975c52f2165702ff40bec7ff69728bb" +checksum = "8b2d3eab1e050fdc3e036c803784cf45582661ae2dd07bac3bd373ba9c049715" dependencies = [ "borsh", "near-account-id", @@ -1659,6 +1966,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.3.3" @@ -1708,19 +2025,10 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.3", "libc", ] -[[package]] -name = "object" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.18.0" @@ -1733,24 +2041,155 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "opentelemetry", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "outref" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "percent-encoding" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -1783,6 +2222,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1814,6 +2259,30 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -1823,6 +2292,80 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes", + "heck 0.3.3", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.33" @@ -1921,10 +2464,19 @@ dependencies = [ "ryu", "sha1 0.6.1", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.3", "url", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "reed-solomon-erasure" version = "4.0.2" @@ -1942,8 +2494,17 @@ checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1954,15 +2515,59 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.2", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -1992,12 +2597,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - [[package]] name = "rustc_version" version = "0.4.0" @@ -2007,6 +2606,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustls" version = "0.20.9" @@ -2037,7 +2649,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.5", ] [[package]] @@ -2061,6 +2673,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "sct" version = "0.7.1" @@ -2073,9 +2691,9 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.27.0" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" +checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" dependencies = [ "rand 0.8.5", "secp256k1-sys", @@ -2083,9 +2701,9 @@ dependencies = [ [[package]] name = "secp256k1-sys" -version = "0.8.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" +checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" dependencies = [ "cc", ] @@ -2096,7 +2714,7 @@ version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2162,32 +2780,15 @@ dependencies = [ ] [[package]] -name = "serde_with" -version = "3.4.0" +name = "serde_urlencoded" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ - "base64", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.1.0", + "form_urlencoded", + "itoa", + "ryu", "serde", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" -dependencies = [ - "darling 0.20.3", - "proc-macro2", - "quote", - "syn 2.0.39", ] [[package]] @@ -2253,6 +2854,24 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "signature" version = "1.6.4" @@ -2304,16 +2923,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "socket2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" -dependencies = [ - "libc", - "windows-sys", -] - [[package]] name = "spin" version = "0.5.2" @@ -2353,7 +2962,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -2388,6 +2997,55 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand 2.0.1", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.50" @@ -2408,6 +3066,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.30" @@ -2454,30 +3122,53 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ - "backtrace", "bytes", "libc", + "memchr", "mio", "num_cpus", + "once_cell", + "parking_lot", "pin-project-lite", - "socket2 0.5.5", + "signal-hook-registry", + "socket2", "tokio-macros", - "windows-sys", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 1.0.109", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", ] [[package]] @@ -2518,9 +3209,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", @@ -2539,6 +3230,49 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.13.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn 1.0.109", +] + [[package]] name = "tower" version = "0.4.13" @@ -2547,9 +3281,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util 0.7.3", "tower-layer", "tower-service", "tracing", @@ -2579,6 +3317,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.27" @@ -2597,6 +3347,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log 0.2.0", ] [[package]] @@ -2644,6 +3459,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + [[package]] name = "unsafe-libyaml" version = "0.2.9" @@ -2679,6 +3500,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -2731,6 +3564,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.88" @@ -2780,6 +3625,24 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "wildmatch" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee583bdc5ff1cf9db20e9db5bb3ff4c3089a8f6b8b31aff265c9aba85812db86" + [[package]] name = "winapi" version = "0.3.9" @@ -2796,6 +3659,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2877,6 +3749,16 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index cbe334501..a0b3800e9 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -7,10 +7,19 @@ edition = "2021" [dependencies] anyhow = "1.0.57" - +aws-sdk-s3 = "0.23.0" +aws-config = "0.53.0" +borsh = "0.10.2" +chrono = "0.4.25" +futures = "0.3.5" redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.55" tracing = "0.1.34" tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-util = "0.6.7" +wildmatch = "2.1.1" near-lake-framework = "0.7.1" +near-jsonrpc-client = "0.5.1" +near-jsonrpc-primitives = "0.16.0" diff --git a/block-streamer/src/historical_block_processing.rs b/block-streamer/src/historical_block_processing.rs index 17155d1e8..44ee60c47 100644 --- a/block-streamer/src/historical_block_processing.rs +++ b/block-streamer/src/historical_block_processing.rs @@ -1,10 +1,10 @@ use crate::indexer_types::IndexerFunction; +use crate::rules::types::indexer_rule_match::ChainId; +use crate::rules::MatchingRule; use crate::s3; use anyhow::{bail, Context}; use aws_sdk_s3::Client as S3Client; use chrono::{DateTime, LocalResult, TimeZone, Utc}; -use indexer_rule_type::indexer_rule::MatchingRule; -use indexer_rules_engine::types::indexer_rule_match::ChainId; use near_jsonrpc_client::JsonRpcClient; use near_jsonrpc_primitives::types::blocks::RpcBlockRequest; use near_lake_framework::near_indexer_primitives::types::{BlockHeight, BlockId, BlockReference}; @@ -19,27 +19,26 @@ pub struct Task { cancellation_token: tokio_util::sync::CancellationToken, } -/// Represents the async task used to process and push historical messages -pub struct Streamer { +pub struct BlockStreamer { task: Option, } -impl Streamer { +impl BlockStreamer { pub fn new() -> Self { - Streamer { task: None } + Self { task: None } } pub fn start( &mut self, current_block_height: BlockHeight, indexer: IndexerFunction, - redis_connection_manager: storage::ConnectionManager, + redis_connection_manager: crate::redis::ConnectionManager, s3_client: S3Client, chain_id: ChainId, json_rpc_client: JsonRpcClient, ) -> anyhow::Result<()> { if self.task.is_some() { - return Err(anyhow::anyhow!("Streamer has already been started",)); + return Err(anyhow::anyhow!("BlockStreamer has already been started",)); } let cancellation_token = tokio_util::sync::CancellationToken::new(); @@ -49,7 +48,7 @@ impl Streamer { tokio::select! { _ = cancellation_token_clone.cancelled() => { tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Cancelling existing historical backfill for indexer: {}", indexer.get_full_name(), ); @@ -63,7 +62,7 @@ impl Streamer { &json_rpc_client, ) => { tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Finished historical backfill for indexer: {}", indexer.get_full_name(), ); @@ -88,7 +87,7 @@ impl Streamer { } Err(anyhow::anyhow!( - "Attempted to cancel already cancelled, or not started, Streamer" + "Attempted to cancel already cancelled, or not started, BlockStreamer" )) } } @@ -96,7 +95,7 @@ impl Streamer { pub(crate) async fn process_historical_messages_or_handle_error( current_block_height: BlockHeight, indexer_function: IndexerFunction, - redis_connection_manager: &storage::ConnectionManager, + redis_connection_manager: &crate::redis::ConnectionManager, s3_client: &S3Client, chain_id: &ChainId, json_rpc_client: &JsonRpcClient, @@ -115,7 +114,7 @@ pub(crate) async fn process_historical_messages_or_handle_error( Err(err) => { // todo: when Coordinator can send log messages to Runner, send this error to Runner tracing::error!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Error processing historical messages: {:?}", err ); @@ -126,7 +125,7 @@ pub(crate) async fn process_historical_messages_or_handle_error( pub(crate) async fn process_historical_messages( current_block_height: BlockHeight, indexer_function: IndexerFunction, - redis_connection_manager: &storage::ConnectionManager, + redis_connection_manager: &crate::redis::ConnectionManager, s3_client: &S3Client, chain_id: &ChainId, json_rpc_client: &JsonRpcClient, @@ -148,25 +147,25 @@ pub(crate) async fn process_historical_messages( } 1..=i64::MAX => { tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Back filling {block_difference} blocks from {start_block} to current block height {current_block_height}: {}", indexer_function.get_full_name(), ); - storage::del( + crate::redis::del( redis_connection_manager, - storage::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), ) .await?; - storage::sadd( + crate::redis::sadd( redis_connection_manager, - storage::STREAMS_SET_KEY, - storage::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::STREAMS_SET_KEY, + crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), ) .await?; - storage::set( + crate::redis::set( redis_connection_manager, - storage::generate_historical_storage_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), serde_json::to_string(&indexer_function)?, None, ) @@ -186,16 +185,16 @@ pub(crate) async fn process_historical_messages( .await?; tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Flushing {} block heights from index files to historical Stream for indexer: {}", blocks_from_index.len(), indexer_function.get_full_name(), ); for block in &blocks_from_index { - storage::xadd( + crate::redis::xadd( redis_connection_manager, - storage::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), &[("block_height", block)], ) .await?; @@ -212,7 +211,7 @@ pub(crate) async fn process_historical_messages( let unindexed_block_difference = current_block_height - last_indexed_block; tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Filtering {} unindexed blocks from lake: from block {last_indexed_block} to {current_block_height} for indexer: {}", unindexed_block_difference, indexer_function.get_full_name(), @@ -220,7 +219,7 @@ pub(crate) async fn process_historical_messages( if unindexed_block_difference > UNINDEXED_BLOCKS_SOFT_LIMIT { tracing::warn!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Unindexed block difference exceeds soft limit of: {UNINDEXED_BLOCKS_SOFT_LIMIT} for indexer: {}", indexer_function.get_full_name(), ); @@ -243,7 +242,7 @@ pub(crate) async fn process_historical_messages( break; } - let matches = indexer_rules_engine::reduce_indexer_rule_matches( + let matches = crate::rules::reduce_indexer_rule_matches( &indexer_function.indexer_rule, &streamer_message, chain_id.clone(), @@ -252,9 +251,11 @@ pub(crate) async fn process_historical_messages( if !matches.is_empty() { filtered_block_count += 1; - storage::xadd( + crate::redis::xadd( redis_connection_manager, - storage::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key( + &indexer_function.get_full_name(), + ), &[("block_height", block_height)], ) .await?; @@ -263,7 +264,7 @@ pub(crate) async fn process_historical_messages( drop(sender); tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Flushed {} unindexed block heights to historical Stream for indexer: {}", filtered_block_count, indexer_function.get_full_name(), @@ -292,7 +293,7 @@ pub(crate) async fn last_indexed_block_from_metadata( let last_indexed_block = from_str(last_indexed_block).context("last_indexed_block couldn't be converted to u64")?; tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Last indexed block from latest_block.json: {:?}", last_indexed_block ); @@ -336,7 +337,7 @@ pub(crate) async fn filter_matching_blocks_from_index_files( }?; tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Found {file_count} index files for function {:?} {:?} with matching rule {indexer_rule:?}", indexer_function.account_id, indexer_function.function_name, @@ -349,7 +350,7 @@ pub(crate) async fn filter_matching_blocks_from_index_files( blocks_to_process.dedup(); } tracing::info!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Found {block_count} indexed blocks to process for function {:?} {:?}", indexer_function.account_id, indexer_function.function_name, @@ -377,7 +378,7 @@ fn parse_blocks_from_index_files( .collect() } else { tracing::error!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Unable to parse index file, no heights found: {:?}", file_content ); @@ -385,7 +386,7 @@ fn parse_blocks_from_index_files( } } else { tracing::error!( - target: crate::INDEXER, + target: crate::LOG_TARGET, "Unable to parse index file: {:?}", file_content ); diff --git a/block-streamer/src/indexer_rule.rs b/block-streamer/src/indexer_rule.rs new file mode 100644 index 000000000..5ae310e28 --- /dev/null +++ b/block-streamer/src/indexer_rule.rs @@ -0,0 +1,54 @@ +#[cfg(not(feature = "near-sdk"))] +use borsh::{self, BorshDeserialize, BorshSerialize}; +#[cfg(not(feature = "near-sdk"))] +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "near-sdk")] +use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; +#[cfg(feature = "near-sdk")] +use near_sdk::serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +pub struct IndexerRule { + pub indexer_rule_kind: IndexerRuleKind, + pub matching_rule: MatchingRule, + pub id: Option, + pub name: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +pub enum IndexerRuleKind { + Action, + Event, + AnyBlock, + Shard, +} +// future: ComposedRuleKind for multiple actions or events + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Status { + Any, + Success, + Fail, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(tag = "rule", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum MatchingRule { + ActionAny { + affected_account_id: String, + status: Status, + }, + ActionFunctionCall { + affected_account_id: String, + status: Status, + function: String, + }, + Event { + contract_account_id: String, + standard: String, + version: String, + event: String, + }, +} diff --git a/block-streamer/src/indexer_types.rs b/block-streamer/src/indexer_types.rs new file mode 100644 index 000000000..f3a155c16 --- /dev/null +++ b/block-streamer/src/indexer_types.rs @@ -0,0 +1,29 @@ +use crate::rules::IndexerRule; +use near_lake_framework::near_indexer_primitives::types::AccountId; +use std::collections::HashMap; + +pub type IndexerRegistry = HashMap>; + +#[derive( + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, + Clone, + Debug, +)] +pub struct IndexerFunction { + pub account_id: AccountId, + pub function_name: String, + pub code: String, + pub start_block_height: Option, + pub schema: Option, + pub provisioned: bool, + pub indexer_rule: IndexerRule, +} + +impl IndexerFunction { + pub fn get_full_name(&self) -> String { + format!("{}/{}", self.account_id, self.function_name) + } +} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index a3a18b9ac..4beb71f54 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -1,5 +1,8 @@ mod block_streamer; +mod indexer_types; mod redis; +mod rules; +mod s3; pub(crate) const LOG_TARGET: &str = "block_streamer"; diff --git a/block-streamer/src/rules/matcher.rs b/block-streamer/src/rules/matcher.rs new file mode 100644 index 000000000..ee1149a4d --- /dev/null +++ b/block-streamer/src/rules/matcher.rs @@ -0,0 +1,141 @@ +use near_lake_framework::near_indexer_primitives::{ + views::{ActionView, ExecutionStatusView, ReceiptEnumView}, + IndexerExecutionOutcomeWithReceipt, +}; + +use crate::rules::types::events::Event; +use crate::rules::{MatchingRule, Status}; + +pub fn matches( + matching_rule: &MatchingRule, + receipt_execution_outcome: &IndexerExecutionOutcomeWithReceipt, +) -> bool { + match matching_rule { + MatchingRule::ActionAny { + affected_account_id, + status, + } => match_action_any(affected_account_id, status, receipt_execution_outcome), + MatchingRule::ActionFunctionCall { + affected_account_id, + status, + function, + } => match_action_function_call( + affected_account_id, + status, + function, + receipt_execution_outcome, + ), + MatchingRule::Event { + contract_account_id, + event, + standard, + version, + } => match_event( + contract_account_id, + event, + standard, + version, + receipt_execution_outcome, + ), + } +} + +fn match_action_any( + account_id: &str, + status: &Status, + outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt, +) -> bool { + if match_account(account_id, outcome_with_receipt) { + return match_status( + status, + &outcome_with_receipt.execution_outcome.outcome.status, + ); + } + false +} + +fn match_action_function_call( + account_id: &str, + status: &Status, + function: &str, + outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt, +) -> bool { + if match_account(account_id, outcome_with_receipt) { + if let ReceiptEnumView::Action { actions, .. } = &outcome_with_receipt.receipt.receipt { + let is_any_matching_function_call = actions.iter().any(|action| { + if let ActionView::FunctionCall { method_name, .. } = action { + wildmatch::WildMatch::new(function).matches(method_name) + } else { + false + } + }); + if is_any_matching_function_call { + return match_status( + status, + &outcome_with_receipt.execution_outcome.outcome.status, + ); + } else { + return false; + } + } + } + false +} + +fn match_event( + account_id: &str, + event: &str, + standard: &str, + version: &str, + outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt, +) -> bool { + if match_account(account_id, outcome_with_receipt) { + outcome_with_receipt + .execution_outcome + .outcome + .logs + .iter() + .filter_map(|log| Event::from_log(log).ok()) + .any(|near_event| { + vec![ + wildmatch::WildMatch::new(event).matches(&near_event.event), + wildmatch::WildMatch::new(standard).matches(&near_event.standard), + wildmatch::WildMatch::new(version).matches(&near_event.version), + ] + .into_iter() + .all(|val| val) + }) + } else { + false + } +} + +fn match_account( + account_id: &str, + outcome_with_receipt: &IndexerExecutionOutcomeWithReceipt, +) -> bool { + match account_id { + x if x.contains(',') => x + .split(',') + .any(|sub_account_id| match_account(sub_account_id.trim(), outcome_with_receipt)), + _ => { + wildmatch::WildMatch::new(account_id).matches(&outcome_with_receipt.receipt.receiver_id) + || wildmatch::WildMatch::new(account_id) + .matches(&outcome_with_receipt.receipt.predecessor_id) + } + } +} + +fn match_status(status: &Status, execution_outcome_status: &ExecutionStatusView) -> bool { + match status { + Status::Any => true, + Status::Success => matches!( + execution_outcome_status, + ExecutionStatusView::SuccessValue(_) | ExecutionStatusView::SuccessReceiptId(_) + ), + Status::Fail => !matches!( + execution_outcome_status, + ExecutionStatusView::SuccessValue(_) | ExecutionStatusView::SuccessReceiptId(_) + ), + } +} diff --git a/block-streamer/src/rules/mod.rs b/block-streamer/src/rules/mod.rs new file mode 100644 index 000000000..a5b0e9292 --- /dev/null +++ b/block-streamer/src/rules/mod.rs @@ -0,0 +1,79 @@ +pub mod matcher; +pub mod outcomes_reducer; +pub mod types; + +use near_lake_framework::near_indexer_primitives::StreamerMessage; +use types::indexer_rule_match::{ChainId, IndexerRuleMatch}; + +#[cfg(not(feature = "near-sdk"))] +use borsh::{self, BorshDeserialize, BorshSerialize}; +#[cfg(not(feature = "near-sdk"))] +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "near-sdk")] +use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; +#[cfg(feature = "near-sdk")] +use near_sdk::serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +pub struct IndexerRule { + pub indexer_rule_kind: IndexerRuleKind, + pub matching_rule: MatchingRule, + pub id: Option, + pub name: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +pub enum IndexerRuleKind { + Action, + Event, + AnyBlock, + Shard, +} +// future: ComposedRuleKind for multiple actions or events + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum Status { + Any, + Success, + Fail, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] +#[serde(tag = "rule", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum MatchingRule { + ActionAny { + affected_account_id: String, + status: Status, + }, + ActionFunctionCall { + affected_account_id: String, + status: Status, + function: String, + }, + Event { + contract_account_id: String, + standard: String, + version: String, + event: String, + }, +} + +pub fn reduce_indexer_rule_matches( + indexer_rule: &IndexerRule, + streamer_message: &StreamerMessage, + chain_id: ChainId, +) -> Vec { + match &indexer_rule.matching_rule { + MatchingRule::ActionAny { .. } + | MatchingRule::ActionFunctionCall { .. } + | MatchingRule::Event { .. } => { + outcomes_reducer::reduce_indexer_rule_matches_from_outcomes( + indexer_rule, + streamer_message, + chain_id, + ) + } + } +} diff --git a/block-streamer/src/rules/outcomes_reducer.rs b/block-streamer/src/rules/outcomes_reducer.rs new file mode 100644 index 000000000..72433c9b7 --- /dev/null +++ b/block-streamer/src/rules/outcomes_reducer.rs @@ -0,0 +1,262 @@ +use crate::rules::matcher; +use crate::rules::types::events::Event; +use crate::rules::types::indexer_rule_match::{ChainId, IndexerRuleMatch, IndexerRuleMatchPayload}; +use crate::rules::{IndexerRule, MatchingRule}; +use near_lake_framework::near_indexer_primitives::{ + IndexerExecutionOutcomeWithReceipt, StreamerMessage, +}; + +pub fn reduce_indexer_rule_matches_from_outcomes( + indexer_rule: &IndexerRule, + streamer_message: &StreamerMessage, + chain_id: ChainId, +) -> Vec { + streamer_message + .shards + .iter() + .flat_map(|shard| { + shard + .receipt_execution_outcomes + .iter() + // future: when extracting Actions, Events, etc this will be a filter operation + .find(|receipt_execution_outcome| { + matcher::matches(&indexer_rule.matching_rule, receipt_execution_outcome) + }) + }) + .map(|receipt_execution_outcome| { + build_indexer_rule_match( + indexer_rule, + receipt_execution_outcome, + streamer_message.block.header.hash.to_string(), + streamer_message.block.header.height, + chain_id.clone(), + ) + }) + .collect() +} + +fn build_indexer_rule_match( + indexer_rule: &IndexerRule, + receipt_execution_outcome: &IndexerExecutionOutcomeWithReceipt, + block_header_hash: String, + block_height: u64, + chain_id: ChainId, +) -> IndexerRuleMatch { + IndexerRuleMatch { + chain_id: chain_id.clone(), + indexer_rule_id: indexer_rule.id, + indexer_rule_name: indexer_rule.name.clone(), + payload: build_indexer_rule_match_payload( + indexer_rule, + receipt_execution_outcome, + block_header_hash, + ), + block_height, + } +} + +fn build_indexer_rule_match_payload( + indexer_rule: &IndexerRule, + receipt_execution_outcome: &IndexerExecutionOutcomeWithReceipt, + block_header_hash: String, +) -> IndexerRuleMatchPayload { + // future enhancement will extract and enrich fields from block & context as + // specified in the indexer function config. + let transaction_hash = None; + + match &indexer_rule.matching_rule { + MatchingRule::ActionAny { .. } | MatchingRule::ActionFunctionCall { .. } => { + IndexerRuleMatchPayload::Actions { + block_hash: block_header_hash.to_string(), + receipt_id: receipt_execution_outcome.receipt.receipt_id.to_string(), + transaction_hash, + } + } + MatchingRule::Event { + event, + standard, + version, + .. + } => { + let event = receipt_execution_outcome + .execution_outcome + .outcome + .logs + .iter() + .filter_map(|log| Event::from_log(log).ok()) + .filter_map(|near_event| { + if vec![ + wildmatch::WildMatch::new(event).matches(&near_event.event), + wildmatch::WildMatch::new(standard).matches(&near_event.standard), + wildmatch::WildMatch::new(version).matches(&near_event.version), + ].into_iter().all(|val| val) { + Some(near_event) + } else { + None + } + }) + .collect::>() + .first() + .expect("Failed to get the matched Event itself while building the IndexerRuleMatchPayload") + .clone(); + + IndexerRuleMatchPayload::Events { + block_hash: block_header_hash.to_string(), + receipt_id: receipt_execution_outcome.receipt.receipt_id.to_string(), + transaction_hash, + event: event.event.clone(), + standard: event.standard.clone(), + version: event.version.clone(), + data: event.data.as_ref().map(|data| data.to_string()), + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::rules::outcomes_reducer::reduce_indexer_rule_matches_from_outcomes; + use crate::rules::types::indexer_rule_match::{ChainId, IndexerRuleMatch}; + use crate::rules::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + use near_lake_framework::near_indexer_primitives::StreamerMessage; + + fn read_local_file(path: &str) -> String { + std::fs::read_to_string(path).unwrap() + } + fn read_local_streamer_message(block_height: u64) -> StreamerMessage { + let path = format!( + "{}/blocks/{}.json", + env!("CARGO_MANIFEST_DIR"), + block_height + ); + serde_json::from_str(&read_local_file(&path)).unwrap() + } + + #[tokio::test] + async fn match_wildcard_no_match() { + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "*.nearcrow.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let streamer_message = read_local_streamer_message(93085141); + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 0); + } + + #[tokio::test] + async fn match_wildcard_contract_subaccount_name() { + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "*.nearcrowd.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let streamer_message = read_local_streamer_message(93085141); + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching) + } + + #[tokio::test] + async fn match_wildcard_mid_contract_name() { + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "*crowd.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let streamer_message = read_local_streamer_message(93085141); + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 1); // see Extraction note in previous test + + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "app.nea*owd.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 1); // see Extraction note in previous test + } + + #[tokio::test] + async fn match_csv_account() { + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "notintheblockaccount.near, app.nearcrowd.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let streamer_message = read_local_streamer_message(93085141); + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching) + } + + #[tokio::test] + async fn match_csv_wildcard_account() { + let wildcard_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "notintheblockaccount.near, *.nearcrowd.near".to_string(), + status: Status::Success, + }, + id: None, + name: None, + }; + + let streamer_message = read_local_streamer_message(93085141); + let result: Vec = reduce_indexer_rule_matches_from_outcomes( + &wildcard_rule, + &streamer_message, + ChainId::Testnet, + ); + + assert_eq!(result.len(), 1); // There are two matches, until we add Extraction we are just matching the first one (block matching) + } +} diff --git a/block-streamer/src/rules/types/events.rs b/block-streamer/src/rules/types/events.rs new file mode 100644 index 000000000..68bb176d0 --- /dev/null +++ b/block-streamer/src/rules/types/events.rs @@ -0,0 +1,20 @@ +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct Event { + pub event: String, + pub standard: String, + pub version: String, + pub data: Option, +} + +impl Event { + pub fn from_log(log: &str) -> anyhow::Result { + let prefix = "EVENT_JSON:"; + if !log.starts_with(prefix) { + anyhow::bail!("log message doesn't start from required prefix"); + } + + Ok(serde_json::from_str::<'_, Self>( + log[prefix.len()..].trim(), + )?) + } +} diff --git a/block-streamer/src/rules/types/indexer_rule_match.rs b/block-streamer/src/rules/types/indexer_rule_match.rs new file mode 100644 index 000000000..985f925a2 --- /dev/null +++ b/block-streamer/src/rules/types/indexer_rule_match.rs @@ -0,0 +1,146 @@ +use std::fmt; + +pub type TransactionHashString = String; +pub type ReceiptIdString = String; +pub type BlockHashString = String; + +#[derive( + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, + Clone, + Debug, +)] +pub struct IndexerRuleMatch { + pub chain_id: ChainId, + pub indexer_rule_id: Option, + pub indexer_rule_name: Option, + pub payload: IndexerRuleMatchPayload, + pub block_height: u64, +} + +impl IndexerRuleMatch { + pub fn explorer_link(&self) -> String { + match self.chain_id { + ChainId::Testnet => { + if let Some(tx_hash) = self.payload.transaction_hash() { + if let Some(receipt_id) = self.payload.receipt_id() { + format!( + "https://explorer.testnet.near.org/transactions/{}#{}", + tx_hash, receipt_id, + ) + } else { + format!("https://explorer.testnet.near.org/transactions/{}", tx_hash) + } + } else { + format!( + "https://explorer.testnet.near.org/block/{}", + self.payload.block_hash() + ) + } + } + ChainId::Mainnet => { + if let Some(tx_hash) = self.payload.transaction_hash() { + if let Some(receipt_id) = self.payload.receipt_id() { + format!( + "https://explorer.near.org/transactions/{}#{}", + tx_hash, receipt_id, + ) + } else { + format!("https://explorer.near.org/transactions/{}", tx_hash) + } + } else { + format!( + "https://explorer.near.org/block/{}", + self.payload.block_hash() + ) + } + } + } + } +} + +#[derive( + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, + Clone, + Debug, +)] +pub enum IndexerRuleMatchPayload { + Actions { + block_hash: BlockHashString, + receipt_id: ReceiptIdString, + transaction_hash: Option, + }, + Events { + block_hash: BlockHashString, + receipt_id: ReceiptIdString, + transaction_hash: Option, + event: String, + standard: String, + version: String, + data: Option, + }, + StateChanges { + block_hash: BlockHashString, + receipt_id: Option, + transaction_hash: Option, + }, +} + +impl IndexerRuleMatchPayload { + pub fn block_hash(&self) -> BlockHashString { + match self { + Self::Actions { block_hash, .. } + | Self::Events { block_hash, .. } + | Self::StateChanges { block_hash, .. } => block_hash.to_string(), + } + } + + pub fn receipt_id(&self) -> Option { + match self { + Self::Actions { receipt_id, .. } | Self::Events { receipt_id, .. } => { + Some(receipt_id.to_string()) + } + Self::StateChanges { receipt_id, .. } => receipt_id.clone(), + } + } + + pub fn transaction_hash(&self) -> Option { + match self { + Self::Actions { + transaction_hash, .. + } + | Self::Events { + transaction_hash, .. + } + | Self::StateChanges { + transaction_hash, .. + } => transaction_hash.clone(), + } + } +} + +#[derive( + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, + Clone, + Debug, +)] +pub enum ChainId { + Mainnet, + Testnet, +} +impl fmt::Display for ChainId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ChainId::Mainnet => write!(f, "mainnet"), + ChainId::Testnet => write!(f, "testnet"), + } + } +} diff --git a/block-streamer/src/rules/types/mod.rs b/block-streamer/src/rules/types/mod.rs new file mode 100644 index 000000000..d82357aff --- /dev/null +++ b/block-streamer/src/rules/types/mod.rs @@ -0,0 +1,3 @@ +pub mod events; +pub mod indexer_rule_match; +pub mod transactions; diff --git a/block-streamer/src/rules/types/transactions.rs b/block-streamer/src/rules/types/transactions.rs new file mode 100644 index 000000000..7f9ddd30b --- /dev/null +++ b/block-streamer/src/rules/types/transactions.rs @@ -0,0 +1,20 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use near_lake_framework::near_indexer_primitives::{views, IndexerTransactionWithOutcome}; +use serde::{Deserialize, Serialize}; + +#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, Debug, Clone)] +pub struct TransactionDetails { + pub transaction: views::SignedTransactionView, + pub receipts: Vec, + pub execution_outcomes: Vec, +} + +impl TransactionDetails { + pub fn from_indexer_tx(transaction: IndexerTransactionWithOutcome) -> Self { + Self { + transaction: transaction.transaction.clone(), + receipts: vec![], + execution_outcomes: vec![transaction.outcome.execution_outcome], + } + } +} diff --git a/block-streamer/src/s3.rs b/block-streamer/src/s3.rs new file mode 100644 index 000000000..eaf3bc003 --- /dev/null +++ b/block-streamer/src/s3.rs @@ -0,0 +1,342 @@ +use anyhow::{bail, Context, Result}; +use aws_sdk_s3::Client as S3Client; +use chrono::{DateTime, NaiveDate, Utc}; +use futures::future::try_join_all; + +// Sanity check, if we hit this we have 1M S3 results. +// Currently that would be either 2,700 years of FunctionCall data or 1M contract folders. +// If we hit 1M contracts we should build an index to support efficient wildcard contract matching. +const MAX_S3_LIST_REQUESTS: usize = 1000; +pub const INDEXED_DATA_FILES_BUCKET: &str = "near-delta-lake"; +pub const INDEXED_ACTIONS_FILES_FOLDER: &str = "silver/accounts/action_receipt_actions/metadata"; + +fn storage_path_for_account(account: &str) -> String { + let mut folders = account.split('.').collect::>(); + folders.reverse(); + folders.join("/") +} + +pub async fn find_index_files_by_pattern( + s3_client: &S3Client, + s3_bucket: &str, + s3_folder: &str, + pattern: &str, +) -> Result> { + Ok(match pattern { + x if x.contains(',') => { + let account_array = x.split(','); + let mut results = vec![]; + for account in account_array { + let account = account.trim(); + let sub_results = if account.contains('*') { + list_index_files_by_wildcard(s3_client, s3_bucket, s3_folder, &account).await? + } else { + list_s3_bucket_by_prefix( + s3_client, + s3_bucket, + &format!("{}/{}/", s3_folder, storage_path_for_account(account)), + ) + .await? + }; + results.extend(sub_results); + } + results + } + x if x.contains('*') => { + list_index_files_by_wildcard(s3_client, s3_bucket, s3_folder, &x).await? + } + _ => { + list_s3_bucket_by_prefix( + s3_client, + s3_bucket, + &format!("{}/{}/", s3_folder, storage_path_for_account(pattern),), + ) + .await? + } + }) +} + +async fn list_index_files_by_wildcard( + s3_client: &S3Client, + s3_bucket: &str, + s3_folder: &str, + pattern: &&str, +) -> Result> { + // remove sub-account wildcard from pattern + let pattern = pattern.replace("*.", ""); + let path = storage_path_for_account(&pattern); + + let folders = + list_s3_bucket_by_prefix(s3_client, s3_bucket, &format!("{}/{}/", s3_folder, path)).await?; + // for each matching folder list files + let mut results = vec![]; + for folder in folders { + results.extend(list_s3_bucket_by_prefix(s3_client, s3_bucket, &folder).await?); + } + Ok(results) +} + +async fn list_s3_bucket_by_prefix( + s3_client: &S3Client, + s3_bucket: &str, + s3_prefix: &str, +) -> Result> { + let mut results = vec![]; + let mut continuation_token: Option = None; + + let mut counter = 0; + loop { + let mut configured_client = s3_client + .list_objects_v2() + .bucket(s3_bucket) + .prefix(s3_prefix) + .delimiter("/"); + + if continuation_token.is_some() { + configured_client = configured_client.continuation_token(continuation_token.unwrap()); + } + + let file_list = configured_client.send().await?; + if let Some(common_prefixes) = file_list.common_prefixes { + let keys: Vec = common_prefixes + .into_iter() + .map(|o| o.prefix.unwrap()) + .collect(); + results.extend(keys); + } + if let Some(objects) = file_list.contents { + let keys: Vec = objects.into_iter().map(|o| o.key.unwrap()).collect(); + results.extend(keys); + } + if file_list.next_continuation_token.is_some() { + continuation_token = file_list.next_continuation_token; + counter += 1; + if counter > MAX_S3_LIST_REQUESTS { + bail!("Exceeded internal limit of {MAX_S3_LIST_REQUESTS}") + } + } else { + break; + } + } + Ok(results) +} + +pub async fn fetch_contract_index_files( + s3_client: &S3Client, + s3_bucket: &str, + s3_folder: &str, + start_date: DateTime, + contract_pattern: &str, +) -> Result> { + // list all index files + let file_list = + find_index_files_by_pattern(s3_client, s3_bucket, s3_folder, contract_pattern).await?; + + let fetch_and_parse_tasks = file_list + .into_iter() + .filter(|index_file_listing| file_name_date_after(start_date, index_file_listing)) + .map(|key| { + let s3_client = s3_client.clone(); + async move { + // Fetch the file + fetch_text_file_from_s3(s3_bucket, key, &s3_client).await + } + }) + .collect::>(); + + // Execute all tasks in parallel and wait for completion + let file_contents: Vec = try_join_all(fetch_and_parse_tasks).await?; + Ok(file_contents + .into_iter() + .filter(|file_contents| !file_contents.is_empty()) + .collect::>()) +} + +pub async fn fetch_text_file_from_s3( + s3_bucket: &str, + key: String, + s3_client: &S3Client, +) -> Result { + // todo: can we retry if this fails like the lake s3_fetcher fn does? + // If so, can we differentiate between a file not existing (block height does not exist) and a network error? + let get_object_output = s3_client + .get_object() + .bucket(s3_bucket) + .key(key.clone()) + .send() + .await + .with_context(|| format!("Error fetching index file {key}"))?; + + let bytes = get_object_output + .body + .collect() + .await + .with_context(|| format!("Error reading bytes of index file {key}"))?; + String::from_utf8(bytes.to_vec()).with_context(|| format!("Error parsing index file {key}")) +} + +/// check whether the filename is a date after the start date +/// filename is in format 2022-10-03.json +fn file_name_date_after(start_date: DateTime, file_name: &str) -> bool { + let file_name_date = file_name.split('/').last().unwrap().replace(".json", ""); + let file_name_date = NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d"); + match file_name_date { + Ok(file_name_date) => file_name_date >= start_date.date_naive(), + Err(e) => { + // if we can't parse the date assume a file this code is not meant to handle + tracing::debug!( + target: crate::LOG_TARGET, + "Error parsing file name date: {:?}", + e + ); + false + } + } +} + +#[cfg(test)] +mod tests { + use crate::s3::{ + fetch_text_file_from_s3, find_index_files_by_pattern, list_s3_bucket_by_prefix, + INDEXED_ACTIONS_FILES_FOLDER, INDEXED_DATA_FILES_BUCKET, + }; + + /// Parses env vars from .env, Run with + /// cargo test s3::tests::list_delta_bucket -- mainnet from-latest; + #[tokio::test] + #[ignore] + async fn list_delta_bucket() { + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let list = list_s3_bucket_by_prefix( + &s3_client, + INDEXED_DATA_FILES_BUCKET, + &format!("{}/", INDEXED_ACTIONS_FILES_FOLDER), + ) + .await + .unwrap(); + assert_eq!(list.len(), 4); + } + + /// cargo test s3::tests::list_with_single_contract -- mainnet from-latest + #[tokio::test] + #[ignore] + async fn list_with_single_contract() { + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let list = find_index_files_by_pattern( + &s3_client, + INDEXED_DATA_FILES_BUCKET, + INDEXED_ACTIONS_FILES_FOLDER, + "hackathon.agency.near", + ) + .await + .unwrap(); + assert_eq!(list.len(), 1); + } + + /// cargo test s3::tests::list_with_csv_contracts -- mainnet from-latest + #[tokio::test] + #[ignore] + async fn list_with_csv_contracts() { + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let list = find_index_files_by_pattern( + &s3_client, + INDEXED_DATA_FILES_BUCKET, + INDEXED_ACTIONS_FILES_FOLDER, + "hackathon.agency.near, hackathon.aurora-silo-dev.near, hackathon.sputnik-dao.near", + ) + .await + .unwrap(); + assert!(list.len() >= 15); // expecting 15 but these contracts could get randomly called sometime + } + + /// cargo test s3::tests::list_with_wildcard_contracts -- mainnet from-latest + #[tokio::test] + #[ignore] + async fn list_with_wildcard_contracts() { + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let list = find_index_files_by_pattern( + &s3_client, + INDEXED_DATA_FILES_BUCKET, + INDEXED_ACTIONS_FILES_FOLDER, + "*.keypom.near", + ) + .await + .unwrap(); + assert!(list.len() >= 550); + } + + /// cargo test s3::tests::list_with_csv_and_wildcard_contracts -- mainnet from-latest + #[tokio::test] + #[ignore] + async fn list_with_csv_and_wildcard_contracts() { + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let list = find_index_files_by_pattern( + &s3_client, + INDEXED_DATA_FILES_BUCKET, + INDEXED_ACTIONS_FILES_FOLDER, + "*.keypom.near, hackathon.agency.near, *.nearcrowd.near", + ) + .await + .unwrap(); + assert!(list.len() > 1370); + } + + #[test] + fn storage_path_for_account_splits_and_reverses_into_folders() { + let account = "buildnear.testnet"; + let expected = "testnet/buildnear"; + let actual = super::storage_path_for_account(account); + assert_eq!(expected, actual); + + let account = "v2.keypom.near"; + let expected = "near/keypom/v2"; + let actual = super::storage_path_for_account(account); + assert_eq!(expected, actual); + + let account = "0.app5.hipodev.near"; + let expected = "near/hipodev/app5/0"; + let actual = super::storage_path_for_account(account); + assert_eq!(expected, actual); + } + + #[tokio::test] + #[ignore] + async fn handle_key_404() { + let mut success = false; + + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let s3_result = fetch_text_file_from_s3( + "near-lake-data-mainnet", + "does_not_exist/block.json".to_string(), + &s3_client, + ) + .await; + + if s3_result.is_err() { + let wrapped_error = s3_result.err().unwrap(); + let error = wrapped_error.root_cause(); + if error + .downcast_ref::() + .is_some() + { + success = true; + } else { + println!("Failed to downcast error: {:?}", error); + } + } + + assert!(success); + } +} From 694f04d43579834653fcd2d7be5602e3a84ef80b Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 09:20:56 +1300 Subject: [PATCH 05/40] refactor: Rename historical processing to `BlockStreamer` --- .../src/{historical_block_processing.rs => block_streamer.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename block-streamer/src/{historical_block_processing.rs => block_streamer.rs} (100%) diff --git a/block-streamer/src/historical_block_processing.rs b/block-streamer/src/block_streamer.rs similarity index 100% rename from block-streamer/src/historical_block_processing.rs rename to block-streamer/src/block_streamer.rs From 503a069201fc7b2c4d27949613a2c6894e05093f Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 09:28:53 +1300 Subject: [PATCH 06/40] refactor: Rename `IndexerFunction` > `IndexerConfig` --- block-streamer/src/block_streamer.rs | 60 ++++++++++++++-------------- block-streamer/src/indexer_types.rs | 29 -------------- block-streamer/src/main.rs | 2 +- 3 files changed, 30 insertions(+), 61 deletions(-) delete mode 100644 block-streamer/src/indexer_types.rs diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 44ee60c47..cc95a98c7 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -1,4 +1,4 @@ -use crate::indexer_types::IndexerFunction; +use crate::indexer_config::IndexerConfig; use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::MatchingRule; use crate::s3; @@ -31,7 +31,7 @@ impl BlockStreamer { pub fn start( &mut self, current_block_height: BlockHeight, - indexer: IndexerFunction, + indexer: IndexerConfig, redis_connection_manager: crate::redis::ConnectionManager, s3_client: S3Client, chain_id: ChainId, @@ -94,7 +94,7 @@ impl BlockStreamer { pub(crate) async fn process_historical_messages_or_handle_error( current_block_height: BlockHeight, - indexer_function: IndexerFunction, + indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, s3_client: &S3Client, chain_id: &ChainId, @@ -102,7 +102,7 @@ pub(crate) async fn process_historical_messages_or_handle_error( ) -> i64 { match process_historical_messages( current_block_height, - indexer_function, + indexer, redis_connection_manager, s3_client, chain_id, @@ -124,49 +124,49 @@ pub(crate) async fn process_historical_messages_or_handle_error( } pub(crate) async fn process_historical_messages( current_block_height: BlockHeight, - indexer_function: IndexerFunction, + indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, s3_client: &S3Client, chain_id: &ChainId, json_rpc_client: &JsonRpcClient, ) -> anyhow::Result { - let start_block = indexer_function.start_block_height.unwrap(); + let start_block = indexer.start_block_height.unwrap(); let block_difference: i64 = (current_block_height - start_block) as i64; match block_difference { i64::MIN..=-1 => { bail!( "Skipping back fill, start_block_height is greater than current block height: {}", - indexer_function.get_full_name(), + indexer.get_full_name(), ); } 0 => { bail!( "Skipping back fill, start_block_height is equal to current block height: {}", - indexer_function.get_full_name(), + indexer.get_full_name(), ); } 1..=i64::MAX => { tracing::info!( target: crate::LOG_TARGET, "Back filling {block_difference} blocks from {start_block} to current block height {current_block_height}: {}", - indexer_function.get_full_name(), + indexer.get_full_name(), ); crate::redis::del( redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), ) .await?; crate::redis::sadd( redis_connection_manager, crate::redis::STREAMS_SET_KEY, - crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), ) .await?; crate::redis::set( redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), - serde_json::to_string(&indexer_function)?, + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), + serde_json::to_string(&indexer)?, None, ) .await?; @@ -178,7 +178,7 @@ pub(crate) async fn process_historical_messages( let blocks_from_index = filter_matching_blocks_from_index_files( start_block, - &indexer_function, + &indexer, s3_client, start_date, ) @@ -188,13 +188,13 @@ pub(crate) async fn process_historical_messages( target: crate::LOG_TARGET, "Flushing {} block heights from index files to historical Stream for indexer: {}", blocks_from_index.len(), - indexer_function.get_full_name(), + indexer.get_full_name(), ); for block in &blocks_from_index { crate::redis::xadd( redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer_function.get_full_name()), + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), &[("block_height", block)], ) .await?; @@ -214,14 +214,14 @@ pub(crate) async fn process_historical_messages( target: crate::LOG_TARGET, "Filtering {} unindexed blocks from lake: from block {last_indexed_block} to {current_block_height} for indexer: {}", unindexed_block_difference, - indexer_function.get_full_name(), + indexer.get_full_name(), ); if unindexed_block_difference > UNINDEXED_BLOCKS_SOFT_LIMIT { tracing::warn!( target: crate::LOG_TARGET, "Unindexed block difference exceeds soft limit of: {UNINDEXED_BLOCKS_SOFT_LIMIT} for indexer: {}", - indexer_function.get_full_name(), + indexer.get_full_name(), ); } @@ -243,7 +243,7 @@ pub(crate) async fn process_historical_messages( } let matches = crate::rules::reduce_indexer_rule_matches( - &indexer_function.indexer_rule, + &indexer.indexer_rule, &streamer_message, chain_id.clone(), ); @@ -253,9 +253,7 @@ pub(crate) async fn process_historical_messages( crate::redis::xadd( redis_connection_manager, - crate::redis::generate_historical_stream_key( - &indexer_function.get_full_name(), - ), + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), &[("block_height", block_height)], ) .await?; @@ -267,7 +265,7 @@ pub(crate) async fn process_historical_messages( target: crate::LOG_TARGET, "Flushed {} unindexed block heights to historical Stream for indexer: {}", filtered_block_count, - indexer_function.get_full_name(), + indexer.get_full_name(), ); } } @@ -302,14 +300,14 @@ pub(crate) async fn last_indexed_block_from_metadata( pub(crate) async fn filter_matching_blocks_from_index_files( start_block_height: BlockHeight, - indexer_function: &IndexerFunction, + indexer: &IndexerConfig, s3_client: &S3Client, start_date: DateTime, ) -> anyhow::Result> { let s3_bucket = s3::INDEXED_DATA_FILES_BUCKET; let mut needs_dedupe_and_sort = false; - let indexer_rule = &indexer_function.indexer_rule; + let indexer_rule = &indexer.indexer_rule; let index_files_content = match &indexer_rule.matching_rule { MatchingRule::ActionAny { @@ -329,18 +327,18 @@ pub(crate) async fn filter_matching_blocks_from_index_files( .await } MatchingRule::ActionFunctionCall { .. } => { - bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer_function.account_id, indexer_function.function_name); + bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer.account_id, indexer.function_name); } MatchingRule::Event { .. } => { - bail!("Event matching rule not yet supported for historical processing, function {:?} {:?}", indexer_function.account_id, indexer_function.function_name); + bail!("Event matching rule not yet supported for historical processing, function {:?} {:?}", indexer.account_id, indexer.function_name); } }?; tracing::info!( target: crate::LOG_TARGET, "Found {file_count} index files for function {:?} {:?} with matching rule {indexer_rule:?}", - indexer_function.account_id, - indexer_function.function_name, + indexer.account_id, + indexer.function_name, file_count = index_files_content.len() ); let mut blocks_to_process: Vec = @@ -352,8 +350,8 @@ pub(crate) async fn filter_matching_blocks_from_index_files( tracing::info!( target: crate::LOG_TARGET, "Found {block_count} indexed blocks to process for function {:?} {:?}", - indexer_function.account_id, - indexer_function.function_name, + indexer.account_id, + indexer.function_name, block_count = blocks_to_process.len() ); diff --git a/block-streamer/src/indexer_types.rs b/block-streamer/src/indexer_types.rs deleted file mode 100644 index f3a155c16..000000000 --- a/block-streamer/src/indexer_types.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::rules::IndexerRule; -use near_lake_framework::near_indexer_primitives::types::AccountId; -use std::collections::HashMap; - -pub type IndexerRegistry = HashMap>; - -#[derive( - borsh::BorshSerialize, - borsh::BorshDeserialize, - serde::Serialize, - serde::Deserialize, - Clone, - Debug, -)] -pub struct IndexerFunction { - pub account_id: AccountId, - pub function_name: String, - pub code: String, - pub start_block_height: Option, - pub schema: Option, - pub provisioned: bool, - pub indexer_rule: IndexerRule, -} - -impl IndexerFunction { - pub fn get_full_name(&self) -> String { - format!("{}/{}", self.account_id, self.function_name) - } -} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 4beb71f54..65c2af8dc 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -1,5 +1,5 @@ mod block_streamer; -mod indexer_types; +mod indexer_config; mod redis; mod rules; mod s3; From af9faf72fcbbe8c8f2a3356e63de8efb7f3f2c2a Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:21:06 +1300 Subject: [PATCH 07/40] feat: Use default log target of module path --- block-streamer/Cargo.lock | 1 + block-streamer/Cargo.toml | 3 ++- block-streamer/src/block_streamer.rs | 23 ++--------------------- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index bf57e90c6..71f597855 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -607,6 +607,7 @@ dependencies = [ "tokio", "tokio-util 0.6.10", "tracing", + "tracing-subscriber", "wildmatch", ] diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index a0b3800e9..1fb4407a7 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -15,7 +15,8 @@ futures = "0.3.5" redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" -tracing = "0.1.34" +tracing = "0.1.40" +tracing-subscriber = "0.3.18" tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-util = "0.6.7" wildmatch = "2.1.1" diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index cc95a98c7..f2621d54e 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -48,7 +48,6 @@ impl BlockStreamer { tokio::select! { _ = cancellation_token_clone.cancelled() => { tracing::info!( - target: crate::LOG_TARGET, "Cancelling existing historical backfill for indexer: {}", indexer.get_full_name(), ); @@ -62,7 +61,6 @@ impl BlockStreamer { &json_rpc_client, ) => { tracing::info!( - target: crate::LOG_TARGET, "Finished historical backfill for indexer: {}", indexer.get_full_name(), ); @@ -113,11 +111,7 @@ pub(crate) async fn process_historical_messages_or_handle_error( Ok(block_difference) => block_difference, Err(err) => { // todo: when Coordinator can send log messages to Runner, send this error to Runner - tracing::error!( - target: crate::LOG_TARGET, - "Error processing historical messages: {:?}", - err - ); + tracing::error!("Error processing historical messages: {:?}", err); 0 } } @@ -147,7 +141,6 @@ pub(crate) async fn process_historical_messages( } 1..=i64::MAX => { tracing::info!( - target: crate::LOG_TARGET, "Back filling {block_difference} blocks from {start_block} to current block height {current_block_height}: {}", indexer.get_full_name(), ); @@ -185,7 +178,6 @@ pub(crate) async fn process_historical_messages( .await?; tracing::info!( - target: crate::LOG_TARGET, "Flushing {} block heights from index files to historical Stream for indexer: {}", blocks_from_index.len(), indexer.get_full_name(), @@ -211,7 +203,6 @@ pub(crate) async fn process_historical_messages( let unindexed_block_difference = current_block_height - last_indexed_block; tracing::info!( - target: crate::LOG_TARGET, "Filtering {} unindexed blocks from lake: from block {last_indexed_block} to {current_block_height} for indexer: {}", unindexed_block_difference, indexer.get_full_name(), @@ -219,7 +210,6 @@ pub(crate) async fn process_historical_messages( if unindexed_block_difference > UNINDEXED_BLOCKS_SOFT_LIMIT { tracing::warn!( - target: crate::LOG_TARGET, "Unindexed block difference exceeds soft limit of: {UNINDEXED_BLOCKS_SOFT_LIMIT} for indexer: {}", indexer.get_full_name(), ); @@ -262,7 +252,6 @@ pub(crate) async fn process_historical_messages( drop(sender); tracing::info!( - target: crate::LOG_TARGET, "Flushed {} unindexed block heights to historical Stream for indexer: {}", filtered_block_count, indexer.get_full_name(), @@ -291,7 +280,6 @@ pub(crate) async fn last_indexed_block_from_metadata( let last_indexed_block = from_str(last_indexed_block).context("last_indexed_block couldn't be converted to u64")?; tracing::info!( - target: crate::LOG_TARGET, "Last indexed block from latest_block.json: {:?}", last_indexed_block ); @@ -335,7 +323,6 @@ pub(crate) async fn filter_matching_blocks_from_index_files( }?; tracing::info!( - target: crate::LOG_TARGET, "Found {file_count} index files for function {:?} {:?} with matching rule {indexer_rule:?}", indexer.account_id, indexer.function_name, @@ -348,7 +335,6 @@ pub(crate) async fn filter_matching_blocks_from_index_files( blocks_to_process.dedup(); } tracing::info!( - target: crate::LOG_TARGET, "Found {block_count} indexed blocks to process for function {:?} {:?}", indexer.account_id, indexer.function_name, @@ -376,18 +362,13 @@ fn parse_blocks_from_index_files( .collect() } else { tracing::error!( - target: crate::LOG_TARGET, "Unable to parse index file, no heights found: {:?}", file_content ); vec![] } } else { - tracing::error!( - target: crate::LOG_TARGET, - "Unable to parse index file: {:?}", - file_content - ); + tracing::error!("Unable to parse index file: {:?}", file_content); vec![] } }) From 5682f61f70d12d716f3924ed24ac7a9ef0d5fd1b Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:31:52 +1300 Subject: [PATCH 08/40] fix: Store stream storage under correct redis key --- block-streamer/src/block_streamer.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index f2621d54e..134ac484c 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -158,7 +158,7 @@ pub(crate) async fn process_historical_messages( .await?; crate::redis::set( redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer.get_full_name()), + crate::redis::generate_historical_storage_key(&indexer.get_full_name()), serde_json::to_string(&indexer)?, None, ) @@ -189,7 +189,8 @@ pub(crate) async fn process_historical_messages( crate::redis::generate_historical_stream_key(&indexer.get_full_name()), &[("block_height", block)], ) - .await?; + .await + .context("Failed to add block to Redis Stream")?; } // Check for the case where an index file is written right after we get the last_indexed_block metadata From fde65a1f0c7838a357a09aa58ffdc076758f5f18 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:32:13 +1300 Subject: [PATCH 09/40] refactor: Remove unnecessary logging target from redis mod --- block-streamer/src/redis.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/block-streamer/src/redis.rs b/block-streamer/src/redis.rs index e426929ec..7fc8ccbe5 100644 --- a/block-streamer/src/redis.rs +++ b/block-streamer/src/redis.rs @@ -1,7 +1,5 @@ pub use redis::{self, aio::ConnectionManager, FromRedisValue, ToRedisArgs}; -const STORAGE: &str = "storage_alertexer"; - pub const LAKE_BUCKET_PREFIX: &str = "near-lake-data-"; pub const STREAMS_SET_KEY: &str = "streams"; @@ -44,7 +42,7 @@ pub async fn del( .arg(&key) .query_async(&mut redis_connection_manager.clone()) .await?; - tracing::debug!(target: STORAGE, "DEL: {:?}", key); + tracing::debug!("DEL: {:?}", key); Ok(()) } @@ -64,13 +62,7 @@ pub async fn set( cmd.query_async(&mut redis_connection_manager.clone()) .await?; - tracing::debug!( - target: STORAGE, - "SET: {:?}: {:?} Ex: {:?}", - key, - value, - expiration_seconds - ); + tracing::debug!("SET: {:?}: {:?} Ex: {:?}", key, value, expiration_seconds); Ok(()) } @@ -82,7 +74,7 @@ pub async fn get( .arg(&key) .query_async(&mut redis_connection_manager.clone()) .await?; - tracing::debug!(target: STORAGE, "GET: {:?}: {:?}", &key, &value,); + tracing::debug!("GET: {:?}: {:?}", &key, &value,); Ok(value) } @@ -91,7 +83,7 @@ pub async fn sadd( key: impl ToRedisArgs + std::fmt::Debug, value: impl ToRedisArgs + std::fmt::Debug, ) -> anyhow::Result<()> { - tracing::debug!(target: STORAGE, "SADD: {:?}: {:?}", key, value); + tracing::debug!("SADD: {:?}: {:?}", key, value); redis::cmd("SADD") .arg(key) @@ -107,7 +99,7 @@ pub async fn xadd( stream_key: impl ToRedisArgs + std::fmt::Debug, fields: &[(&str, impl ToRedisArgs + std::fmt::Debug)], ) -> anyhow::Result<()> { - tracing::debug!(target: STORAGE, "XADD: {:?}, {:?}", stream_key, fields); + tracing::debug!("XADD: {:?}, {:?}", stream_key, fields); let mut cmd = redis::cmd("XADD"); cmd.arg(stream_key).arg("*"); From 4c1b947109cd18af5e4ba9683109490c85178bd7 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:32:40 +1300 Subject: [PATCH 10/40] feat: Start dummy `BlockStreamer` to test --- block-streamer/src/block_streamer.rs | 4 ++ block-streamer/src/indexer_config.rs | 26 ++++++++++++ block-streamer/src/main.rs | 61 +++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 block-streamer/src/indexer_config.rs diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 134ac484c..e7fa348bf 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -88,6 +88,10 @@ impl BlockStreamer { "Attempted to cancel already cancelled, or not started, BlockStreamer" )) } + + pub async fn wait(&mut self) { + let _ = self.task.take().unwrap().handle.await; + } } pub(crate) async fn process_historical_messages_or_handle_error( diff --git a/block-streamer/src/indexer_config.rs b/block-streamer/src/indexer_config.rs new file mode 100644 index 000000000..06328d69d --- /dev/null +++ b/block-streamer/src/indexer_config.rs @@ -0,0 +1,26 @@ +use crate::rules::IndexerRule; +use near_lake_framework::near_indexer_primitives::types::AccountId; + +#[derive( + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, + Clone, + Debug, +)] +pub struct IndexerConfig { + pub account_id: AccountId, + pub function_name: String, + pub code: String, + pub start_block_height: Option, + pub schema: Option, + pub provisioned: bool, + pub indexer_rule: IndexerRule, +} + +impl IndexerConfig { + pub fn get_full_name(&self) -> String { + format!("{}/{}", self.account_id, self.function_name) + } +} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 65c2af8dc..577ddab3f 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -1,3 +1,10 @@ +use near_jsonrpc_client::JsonRpcClient; +use tracing_subscriber::prelude::*; + +use crate::indexer_config::IndexerConfig; +use crate::rules::types::indexer_rule_match::ChainId; +use crate::rules::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + mod block_streamer; mod indexer_config; mod redis; @@ -6,6 +13,56 @@ mod s3; pub(crate) const LOG_TARGET: &str = "block_streamer"; -fn main() { - println!("Hello, world!"); +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer()) + .with(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + tracing::info!("Starting {}", crate::LOG_TARGET); + + let json_rpc_client = JsonRpcClient::connect("https://archival-rpc.mainnet.near.org"); + let redis_connection_manager = redis::connect("redis://127.0.0.1").await?; + + let aws_config = aws_config::from_env().load().await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + let contract = "queryapi.dataplatform.near"; + let matching_rule = MatchingRule::ActionAny { + affected_account_id: contract.to_string(), + status: Status::Any, + }; + let filter_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule, + id: None, + name: None, + }; + let indexer = IndexerConfig { + account_id: "buildnear.testnet".to_string().parse().unwrap(), + function_name: "index_stuff".to_string(), + code: "".to_string(), + start_block_height: Some(85376002), + schema: None, + provisioned: false, + indexer_rule: filter_rule, + }; + + let mut streamer = block_streamer::BlockStreamer::new(); + + streamer.start( + 106230334 + 1000, + indexer, + redis_connection_manager, + s3_client, + ChainId::Mainnet, + json_rpc_client, + )?; + + streamer.wait().await; + + println!("done"); + + Ok(()) } From 781f3f7898d35c2f3c5904697866dc45817b3a1b Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:37:08 +1300 Subject: [PATCH 11/40] refactor: Remove unneeded redis storage --- block-streamer/src/block_streamer.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index e7fa348bf..55055d9f3 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -149,25 +149,6 @@ pub(crate) async fn process_historical_messages( indexer.get_full_name(), ); - crate::redis::del( - redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer.get_full_name()), - ) - .await?; - crate::redis::sadd( - redis_connection_manager, - crate::redis::STREAMS_SET_KEY, - crate::redis::generate_historical_stream_key(&indexer.get_full_name()), - ) - .await?; - crate::redis::set( - redis_connection_manager, - crate::redis::generate_historical_storage_key(&indexer.get_full_name()), - serde_json::to_string(&indexer)?, - None, - ) - .await?; - let start_date = lookup_block_date_or_next_block_date(start_block, json_rpc_client).await?; From e63824f0e062e96893cd976ee3bcfb9cfd4814d2 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 10:43:05 +1300 Subject: [PATCH 12/40] refactor: Return `handle` rather than awaiting implicitly --- block-streamer/src/block_streamer.rs | 4 ++-- block-streamer/src/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 55055d9f3..fe0e93d75 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -89,8 +89,8 @@ impl BlockStreamer { )) } - pub async fn wait(&mut self) { - let _ = self.task.take().unwrap().handle.await; + pub fn take_handle(&mut self) -> Option> { + self.task.take().map(|task| task.handle) } } diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 577ddab3f..b539977e7 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -60,7 +60,7 @@ async fn main() -> anyhow::Result<()> { json_rpc_client, )?; - streamer.wait().await; + streamer.take_handle().unwrap().await?; println!("done"); From 76641bda8cbc10ca9885a66c8c147aa143ccfc2a Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 12:16:39 +1300 Subject: [PATCH 13/40] feat: Start `BlockStreamer` from arbitrary block height --- block-streamer/src/block_streamer.rs | 222 ++++++++++----------------- block-streamer/src/main.rs | 2 +- 2 files changed, 85 insertions(+), 139 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index fe0e93d75..e0398dbb0 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -30,7 +30,7 @@ impl BlockStreamer { pub fn start( &mut self, - current_block_height: BlockHeight, + start_block_height: BlockHeight, indexer: IndexerConfig, redis_connection_manager: crate::redis::ConnectionManager, s3_client: S3Client, @@ -52,8 +52,8 @@ impl BlockStreamer { indexer.get_full_name(), ); }, - _ = process_historical_messages_or_handle_error( - current_block_height, + _ = process_historical_messages( + start_block_height, indexer.clone(), &redis_connection_manager, &s3_client, @@ -94,157 +94,103 @@ impl BlockStreamer { } } -pub(crate) async fn process_historical_messages_or_handle_error( - current_block_height: BlockHeight, - indexer: IndexerConfig, - redis_connection_manager: &crate::redis::ConnectionManager, - s3_client: &S3Client, - chain_id: &ChainId, - json_rpc_client: &JsonRpcClient, -) -> i64 { - match process_historical_messages( - current_block_height, - indexer, - redis_connection_manager, - s3_client, - chain_id, - json_rpc_client, - ) - .await - { - Ok(block_difference) => block_difference, - Err(err) => { - // todo: when Coordinator can send log messages to Runner, send this error to Runner - tracing::error!("Error processing historical messages: {:?}", err); - 0 - } - } -} pub(crate) async fn process_historical_messages( - current_block_height: BlockHeight, + start_block_height: BlockHeight, indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, s3_client: &S3Client, chain_id: &ChainId, json_rpc_client: &JsonRpcClient, -) -> anyhow::Result { - let start_block = indexer.start_block_height.unwrap(); - let block_difference: i64 = (current_block_height - start_block) as i64; - match block_difference { - i64::MIN..=-1 => { - bail!( - "Skipping back fill, start_block_height is greater than current block height: {}", - indexer.get_full_name(), - ); - } - 0 => { - bail!( - "Skipping back fill, start_block_height is equal to current block height: {}", - indexer.get_full_name(), - ); - } - 1..=i64::MAX => { - tracing::info!( - "Back filling {block_difference} blocks from {start_block} to current block height {current_block_height}: {}", - indexer.get_full_name(), - ); - - let start_date = - lookup_block_date_or_next_block_date(start_block, json_rpc_client).await?; +) -> anyhow::Result<()> { + tracing::info!( + "Starting block stream from {start_block_height} for indexer: {}", + indexer.get_full_name(), + ); - let last_indexed_block = last_indexed_block_from_metadata(s3_client).await?; + let start_date = + lookup_block_date_or_next_block_date(start_block_height, json_rpc_client).await?; - let blocks_from_index = filter_matching_blocks_from_index_files( - start_block, - &indexer, - s3_client, - start_date, - ) - .await?; - - tracing::info!( - "Flushing {} block heights from index files to historical Stream for indexer: {}", - blocks_from_index.len(), - indexer.get_full_name(), - ); - - for block in &blocks_from_index { - crate::redis::xadd( - redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer.get_full_name()), - &[("block_height", block)], - ) - .await - .context("Failed to add block to Redis Stream")?; - } + let last_indexed_block = last_indexed_block_from_metadata(s3_client).await?; - // Check for the case where an index file is written right after we get the last_indexed_block metadata - let last_block_in_data = blocks_from_index.last().unwrap_or(&start_block); - let last_indexed_block = if last_block_in_data > &last_indexed_block { - *last_block_in_data - } else { - last_indexed_block - }; - - let unindexed_block_difference = current_block_height - last_indexed_block; - - tracing::info!( - "Filtering {} unindexed blocks from lake: from block {last_indexed_block} to {current_block_height} for indexer: {}", - unindexed_block_difference, - indexer.get_full_name(), - ); - - if unindexed_block_difference > UNINDEXED_BLOCKS_SOFT_LIMIT { - tracing::warn!( - "Unindexed block difference exceeds soft limit of: {UNINDEXED_BLOCKS_SOFT_LIMIT} for indexer: {}", - indexer.get_full_name(), - ); - } + let blocks_from_index = filter_matching_blocks_from_index_files( + start_block_height, + &indexer, + s3_client, + start_date, + ) + .await?; - let lake_config = match &chain_id { - ChainId::Mainnet => near_lake_framework::LakeConfigBuilder::default().mainnet(), - ChainId::Testnet => near_lake_framework::LakeConfigBuilder::default().testnet(), - } - .start_block_height(last_indexed_block) - .build() - .context("Failed to build lake config")?; + tracing::info!( + "Flushing {} block heights from index files to historical Stream for indexer: {}", + blocks_from_index.len(), + indexer.get_full_name(), + ); - let (sender, mut stream) = near_lake_framework::streamer(lake_config); + for block in &blocks_from_index { + crate::redis::xadd( + redis_connection_manager, + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), + &[("block_height", block)], + ) + .await + .context("Failed to add block to Redis Stream")?; + } - let mut filtered_block_count = 0; - while let Some(streamer_message) = stream.recv().await { - let block_height = streamer_message.block.header.height; - if block_height == current_block_height { - break; - } + // Check for the case where an index file is written right after we get the last_indexed_block metadata + let last_block_in_data = blocks_from_index.last().unwrap_or(&start_block_height); + let last_indexed_block = if last_block_in_data > &last_indexed_block { + *last_block_in_data + } else { + last_indexed_block + }; - let matches = crate::rules::reduce_indexer_rule_matches( - &indexer.indexer_rule, - &streamer_message, - chain_id.clone(), - ); - - if !matches.is_empty() { - filtered_block_count += 1; - - crate::redis::xadd( - redis_connection_manager, - crate::redis::generate_historical_stream_key(&indexer.get_full_name()), - &[("block_height", block_height)], - ) - .await?; - } - } - drop(sender); + tracing::info!( + "Starting near-lake-framework from {last_indexed_block} for indexer: {}", + indexer.get_full_name(), + ); - tracing::info!( - "Flushed {} unindexed block heights to historical Stream for indexer: {}", - filtered_block_count, - indexer.get_full_name(), - ); + let lake_config = match &chain_id { + ChainId::Mainnet => near_lake_framework::LakeConfigBuilder::default().mainnet(), + ChainId::Testnet => near_lake_framework::LakeConfigBuilder::default().testnet(), + } + .start_block_height(last_indexed_block) + .build() + .context("Failed to build lake config")?; + + let (sender, mut stream) = near_lake_framework::streamer(lake_config); + + let mut filtered_block_count = 0; + while let Some(streamer_message) = stream.recv().await { + let block_height = streamer_message.block.header.height; + eprintln!("block_height = {:?}", block_height); + + let matches = crate::rules::reduce_indexer_rule_matches( + &indexer.indexer_rule, + &streamer_message, + chain_id.clone(), + ); + + if !matches.is_empty() { + filtered_block_count += 1; + + crate::redis::xadd( + redis_connection_manager, + crate::redis::generate_historical_stream_key(&indexer.get_full_name()), + &[("block_height", block_height)], + ) + .await?; } } - Ok(block_difference) + + drop(sender); + + tracing::info!( + "Flushed {} unindexed block heights to historical Stream for indexer: {}", + filtered_block_count, + indexer.get_full_name(), + ); + + Ok(()) } pub(crate) async fn last_indexed_block_from_metadata( diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index b539977e7..45f29a952 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -52,7 +52,7 @@ async fn main() -> anyhow::Result<()> { let mut streamer = block_streamer::BlockStreamer::new(); streamer.start( - 106230334 + 1000, + 106000000, indexer, redis_connection_manager, s3_client, From 93facb4a91bbfcc79e1df90c4fac3aed546a9813 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 13:20:24 +1300 Subject: [PATCH 14/40] refactor: Rename `process_historical_messages` to `start_block_stream` --- block-streamer/src/block_streamer.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index e0398dbb0..1cc58b5a2 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -12,7 +12,6 @@ use serde_json::from_str; use tokio::task::JoinHandle; pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; -pub const UNINDEXED_BLOCKS_SOFT_LIMIT: u64 = 7200; pub struct Task { handle: JoinHandle<()>, @@ -48,11 +47,11 @@ impl BlockStreamer { tokio::select! { _ = cancellation_token_clone.cancelled() => { tracing::info!( - "Cancelling existing historical backfill for indexer: {}", + "Cancelling existing block stream task for indexer: {}", indexer.get_full_name(), ); }, - _ = process_historical_messages( + _ = start_block_stream( start_block_height, indexer.clone(), &redis_connection_manager, @@ -61,7 +60,7 @@ impl BlockStreamer { &json_rpc_client, ) => { tracing::info!( - "Finished historical backfill for indexer: {}", + "Finished streaming blocks for indexer: {}", indexer.get_full_name(), ); } @@ -94,7 +93,7 @@ impl BlockStreamer { } } -pub(crate) async fn process_historical_messages( +pub(crate) async fn start_block_stream( start_block_height: BlockHeight, indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, @@ -162,7 +161,6 @@ pub(crate) async fn process_historical_messages( let mut filtered_block_count = 0; while let Some(streamer_message) = stream.recv().await { let block_height = streamer_message.block.header.height; - eprintln!("block_height = {:?}", block_height); let matches = crate::rules::reduce_indexer_rule_matches( &indexer.indexer_rule, From 34e504964dd74adf356d269fb74efe937ddcc708 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 14:09:30 +1300 Subject: [PATCH 15/40] feat: Log and propogate errors in block stream task --- block-streamer/src/block_streamer.rs | 22 ++++++++++++++-------- block-streamer/src/main.rs | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 1cc58b5a2..f91977c07 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -14,7 +14,7 @@ use tokio::task::JoinHandle; pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; pub struct Task { - handle: JoinHandle<()>, + handle: JoinHandle>, cancellation_token: tokio_util::sync::CancellationToken, } @@ -50,8 +50,10 @@ impl BlockStreamer { "Cancelling existing block stream task for indexer: {}", indexer.get_full_name(), ); + + Ok(()) }, - _ = start_block_stream( + result = start_block_stream( start_block_height, indexer.clone(), &redis_connection_manager, @@ -59,10 +61,14 @@ impl BlockStreamer { &chain_id, &json_rpc_client, ) => { - tracing::info!( - "Finished streaming blocks for indexer: {}", - indexer.get_full_name(), - ); + result.map_err(|err| { + tracing::error!( + "Block stream task for indexer: {} stopped due to error: {:?}", + indexer.get_full_name(), + err, + ); + err + }) } } }); @@ -78,7 +84,7 @@ impl BlockStreamer { pub async fn cancel(&mut self) -> anyhow::Result<()> { if let Some(task) = self.task.take() { task.cancellation_token.cancel(); - task.handle.await?; + task.handle.await??; return Ok(()); } @@ -88,7 +94,7 @@ impl BlockStreamer { )) } - pub fn take_handle(&mut self) -> Option> { + pub fn take_handle(&mut self) -> Option>> { self.task.take().map(|task| task.handle) } } diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 45f29a952..49b26cb4b 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -60,7 +60,7 @@ async fn main() -> anyhow::Result<()> { json_rpc_client, )?; - streamer.take_handle().unwrap().await?; + streamer.take_handle().unwrap().await??; println!("done"); From ae2eab17cf6a8332dff56c32cf3dc2d964e7d6da Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 14:17:11 +1300 Subject: [PATCH 16/40] feat: Ignore error in block stream when cancelling --- block-streamer/src/block_streamer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index f91977c07..1cdcd6631 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -84,7 +84,7 @@ impl BlockStreamer { pub async fn cancel(&mut self) -> anyhow::Result<()> { if let Some(task) = self.task.take() { task.cancellation_token.cancel(); - task.handle.await??; + let _ = task.handle.await?; return Ok(()); } From 57e0c4bff8d52875976be8ab8fb11281679449b8 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Wed, 22 Nov 2023 15:27:40 +1300 Subject: [PATCH 17/40] refactor: Make decision between index/metadata more clear --- block-streamer/src/block_streamer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 1cdcd6631..8140091b0 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -141,13 +141,13 @@ pub(crate) async fn start_block_stream( .context("Failed to add block to Redis Stream")?; } - // Check for the case where an index file is written right after we get the last_indexed_block metadata - let last_block_in_data = blocks_from_index.last().unwrap_or(&start_block_height); - let last_indexed_block = if last_block_in_data > &last_indexed_block { - *last_block_in_data - } else { - last_indexed_block - }; + let last_indexed_block = + blocks_from_index + .last() + .map_or(last_indexed_block, |&last_block_in_index| { + // Check for the case where index files are written right after we fetch the last_indexed_block metadata + std::cmp::max(last_block_in_index, last_indexed_block) + }); tracing::info!( "Starting near-lake-framework from {last_indexed_block} for indexer: {}", From b0876b38addd171e3add23fbeace2d22a24fada1 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 14:39:18 +1300 Subject: [PATCH 18/40] chore: Update crates to latest version --- block-streamer/Cargo.lock | 1245 ++++++++++++++++++++++++++++++------- block-streamer/Cargo.toml | 18 +- block-streamer/src/s3.rs | 2 +- 3 files changed, 1035 insertions(+), 230 deletions(-) diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index 71f597855..2621a191b 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", ] [[package]] @@ -39,9 +39,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" +checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", "tokio", @@ -94,6 +94,54 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.75" @@ -171,17 +219,17 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "741327a7f70e6e639bdb5061964c66250460c70ad3f59c3fe2a3a64ac1484e33" dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", + "aws-credential-types 0.53.0", + "aws-http 0.53.0", + "aws-sdk-sso 0.23.0", + "aws-sdk-sts 0.23.0", + "aws-smithy-async 0.53.1", "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-http 0.53.1", "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", + "aws-smithy-json 0.53.1", + "aws-smithy-types 0.53.1", + "aws-types 0.53.0", "bytes", "hex", "http", @@ -194,28 +242,71 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-config" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e245d7c741a8e4b23133f36750c4bcb3938a66ac49510caaf8b83afd52db1ec" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-runtime", + "aws-sdk-sso 0.39.0", + "aws-sdk-ssooidc", + "aws-sdk-sts 0.39.0", + "aws-smithy-async 1.0.1", + "aws-smithy-http 0.60.0", + "aws-smithy-json 0.60.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", + "bytes", + "fastrand 2.0.1", + "hex", + "http", + "hyper", + "ring 0.17.5", + "time", + "tokio", + "tracing", + "zeroize", +] + [[package]] name = "aws-credential-types" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f99dd587a46af58f8cf37773687ecec19d0373a5954942d7e0f405751fe2369" dependencies = [ - "aws-smithy-async", - "aws-smithy-types", + "aws-smithy-async 0.53.1", + "aws-smithy-types 0.53.1", "tokio", "tracing", "zeroize", ] +[[package]] +name = "aws-credential-types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6dec6f3d42983be70a113f999476185e124884f43f4d60129c7157aede7bda1" +dependencies = [ + "aws-smithy-async 1.0.1", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "zeroize", +] + [[package]] name = "aws-endpoint" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13fdfc00c57d95e10bcf83d2331c4ae9ca460ca84dc983b2cdd692de87640389" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", + "aws-smithy-http 0.53.1", + "aws-smithy-types 0.53.1", + "aws-types 0.53.0", "http", "regex", "tracing", @@ -227,10 +318,10 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74cdac70481d144bf7001c27884b95ee12c8f62e61db90320d59b673ae121cb8" dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-types", - "aws-types", + "aws-credential-types 0.53.0", + "aws-smithy-http 0.53.1", + "aws-smithy-types 0.53.1", + "aws-types 0.53.0", "bytes", "http", "http-body", @@ -240,27 +331,65 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-http" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-runtime" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ba3ad97d674bfaeed684d528a07cee6e81cbf16e6d6c7e272a130b5e71e6b9" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-sigv4 1.0.0", + "aws-smithy-async 1.0.1", + "aws-smithy-eventstream 0.60.0", + "aws-smithy-http 0.60.0", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", + "fastrand 2.0.1", + "http", + "percent-encoding", + "tracing", + "uuid", +] + [[package]] name = "aws-sdk-s3" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ae411cb03ea6df0d4c4340a0d3c15cab7b19715d091f76c5629f31acd6403f3" dependencies = [ - "aws-credential-types", + "aws-credential-types 0.53.0", "aws-endpoint", - "aws-http", + "aws-http 0.53.0", "aws-sig-auth", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", + "aws-sigv4 0.53.2", + "aws-smithy-async 0.53.1", + "aws-smithy-checksums 0.53.1", "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-eventstream 0.53.1", + "aws-smithy-http 0.53.1", "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", + "aws-smithy-json 0.53.1", + "aws-smithy-types 0.53.1", + "aws-smithy-xml 0.53.1", + "aws-types 0.53.0", "bytes", "bytes-utils", "fastrand 1.9.0", @@ -275,23 +404,53 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-s3" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29223b1074621f1d011bac836d995c002936663052b1e7ad02927551b17d6625" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-runtime", + "aws-sigv4 1.0.0", + "aws-smithy-async 1.0.1", + "aws-smithy-checksums 0.60.0", + "aws-smithy-eventstream 0.60.0", + "aws-smithy-http 0.60.0", + "aws-smithy-json 0.60.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-smithy-xml 0.60.0", + "aws-types 1.0.0", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5d2fb56182ac693a19364cc0bde22d95aef9be3663bf9b906ffbd0ab0a7c7d1" dependencies = [ - "aws-credential-types", + "aws-credential-types 0.53.0", "aws-endpoint", - "aws-http", + "aws-http 0.53.0", "aws-sig-auth", - "aws-smithy-async", + "aws-smithy-async 0.53.1", "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-http 0.53.1", "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", + "aws-smithy-json 0.53.1", + "aws-smithy-types 0.53.1", + "aws-types 0.53.0", "bytes", "http", "regex", @@ -300,25 +459,69 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sso" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5786afe1fd164e53f108b2bd4982a31c5a821dc1677d05a12de65ebcc6ede52a" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-runtime", + "aws-smithy-async 1.0.1", + "aws-smithy-http 0.60.0", + "aws-smithy-json 0.60.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31569ac7750ebc3097058c1e72d79576e16b3fb262aa0d6510188bff623f9804" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-runtime", + "aws-smithy-async 1.0.1", + "aws-smithy-http 0.60.0", + "aws-smithy-json 0.60.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", + "bytes", + "http", + "regex", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a70adf3e9518c8d6d14f1239f6af04c019ffd260ab791e17deb11f1bce6a9f76" dependencies = [ - "aws-credential-types", + "aws-credential-types 0.53.0", "aws-endpoint", - "aws-http", + "aws-http 0.53.0", "aws-sig-auth", - "aws-smithy-async", + "aws-smithy-async 0.53.1", "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-http 0.53.1", "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", + "aws-smithy-json 0.53.1", + "aws-smithy-query 0.53.1", + "aws-smithy-types 0.53.1", + "aws-smithy-xml 0.53.1", + "aws-types 0.53.0", "bytes", "http", "regex", @@ -327,17 +530,40 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sts" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1d955bacd8c3637908a40a4af2f7a732461ee7d95ec02599a3610ee55781d7" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-http 0.60.0", + "aws-runtime", + "aws-smithy-async 1.0.1", + "aws-smithy-http 0.60.0", + "aws-smithy-json 0.60.0", + "aws-smithy-query 0.60.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "aws-smithy-xml 0.60.0", + "aws-types 1.0.0", + "http", + "regex", + "tracing", +] + [[package]] name = "aws-sig-auth" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22af7f6515f8b51dabef87df1d901c9734e4e367791c6d0e1082f9f31528120e" dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", + "aws-credential-types 0.53.0", + "aws-sigv4 0.53.2", + "aws-smithy-eventstream 0.53.1", + "aws-smithy-http 0.53.1", + "aws-types 0.53.0", "http", "tracing", ] @@ -348,19 +574,48 @@ version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14500f741fb73a3c6cb173f8d96b433319a0e27c370a4e783b9ad693fc86210e" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-eventstream 0.53.1", + "aws-smithy-http 0.53.1", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2 0.10.8", + "time", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d07e2f2fc32acb7423d054ec8ba2b361dafc95fabc5a211baf4a566571d20e" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-smithy-eventstream 0.60.0", + "aws-smithy-http 0.60.0", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", "http", "once_cell", + "p256", "percent-encoding", "regex", + "ring 0.17.5", "sha2 0.10.8", + "subtle", "time", "tracing", + "zeroize", ] [[package]] @@ -375,14 +630,46 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "aws-smithy-async" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbfa248f7f966d73e325dbc85851a5500042b6d96e3c3b535a8527707f36fe4" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + [[package]] name = "aws-smithy-checksums" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e9e4d3c2296bcec2c03f9f769ac9b2424d972c2fe7afc0b59235447ac3a5c3" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.53.1", + "aws-smithy-types 0.53.1", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1 0.10.6", + "sha2 0.10.8", + "tracing", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" +dependencies = [ + "aws-smithy-http 0.60.0", + "aws-smithy-types 1.0.1", "bytes", "crc32c", "crc32fast", @@ -402,16 +689,16 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "710ca0f8dacddda5fbcaf5c3cd9d02da7913fd463a2ee9555b617bf168bedacb" dependencies = [ - "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-async 0.53.1", + "aws-smithy-http 0.53.1", "aws-smithy-http-tower", - "aws-smithy-types", + "aws-smithy-types 0.53.1", "bytes", "fastrand 1.9.0", "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", "tokio", @@ -425,7 +712,18 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d1ff11ee22de3581114b60d4ae8e700638dacb5b5bbe6769726e251e6c3f20a" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.53.1", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c669e1e5fc0d79561bf7a122b118bd50c898758354fe2c53eb8f2d31507cbc3" +dependencies = [ + "aws-smithy-types 1.0.1", "bytes", "crc32fast", ] @@ -436,8 +734,8 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29dcab29afbea7726f5c10c7be0c38666d7eb07db551580b3b26ed7cfb5d1935" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", + "aws-smithy-eventstream 0.53.1", + "aws-smithy-types 0.53.1", "bytes", "bytes-utils", "futures-core", @@ -449,7 +747,28 @@ dependencies = [ "pin-project-lite", "pin-utils", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" +dependencies = [ + "aws-smithy-eventstream 0.60.0", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", "tracing", ] @@ -459,8 +778,8 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5856d2f1063c0f726a85f32dcd2a9f5a1d994eb27b156abccafc7260f3f471d" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.53.1", + "aws-smithy-types 0.53.1", "bytes", "http", "http-body", @@ -475,7 +794,16 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb33659b68480495b5f906b946c8642928440118b1d7e26a25a067303ca01a5" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.53.1", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" +dependencies = [ + "aws-smithy-types 1.0.1", ] [[package]] @@ -484,21 +812,94 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c4b21ee0e30ff046e87c7b7e017b99d445b42a81fe52c6e5139b23b795a98ae" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.53.1", + "urlencoding", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" +dependencies = [ + "aws-smithy-types 1.0.1", "urlencoding", ] +[[package]] +name = "aws-smithy-runtime" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064b808143d80b50744b1b22cce801238a545b84859c6cf8e275997252dd1d25" +dependencies = [ + "aws-smithy-async 1.0.1", + "aws-smithy-http 0.60.0", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", + "bytes", + "fastrand 2.0.1", + "http", + "http-body", + "hyper", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.9", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d27c3235d4972ed976b5c1a82286e7c4457f618f3c2ae6d4ae44f081dd24575" +dependencies = [ + "aws-smithy-async 1.0.1", + "aws-smithy-types 1.0.1", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + [[package]] name = "aws-smithy-types" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2013465a070decdeb3e85ceb3370ae85ba05f56f914abfd89858d7281c4f12c3" dependencies = [ - "base64-simd", + "base64-simd 0.7.0", + "itoa", + "num-integer", + "ryu", + "time", +] + +[[package]] +name = "aws-smithy-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fc32035dc0636a8583cf0c6dd7f1e6d5404103b836d26228b8730907a88d9f" +dependencies = [ + "base64-simd 0.8.0", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", + "serde", "time", + "tokio", + "tokio-util 0.7.10", ] [[package]] @@ -510,22 +911,52 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "aws-smithy-xml" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" +dependencies = [ + "xmlparser", +] + [[package]] name = "aws-types" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61f00f4b0cdd345686e6389f3343a3020f93232d20040802b87673ddc2d02956" dependencies = [ - "aws-credential-types", - "aws-smithy-async", + "aws-credential-types 0.53.0", + "aws-smithy-async 0.53.1", "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.53.1", + "aws-smithy-types 0.53.1", + "http", + "rustc_version", + "tracing", +] + +[[package]] +name = "aws-types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18c0eb301cce69298555c4884795497c67b3bd511aa3f07470382f4bf3ef043" +dependencies = [ + "aws-credential-types 1.0.0", + "aws-smithy-async 1.0.1", + "aws-smithy-runtime-api", + "aws-smithy-types 1.0.1", "http", "rustc_version", "tracing", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -547,6 +978,22 @@ dependencies = [ "simd-abstraction", ] +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref 0.5.1", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -593,11 +1040,16 @@ name = "block-streamer" version = "0.1.0" dependencies = [ "anyhow", - "aws-config", - "aws-sdk-s3", + "async-trait", + "aws-config 1.0.0", + "aws-sdk-s3 0.39.1", + "aws-smithy-http 0.60.0", + "aws-smithy-types 1.0.1", + "aws-types 1.0.0", "borsh", "chrono", "futures", + "mockall", "near-jsonrpc-client", "near-jsonrpc-primitives", "near-lake-framework", @@ -605,7 +1057,8 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-util 0.6.10", + "tokio-stream", + "tokio-util 0.7.10", "tracing", "tracing-subscriber", "wildmatch", @@ -750,42 +1203,49 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.25" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ - "atty", - "bitflags 1.3.2", + "clap_builder", "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +dependencies = [ + "anstream", + "anstyle", "clap_lex", - "indexmap 1.9.3", - "once_cell", "strsim", - "termcolor", - "textwrap", ] [[package]] name = "clap_derive" -version = "3.2.25" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck 0.4.1", - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] name = "clap_lex" -version = "0.2.4" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "combine" @@ -798,9 +1258,15 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", ] +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + [[package]] name = "convert_case" version = "0.4.0" @@ -875,6 +1341,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -897,9 +1385,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "3.2.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder", "digest 0.9.0", @@ -914,8 +1402,18 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core 0.20.3", + "darling_macro 0.20.3", ] [[package]] @@ -932,17 +1430,52 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.39", +] + [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core", + "darling_core 0.14.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core 0.20.3", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.9" @@ -950,6 +1483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -978,7 +1512,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", @@ -1007,6 +1541,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.9.0" @@ -1027,12 +1567,30 @@ dependencies = [ "subtle", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "easy-ext" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53aff6fdc1b181225acdcb5b14c47106726fd8e486707315b1b138baed68ee31" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "ed25519" version = "1.5.3" @@ -1062,6 +1620,26 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -1120,7 +1698,17 @@ dependencies = [ name = "fastrand" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] [[package]] name = "fixed-hash" @@ -1137,6 +1725,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1160,13 +1757,19 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "fs2" version = "0.4.3" @@ -1298,6 +1901,17 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.22" @@ -1313,7 +1927,7 @@ dependencies = [ "indexmap 2.1.0", "slab", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", "tracing", ] @@ -1462,10 +2076,26 @@ dependencies = [ "http", "hyper", "log", - "rustls", + "rustls 0.20.9", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.23.4", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "log", + "rustls 0.21.9", "rustls-native-certs", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -1524,9 +2154,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1540,6 +2170,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1550,6 +2181,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", + "serde", ] [[package]] @@ -1669,11 +2301,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "multimap" version = "0.8.3" @@ -1700,20 +2358,19 @@ dependencies = [ [[package]] name = "near-account-id" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12791d0f273e04609010d68deb6e1cd940659ad420edfa2e48238117154f1d5b" +checksum = "dc0cb40869cab7f5232f934f45db35bffe0f2d2a7cb0cd0346202fbe4ebf2dd7" dependencies = [ - "arbitrary", "borsh", "serde", ] [[package]] name = "near-chain-configs" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84781ba3321e28640c5f0453d4d84305135db6edc399c51b7ee405ebb9381eaa" +checksum = "c4f9a1c805846237d56f99b328ba6ab77e5d43ef59aaaf8d2a41d42fdc708a7b" dependencies = [ "anyhow", "chrono", @@ -1733,9 +2390,9 @@ dependencies = [ [[package]] name = "near-config-utils" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "137b6bbd477dc0ff1149e22c5e1f3f168a1f37d9d67f1519f058a5db81df3506" +checksum = "5523e7dce493c45bc3241eb3100d943ec471852f9b1f84b46a34789eadf17031" dependencies = [ "anyhow", "json_comments", @@ -1745,9 +2402,9 @@ dependencies = [ [[package]] name = "near-crypto" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "659a96750c4d933e4f00a50c66ba9948a32b862e5ecd6a952beee881b1cd2aaf" +checksum = "ff6b382b626e7e0cd372d027c6672ac97b4b6ee6114288c9e58d8180b935d315" dependencies = [ "blake2", "borsh", @@ -1770,11 +2427,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "near-fmt" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c44c842c6cfcd9b8c387cccd4cd0619a5f21920cde5d5c292af3cc5d40510672" +dependencies = [ + "near-primitives-core", +] + [[package]] name = "near-indexer-primitives" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d10699cd998980a8965ee0d1922a77585b7ed1031934de29fb982ef4ae6ad2" +checksum = "b76c87827dcae78979748c3864d209d5906163958a01551afc2092a8ad56fa39" dependencies = [ "near-primitives", "serde", @@ -1783,9 +2449,9 @@ dependencies = [ [[package]] name = "near-jsonrpc-client" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf020bd5fb7952c8f005ab789ac6f7a2015557630571362bcbaf2ead27c5cb6" +checksum = "118f44c02ad211db805c1370ad3ff26576af6ff554093c9fece1b835d29d233a" dependencies = [ "borsh", "lazy_static", @@ -1802,9 +2468,9 @@ dependencies = [ [[package]] name = "near-jsonrpc-primitives" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0968cc71825e020d75f5a1f3577ac0379859ec7b14351ed5754fef1cc89e0d7" +checksum = "97b2934b5ab243e25e951c984525ba0aff0e719ed915c988c5195405aa0f6987" dependencies = [ "arbitrary", "near-chain-configs", @@ -1818,17 +2484,17 @@ dependencies = [ [[package]] name = "near-lake-framework" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b733c11d21cb97f66721ebea525c1fd4fb6fdd4aeab5c7c9562363230f071e" +checksum = "5fbfc3d7c294aa144c3a1817452931e91ce045ee1cf2e34b7b439d4695073634" dependencies = [ "anyhow", "async-stream", "async-trait", - "aws-config", - "aws-credential-types", - "aws-sdk-s3", - "aws-types", + "aws-config 0.53.0", + "aws-credential-types 0.53.0", + "aws-sdk-s3 0.23.0", + "aws-types 0.53.0", "derive_builder", "futures", "near-indexer-primitives", @@ -1842,9 +2508,9 @@ dependencies = [ [[package]] name = "near-o11y" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445cbd86499dbabf68d34d95d9a45c680da5446e226f4982ce488fcf8556b23f" +checksum = "af7d35397b02b131c188c72f3885e97daeccab134ec2fc8cc0073a94cf1cfe19" dependencies = [ "actix", "atty", @@ -1868,9 +2534,9 @@ dependencies = [ [[package]] name = "near-primitives" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4c030f28e8f988698145e7753b83bb54c05838c3afdd44835cc7c32c327ea1e" +checksum = "05f7051aaf199adc4d068620fca6d5f70f906a1540d03a8bb3701271f8881835" dependencies = [ "arbitrary", "borsh", @@ -1882,7 +2548,7 @@ dependencies = [ "enum-map", "hex", "near-crypto", - "near-o11y", + "near-fmt", "near-primitives-core", "near-rpc-error-macro", "near-stdx", @@ -1894,23 +2560,23 @@ dependencies = [ "reed-solomon-erasure", "serde", "serde_json", + "serde_with", "serde_yaml", "smart-default", "strum", "thiserror", "time", - "tokio", "tracing", ] [[package]] name = "near-primitives-core" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2059d16efc42ef7f9514f30910d32b67c01fee9b70c1fd28d50545ca145d88" +checksum = "775fec19ef51a341abdbf792a9dda5b4cb89f488f681b2fd689b9321d24db47b" dependencies = [ "arbitrary", - "base64 0.13.1", + "base64 0.21.5", "borsh", "bs58", "derive_more", @@ -1919,6 +2585,7 @@ dependencies = [ "num-rational", "serde", "serde_repr", + "serde_with", "sha2 0.10.8", "strum", "thiserror", @@ -1926,38 +2593,38 @@ dependencies = [ [[package]] name = "near-rpc-error-core" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f8f38dcfeba3d0d3bc60ce292ddb1f76a428a590e32de7fc3d5d431b9635ea" +checksum = "84c1eda300e2e78f4f945ae58117d49e806899f4a51ee2faa09eda5ebc2e6571" dependencies = [ "quote", "serde", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] name = "near-rpc-error-macro" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca4e8d6887b344f3e2f8281c1ad2cc93dcbcb683b546726a4ce1ab1dfa623a" +checksum = "31d2dadd765101c77e664029dd6fbec090e696877d4ae903c620d02ceda4969a" dependencies = [ "fs2", "near-rpc-error-core", "serde", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] name = "near-stdx" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1279be274b9a49c2cb4b62541241a1ff6745cb77ca81ece7f949cfbc229bff" +checksum = "6540152fba5e96fe5d575b79e8cd244cf2add747bb01362426bdc069bc3a23bc" [[package]] name = "near-vm-errors" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2d3eab1e050fdc3e036c803784cf45582661ae2dd07bac3bd373ba9c049715" +checksum = "ec545d1bede0579e7c15dd2dce9b998dc975c52f2165702ff40bec7ff69728bb" dependencies = [ "borsh", "near-account-id", @@ -1967,6 +2634,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2135,16 +2808,16 @@ dependencies = [ ] [[package]] -name = "os_str_bytes" -version = "6.6.1" +name = "outref" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4" [[package]] name = "outref" -version = "0.1.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" [[package]] name = "overload" @@ -2152,6 +2825,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.8", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2177,9 +2861,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -2223,6 +2907,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -2242,46 +2936,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] -name = "primitive-types" -version = "0.10.1" +name = "predicates" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ - "fixed-hash", - "uint", + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", ] [[package]] -name = "proc-macro-crate" -version = "0.1.5" +name = "predicates-core" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" dependencies = [ - "toml", + "predicates-core", + "termtree", ] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "primitive-types" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", + "fixed-hash", + "uint", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "proc-macro-crate" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "proc-macro2", - "quote", - "version_check", + "toml", ] [[package]] @@ -2465,7 +3165,7 @@ dependencies = [ "ryu", "sha1 0.6.1", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", "url", ] @@ -2569,6 +3269,17 @@ dependencies = [ "winreg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.16.20" @@ -2632,6 +3343,18 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "log", + "ring 0.17.5", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -2653,6 +3376,16 @@ dependencies = [ "base64 0.21.5", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -2690,11 +3423,25 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secp256k1" -version = "0.24.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ "rand 0.8.5", "secp256k1-sys", @@ -2702,9 +3449,9 @@ dependencies = [ [[package]] name = "secp256k1-sys" -version = "0.6.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" dependencies = [ "cc", ] @@ -2792,6 +3539,35 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +dependencies = [ + "base64 0.21.5", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +dependencies = [ + "darling 0.20.3", + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "serde_yaml" version = "0.9.27" @@ -2878,6 +3654,10 @@ name = "signature" version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] [[package]] name = "simd-abstraction" @@ -2885,7 +3665,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cadb29c57caadc51ff8346233b5cec1d240b68ce55cf1afc764818791876987" dependencies = [ - "outref", + "outref 0.1.0", ] [[package]] @@ -2936,6 +3716,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -3033,19 +3823,10 @@ dependencies = [ ] [[package]] -name = "termcolor" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.0" +name = "termtree" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" @@ -3123,22 +3904,21 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.19.2" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ + "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", - "once_cell", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys", ] [[package]] @@ -3153,13 +3933,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] @@ -3178,11 +3958,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.9", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.9", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -3210,9 +4000,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -3288,7 +4078,7 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util 0.7.3", + "tokio-util 0.7.10", "tower-layer", "tower-service", "tracing", @@ -3486,9 +4276,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3501,6 +4291,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" + [[package]] name = "valuable" version = "0.1.0" @@ -3519,6 +4321,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -3660,15 +4468,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -3788,9 +4587,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" dependencies = [ "zeroize_derive", ] diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index 1fb4407a7..9cbc9a893 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -7,20 +7,26 @@ edition = "2021" [dependencies] anyhow = "1.0.57" -aws-sdk-s3 = "0.23.0" -aws-config = "0.53.0" +async-trait = "0.1.74" +aws-smithy-http = "0.60.0" +aws-smithy-types = "1.0.1" +aws-sdk-s3 = "0.39.1" +aws-config = { version = "1.0.0", features = ["behavior-version-latest"]} borsh = "0.10.2" chrono = "0.4.25" futures = "0.3.5" +mockall = "0.11.4" redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" tracing = "0.1.40" tracing-subscriber = "0.3.18" -tokio = { version = "1.1", features = ["sync", "time", "macros", "rt-multi-thread"] } -tokio-util = "0.6.7" +tokio = "1.28.0" +tokio-util = "0.7.10" +tokio-stream = "0.1.14" wildmatch = "2.1.1" near-lake-framework = "0.7.1" -near-jsonrpc-client = "0.5.1" -near-jsonrpc-primitives = "0.16.0" +near-jsonrpc-client = "0.6.0" +near-jsonrpc-primitives = "0.17.0" +aws-types = "1.0.0" diff --git a/block-streamer/src/s3.rs b/block-streamer/src/s3.rs index eaf3bc003..0356184db 100644 --- a/block-streamer/src/s3.rs +++ b/block-streamer/src/s3.rs @@ -328,7 +328,7 @@ mod tests { let wrapped_error = s3_result.err().unwrap(); let error = wrapped_error.root_cause(); if error - .downcast_ref::() + .downcast_ref::() .is_some() { success = true; From 11a07da4dcdd80d64c8af0ac9831714ebb907a34 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 14:52:00 +1300 Subject: [PATCH 19/40] feat: Add mockable s3 client --- block-streamer/src/main.rs | 1 + block-streamer/src/s3_client.rs | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 block-streamer/src/s3_client.rs diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 49b26cb4b..256509e07 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -10,6 +10,7 @@ mod indexer_config; mod redis; mod rules; mod s3; +mod s3_client; pub(crate) const LOG_TARGET: &str = "block_streamer"; diff --git a/block-streamer/src/s3_client.rs b/block-streamer/src/s3_client.rs new file mode 100644 index 000000000..2a6dfa9f0 --- /dev/null +++ b/block-streamer/src/s3_client.rs @@ -0,0 +1,55 @@ +#[async_trait::async_trait] +#[mockall::automock] +pub trait S3ClientTrait { + async fn get_object( + &self, + bucket: &str, + prefix: &str, + ) -> Result< + aws_sdk_s3::operation::get_object::GetObjectOutput, + aws_sdk_s3::error::SdkError, + >; + + async fn get_text_file(&self, bucket: &str, prefix: &str) -> anyhow::Result; +} + +#[derive(Clone, Debug)] +pub struct S3Client { + client: aws_sdk_s3::Client, +} + +impl S3Client { + pub fn new(aws_config: &aws_types::sdk_config::SdkConfig) -> Self { + Self { + client: aws_sdk_s3::Client::new(aws_config), + } + } +} + +#[async_trait::async_trait] +impl S3ClientTrait for S3Client { + async fn get_object( + &self, + bucket: &str, + prefix: &str, + ) -> Result< + aws_sdk_s3::operation::get_object::GetObjectOutput, + aws_sdk_s3::error::SdkError, + > { + self.client + .get_object() + .bucket(bucket) + .key(prefix) + .request_payer(aws_sdk_s3::types::RequestPayer::Requester) + .send() + .await + } + + async fn get_text_file(&self, bucket: &str, prefix: &str) -> anyhow::Result { + let object = self.get_object(bucket, prefix).await?; + + let bytes = object.body.collect().await?; + + Ok(String::from_utf8(bytes.to_vec())?) + } +} From d8116ec1c172da4fac8e971e8cd68706e4774a18 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 14:55:30 +1300 Subject: [PATCH 20/40] feat: Add `DeltaLakeClient` using s3 client mock --- block-streamer/src/delta_lake_client.rs | 17 +++++++++++++++++ block-streamer/src/main.rs | 1 + 2 files changed, 18 insertions(+) create mode 100644 block-streamer/src/delta_lake_client.rs diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs new file mode 100644 index 000000000..c2e1598aa --- /dev/null +++ b/block-streamer/src/delta_lake_client.rs @@ -0,0 +1,17 @@ +use anyhow::Context; + +pub struct DeltaLakeClient +where + T: crate::s3_client::S3ClientTrait, +{ + s3_client: T, +} + +impl DeltaLakeClient +where + T: crate::s3_client::S3ClientTrait, +{ + pub fn new(s3_client: T) -> Self { + DeltaLakeClient { s3_client } + } +} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 256509e07..8a1a9e98e 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -6,6 +6,7 @@ use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; mod block_streamer; +mod delta_lake_client; mod indexer_config; mod redis; mod rules; From aed29b472c1fc917cf2ff79aa1d08f083d5c6a33 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 14:55:43 +1300 Subject: [PATCH 21/40] feat: Fetch latest_block metadata from delta lake --- block-streamer/src/delta_lake_client.rs | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index c2e1598aa..57c777172 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -1,5 +1,18 @@ use anyhow::Context; +pub const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; +pub const LATEST_BLOCK_METADATA_KEY: &str = + "silver/accounts/action_receipt_actions/metadata/latest_block.json"; + +#[derive(serde::Deserialize, Debug, Eq, PartialEq)] +pub struct LatestBlockMetadata { + last_indexed_block: String, + first_indexed_block: String, + last_indexed_block_date: String, + first_indexed_block_date: String, + processed_at_utc: String, +} + pub struct DeltaLakeClient where T: crate::s3_client::S3ClientTrait, @@ -14,4 +27,45 @@ where pub fn new(s3_client: T) -> Self { DeltaLakeClient { s3_client } } + + pub async fn get_latest_block_metadata(&self) -> anyhow::Result { + let metadata_file_content = self + .s3_client + .get_text_file(DELTA_LAKE_BUCKET, LATEST_BLOCK_METADATA_KEY) + .await?; + + serde_json::from_str::(&metadata_file_content) + .context("Unable to parse Metadata") + } +} + +#[cfg(test)] +mod test { + use super::*; + use mockall::predicate::*; + + #[tokio::test] + async fn fetches_metadata_from_s3() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .with(eq(DELTA_LAKE_BUCKET.to_string()), eq(LATEST_BLOCK_METADATA_KEY.to_string())) + .returning(|_bucket, _prefix| Box::pin(async move { Ok("{ \"last_indexed_block\": \"106309326\", \"first_indexed_block\": \"106164983\", \"last_indexed_block_date\": \"2023-11-22\", \"first_indexed_block_date\": \"2023-11-21\", \"processed_at_utc\": \"2023-11-22 23:06:24.358000\" }".to_string()) })); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await.unwrap(); + + assert_eq!( + latest_block_metadata, + LatestBlockMetadata { + last_indexed_block: "106309326".to_string(), + first_indexed_block: "106164983".to_string(), + last_indexed_block_date: "2023-11-22".to_string(), + first_indexed_block_date: "2023-11-21".to_string(), + processed_at_utc: "2023-11-22 23:06:24.358000".to_string(), + } + ) + } } From 38a1d6a0eb08584e796011e5384779fe4bf0d7b6 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 15:01:55 +1300 Subject: [PATCH 22/40] refactor: Use `DeltaLakeClient` to get metadata for block stream --- block-streamer/src/block_streamer.rs | 32 ++++--------------------- block-streamer/src/delta_lake_client.rs | 10 ++++---- block-streamer/src/main.rs | 5 ++++ 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 8140091b0..31c47aaa7 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -8,7 +8,6 @@ use chrono::{DateTime, LocalResult, TimeZone, Utc}; use near_jsonrpc_client::JsonRpcClient; use near_jsonrpc_primitives::types::blocks::RpcBlockRequest; use near_lake_framework::near_indexer_primitives::types::{BlockHeight, BlockId, BlockReference}; -use serde_json::from_str; use tokio::task::JoinHandle; pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; @@ -35,6 +34,7 @@ impl BlockStreamer { s3_client: S3Client, chain_id: ChainId, json_rpc_client: JsonRpcClient, + delta_lake_client: crate::delta_lake_client::DeltaLakeClient, ) -> anyhow::Result<()> { if self.task.is_some() { return Err(anyhow::anyhow!("BlockStreamer has already been started",)); @@ -60,6 +60,7 @@ impl BlockStreamer { &s3_client, &chain_id, &json_rpc_client, + &delta_lake_client, ) => { result.map_err(|err| { tracing::error!( @@ -106,6 +107,7 @@ pub(crate) async fn start_block_stream( s3_client: &S3Client, chain_id: &ChainId, json_rpc_client: &JsonRpcClient, + delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, ) -> anyhow::Result<()> { tracing::info!( "Starting block stream from {start_block_height} for indexer: {}", @@ -115,7 +117,8 @@ pub(crate) async fn start_block_stream( let start_date = lookup_block_date_or_next_block_date(start_block_height, json_rpc_client).await?; - let last_indexed_block = last_indexed_block_from_metadata(s3_client).await?; + let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await?; + let last_indexed_block = latest_block_metadata.last_indexed_block.parse::()?; let blocks_from_index = filter_matching_blocks_from_index_files( start_block_height, @@ -197,31 +200,6 @@ pub(crate) async fn start_block_stream( Ok(()) } -pub(crate) async fn last_indexed_block_from_metadata( - s3_client: &S3Client, -) -> anyhow::Result { - let key = format!( - "{}/{}", - s3::INDEXED_ACTIONS_FILES_FOLDER, - "latest_block.json" - ); - let metadata = - s3::fetch_text_file_from_s3(s3::INDEXED_DATA_FILES_BUCKET, key, s3_client).await?; - - let metadata: serde_json::Value = serde_json::from_str(&metadata).unwrap(); - let last_indexed_block = metadata["last_indexed_block"].clone(); - let last_indexed_block = last_indexed_block - .as_str() - .context("No last_indexed_block found in latest_block.json")?; - let last_indexed_block = - from_str(last_indexed_block).context("last_indexed_block couldn't be converted to u64")?; - tracing::info!( - "Last indexed block from latest_block.json: {:?}", - last_indexed_block - ); - Ok(last_indexed_block) -} - pub(crate) async fn filter_matching_blocks_from_index_files( start_block_height: BlockHeight, indexer: &IndexerConfig, diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 57c777172..d1c049d0a 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -6,11 +6,11 @@ pub const LATEST_BLOCK_METADATA_KEY: &str = #[derive(serde::Deserialize, Debug, Eq, PartialEq)] pub struct LatestBlockMetadata { - last_indexed_block: String, - first_indexed_block: String, - last_indexed_block_date: String, - first_indexed_block_date: String, - processed_at_utc: String, + pub last_indexed_block: String, + pub first_indexed_block: String, + pub last_indexed_block_date: String, + pub first_indexed_block_date: String, + pub processed_at_utc: String, } pub struct DeltaLakeClient diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 8a1a9e98e..ace06c664 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -30,6 +30,10 @@ async fn main() -> anyhow::Result<()> { let aws_config = aws_config::from_env().load().await; let s3_client = aws_sdk_s3::Client::new(&aws_config); + let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new( + crate::s3_client::S3Client::new(&aws_config), + ); + let contract = "queryapi.dataplatform.near"; let matching_rule = MatchingRule::ActionAny { affected_account_id: contract.to_string(), @@ -60,6 +64,7 @@ async fn main() -> anyhow::Result<()> { s3_client, ChainId::Mainnet, json_rpc_client, + delta_lake_client, )?; streamer.take_handle().unwrap().await??; From 3409801adf276aae45e9d3b56171f00a3f199145 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 15:20:52 +1300 Subject: [PATCH 23/40] feat: Add `list_objects` to `s3_client` --- block-streamer/src/s3_client.rs | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/block-streamer/src/s3_client.rs b/block-streamer/src/s3_client.rs index 2a6dfa9f0..c16f21dc7 100644 --- a/block-streamer/src/s3_client.rs +++ b/block-streamer/src/s3_client.rs @@ -10,6 +10,16 @@ pub trait S3ClientTrait { aws_sdk_s3::error::SdkError, >; + async fn list_objects( + &self, + bucket: &str, + prefix: &str, + continuation_token: Option, + ) -> Result< + aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output, + aws_sdk_s3::error::SdkError, + >; + async fn get_text_file(&self, bucket: &str, prefix: &str) -> anyhow::Result; } @@ -40,11 +50,33 @@ impl S3ClientTrait for S3Client { .get_object() .bucket(bucket) .key(prefix) - .request_payer(aws_sdk_s3::types::RequestPayer::Requester) .send() .await } + async fn list_objects( + &self, + bucket: &str, + prefix: &str, + continuation_token: Option, + ) -> Result< + aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output, + aws_sdk_s3::error::SdkError, + > { + let mut builder = self + .client + .list_objects_v2() + .delimiter("/") + .bucket(bucket) + .prefix(prefix); + + if let Some(token) = continuation_token { + builder = builder.continuation_token(token); + } + + builder.send().await + } + async fn get_text_file(&self, bucket: &str, prefix: &str) -> anyhow::Result { let object = self.get_object(bucket, prefix).await?; From 82300c5fa64768d0fda53d003a1a2e0269ffcf68 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 15:25:33 +1300 Subject: [PATCH 24/40] feat: Move s3 logic to `DeltaLakeClient` --- block-streamer/src/delta_lake_client.rs | 160 ++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index d1c049d0a..bcde1dbc4 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -1,5 +1,8 @@ use anyhow::Context; +use chrono::{DateTime, NaiveDate, Utc}; +use futures::future::try_join_all; +const MAX_S3_LIST_REQUESTS: usize = 1000; pub const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; pub const LATEST_BLOCK_METADATA_KEY: &str = "silver/accounts/action_receipt_actions/metadata/latest_block.json"; @@ -37,6 +40,163 @@ where serde_json::from_str::(&metadata_file_content) .context("Unable to parse Metadata") } + + async fn list_s3_bucket_by_prefix( + &self, + s3_bucket: &str, + s3_prefix: &str, + ) -> anyhow::Result> { + let mut results = vec![]; + let mut continuation_token: Option = None; + + let mut counter = 0; + loop { + let file_list = match continuation_token { + Some(token) => self + .s3_client + .list_objects(s3_bucket, s3_prefix, Some(token)), + None => self.s3_client.list_objects(s3_bucket, s3_prefix, None), + } + .await?; + + if let Some(common_prefixes) = file_list.common_prefixes { + let keys: Vec = common_prefixes + .into_iter() + .map(|o| o.prefix.unwrap()) + .collect(); + results.extend(keys); + } + if let Some(objects) = file_list.contents { + let keys: Vec = objects.into_iter().map(|o| o.key.unwrap()).collect(); + results.extend(keys); + } + if file_list.next_continuation_token.is_some() { + continuation_token = file_list.next_continuation_token; + counter += 1; + if counter > MAX_S3_LIST_REQUESTS { + anyhow::bail!("Exceeded internal limit of {MAX_S3_LIST_REQUESTS}") + } + } else { + break; + } + } + Ok(results) + } + + fn storage_path_for_account(&self, account: &str) -> String { + let mut folders = account.split('.').collect::>(); + folders.reverse(); + folders.join("/") + } + + async fn list_index_files_by_wildcard( + &self, + s3_bucket: &str, + s3_folder: &str, + pattern: &&str, + ) -> anyhow::Result> { + // remove sub-account wildcard from pattern + let pattern = pattern.replace("*.", ""); + let path = self.storage_path_for_account(&pattern); + + let folders = self + .list_s3_bucket_by_prefix(s3_bucket, &format!("{}/{}/", s3_folder, path)) + .await?; + // for each matching folder list files + let mut results = vec![]; + for folder in folders { + results.extend(self.list_s3_bucket_by_prefix(s3_bucket, &folder).await?); + } + Ok(results) + } + + pub async fn find_index_files_by_pattern( + &self, + s3_bucket: &str, + s3_folder: &str, + pattern: &str, + ) -> anyhow::Result> { + Ok(match pattern { + x if x.contains(',') => { + let account_array = x.split(','); + let mut results = vec![]; + for account in account_array { + let account = account.trim(); + let sub_results = if account.contains('*') { + self.list_index_files_by_wildcard(s3_bucket, s3_folder, &account) + .await? + } else { + self.list_s3_bucket_by_prefix( + s3_bucket, + &format!("{}/{}/", s3_folder, self.storage_path_for_account(account)), + ) + .await? + }; + results.extend(sub_results); + } + results + } + x if x.contains('*') => { + self.list_index_files_by_wildcard(s3_bucket, s3_folder, &x) + .await? + } + _ => { + self.list_s3_bucket_by_prefix( + s3_bucket, + &format!("{}/{}/", s3_folder, self.storage_path_for_account(pattern),), + ) + .await? + } + }) + } + + fn file_name_date_after(&self, start_date: DateTime, file_name: &str) -> bool { + let file_name_date = file_name.split('/').last().unwrap().replace(".json", ""); + let file_name_date = NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d"); + match file_name_date { + Ok(file_name_date) => file_name_date >= start_date.date_naive(), + Err(e) => { + // if we can't parse the date assume a file this code is not meant to handle + tracing::debug!( + target: crate::LOG_TARGET, + "Error parsing file name date: {:?}", + e + ); + false + } + } + } + + pub async fn fetch_contract_index_files( + &self, + s3_bucket: &str, + s3_folder: &str, + start_date: DateTime, + contract_pattern: &str, + ) -> anyhow::Result> { + // list all index files + let file_list = self + .find_index_files_by_pattern(s3_bucket, s3_folder, contract_pattern) + .await?; + + let fetch_and_parse_tasks = file_list + .into_iter() + .filter(|index_file_listing| self.file_name_date_after(start_date, index_file_listing)) + .map(|key| { + async move { + // Fetch the file + self.s3_client.get_text_file(s3_bucket, &key).await + } + }) + .collect::>(); + + // Execute all tasks in parallel and wait for completion + let file_contents: Vec = try_join_all(fetch_and_parse_tasks).await?; + Ok(file_contents + .into_iter() + .filter(|file_contents| !file_contents.is_empty()) + .collect::>()) + } } #[cfg(test)] From 1f652d1bfb727702d402ccab4151bed577d9fbd1 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 15:26:11 +1300 Subject: [PATCH 25/40] refactor: Use `DeltaLakeClient` to fetch index files --- block-streamer/src/block_streamer.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 31c47aaa7..7b459dd2f 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -123,7 +123,7 @@ pub(crate) async fn start_block_stream( let blocks_from_index = filter_matching_blocks_from_index_files( start_block_height, &indexer, - s3_client, + delta_lake_client, start_date, ) .await?; @@ -203,7 +203,7 @@ pub(crate) async fn start_block_stream( pub(crate) async fn filter_matching_blocks_from_index_files( start_block_height: BlockHeight, indexer: &IndexerConfig, - s3_client: &S3Client, + delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, start_date: DateTime, ) -> anyhow::Result> { let s3_bucket = s3::INDEXED_DATA_FILES_BUCKET; @@ -219,14 +219,14 @@ pub(crate) async fn filter_matching_blocks_from_index_files( if affected_account_id.contains('*') || affected_account_id.contains(',') { needs_dedupe_and_sort = true; } - s3::fetch_contract_index_files( - s3_client, - s3_bucket, - s3::INDEXED_ACTIONS_FILES_FOLDER, - start_date, - affected_account_id, - ) - .await + delta_lake_client + .fetch_contract_index_files( + s3_bucket, + s3::INDEXED_ACTIONS_FILES_FOLDER, + start_date, + affected_account_id, + ) + .await } MatchingRule::ActionFunctionCall { .. } => { bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer.account_id, indexer.function_name); From 95dd389f5ef7f6ef200d5059822b56cd0ce102ce Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 19:59:43 +1300 Subject: [PATCH 26/40] refactor: Inline use of bucket/prefix --- block-streamer/src/block_streamer.rs | 9 +- block-streamer/src/delta_lake_client.rs | 108 ++++++++++++------------ 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 7b459dd2f..776592173 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -206,8 +206,6 @@ pub(crate) async fn filter_matching_blocks_from_index_files( delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, start_date: DateTime, ) -> anyhow::Result> { - let s3_bucket = s3::INDEXED_DATA_FILES_BUCKET; - let mut needs_dedupe_and_sort = false; let indexer_rule = &indexer.indexer_rule; @@ -220,12 +218,7 @@ pub(crate) async fn filter_matching_blocks_from_index_files( needs_dedupe_and_sort = true; } delta_lake_client - .fetch_contract_index_files( - s3_bucket, - s3::INDEXED_ACTIONS_FILES_FOLDER, - start_date, - affected_account_id, - ) + .list_matching_block_heights(start_date, affected_account_id) .await } MatchingRule::ActionFunctionCall { .. } => { diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index bcde1dbc4..877ffe296 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -3,8 +3,9 @@ use chrono::{DateTime, NaiveDate, Utc}; use futures::future::try_join_all; const MAX_S3_LIST_REQUESTS: usize = 1000; -pub const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; -pub const LATEST_BLOCK_METADATA_KEY: &str = +const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; +const INDEXED_ACTIONS_PREFIX: &str = "silver/accounts/action_receipt_actions/metadata"; +const LATEST_BLOCK_METADATA_KEY: &str = "silver/accounts/action_receipt_actions/metadata/latest_block.json"; #[derive(serde::Deserialize, Debug, Eq, PartialEq)] @@ -41,21 +42,20 @@ where .context("Unable to parse Metadata") } - async fn list_s3_bucket_by_prefix( - &self, - s3_bucket: &str, - s3_prefix: &str, - ) -> anyhow::Result> { + async fn list_s3_bucket_by_prefix(&self, s3_prefix: &str) -> anyhow::Result> { let mut results = vec![]; let mut continuation_token: Option = None; let mut counter = 0; loop { let file_list = match continuation_token { - Some(token) => self + Some(token) => { + self.s3_client + .list_objects(DELTA_LAKE_BUCKET, s3_prefix, Some(token)) + } + None => self .s3_client - .list_objects(s3_bucket, s3_prefix, Some(token)), - None => self.s3_client.list_objects(s3_bucket, s3_prefix, None), + .list_objects(DELTA_LAKE_BUCKET, s3_prefix, None), } .await?; @@ -66,10 +66,12 @@ where .collect(); results.extend(keys); } + if let Some(objects) = file_list.contents { let keys: Vec = objects.into_iter().map(|o| o.key.unwrap()).collect(); results.extend(keys); } + if file_list.next_continuation_token.is_some() { continuation_token = file_list.next_continuation_token; counter += 1; @@ -89,65 +91,61 @@ where folders.join("/") } - async fn list_index_files_by_wildcard( - &self, - s3_bucket: &str, - s3_folder: &str, - pattern: &&str, - ) -> anyhow::Result> { + async fn list_index_files_by_wildcard(&self, pattern: &&str) -> anyhow::Result> { // remove sub-account wildcard from pattern let pattern = pattern.replace("*.", ""); let path = self.storage_path_for_account(&pattern); let folders = self - .list_s3_bucket_by_prefix(s3_bucket, &format!("{}/{}/", s3_folder, path)) + .list_s3_bucket_by_prefix(&format!("{}/{}/", INDEXED_ACTIONS_PREFIX, path)) .await?; // for each matching folder list files let mut results = vec![]; for folder in folders { - results.extend(self.list_s3_bucket_by_prefix(s3_bucket, &folder).await?); + results.extend(self.list_s3_bucket_by_prefix(&folder).await?); } Ok(results) } - pub async fn find_index_files_by_pattern( + pub async fn list_matching_index_files( &self, - s3_bucket: &str, - s3_folder: &str, - pattern: &str, + contract_pattern: &str, ) -> anyhow::Result> { - Ok(match pattern { - x if x.contains(',') => { - let account_array = x.split(','); + match contract_pattern { + pattern if pattern.contains(',') => { + let accounts = pattern.split(','); + let mut results = vec![]; - for account in account_array { + + for account in accounts { let account = account.trim(); - let sub_results = if account.contains('*') { - self.list_index_files_by_wildcard(s3_bucket, s3_folder, &account) - .await? + + if account.contains('*') { + results.extend(self.list_index_files_by_wildcard(&account).await?); } else { - self.list_s3_bucket_by_prefix( - s3_bucket, - &format!("{}/{}/", s3_folder, self.storage_path_for_account(account)), - ) - .await? + results.extend( + self.list_s3_bucket_by_prefix(&format!( + "{}/{}/", + INDEXED_ACTIONS_PREFIX, + self.storage_path_for_account(account) + )) + .await?, + ); }; - results.extend(sub_results); } - results - } - x if x.contains('*') => { - self.list_index_files_by_wildcard(s3_bucket, s3_folder, &x) - .await? + + Ok(results) } - _ => { - self.list_s3_bucket_by_prefix( - s3_bucket, - &format!("{}/{}/", s3_folder, self.storage_path_for_account(pattern),), - ) - .await? + pattern if pattern.contains('*') => self.list_index_files_by_wildcard(&pattern).await, + pattern => { + self.list_s3_bucket_by_prefix(&format!( + "{}/{}/", + INDEXED_ACTIONS_PREFIX, + self.storage_path_for_account(pattern), + )) + .await } - }) + } } fn file_name_date_after(&self, start_date: DateTime, file_name: &str) -> bool { @@ -167,17 +165,17 @@ where } } - pub async fn fetch_contract_index_files( + pub async fn list_matching_block_heights( &self, - s3_bucket: &str, - s3_folder: &str, start_date: DateTime, contract_pattern: &str, ) -> anyhow::Result> { - // list all index files - let file_list = self - .find_index_files_by_pattern(s3_bucket, s3_folder, contract_pattern) - .await?; + let file_list = self.list_matching_index_files(contract_pattern).await?; + tracing::debug!( + "Found {} index files matching {}", + file_list.len(), + contract_pattern, + ); let fetch_and_parse_tasks = file_list .into_iter() @@ -185,7 +183,7 @@ where .map(|key| { async move { // Fetch the file - self.s3_client.get_text_file(s3_bucket, &key).await + self.s3_client.get_text_file(DELTA_LAKE_BUCKET, &key).await } }) .collect::>(); From 3c1838f96414550033383aad54accc8d700393e5 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Thu, 23 Nov 2023 20:10:02 +1300 Subject: [PATCH 27/40] refactor: Simplify use of `continuation_token` --- block-streamer/src/delta_lake_client.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 877ffe296..c705924a4 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -48,16 +48,10 @@ where let mut counter = 0; loop { - let file_list = match continuation_token { - Some(token) => { - self.s3_client - .list_objects(DELTA_LAKE_BUCKET, s3_prefix, Some(token)) - } - None => self - .s3_client - .list_objects(DELTA_LAKE_BUCKET, s3_prefix, None), - } - .await?; + let file_list = self + .s3_client + .list_objects(DELTA_LAKE_BUCKET, s3_prefix, continuation_token) + .await?; if let Some(common_prefixes) = file_list.common_prefixes { let keys: Vec = common_prefixes @@ -82,6 +76,7 @@ where break; } } + Ok(results) } @@ -189,8 +184,9 @@ where .collect::>(); // Execute all tasks in parallel and wait for completion - let file_contents: Vec = try_join_all(fetch_and_parse_tasks).await?; - Ok(file_contents + let file_content_list = try_join_all(fetch_and_parse_tasks).await?; + + Ok(file_content_list .into_iter() .filter(|file_contents| !file_contents.is_empty()) .collect::>()) From 1287b7cf663aebfdc8a606973c8db56f6119ad86 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 09:29:03 +1300 Subject: [PATCH 28/40] refactor: Extract S3 listing from `DeltaLakeClient` --- block-streamer/src/delta_lake_client.rs | 86 +++++++++---------------- block-streamer/src/s3_client.rs | 47 ++++++++++++++ 2 files changed, 79 insertions(+), 54 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index c705924a4..1b50905b7 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -42,44 +42,6 @@ where .context("Unable to parse Metadata") } - async fn list_s3_bucket_by_prefix(&self, s3_prefix: &str) -> anyhow::Result> { - let mut results = vec![]; - let mut continuation_token: Option = None; - - let mut counter = 0; - loop { - let file_list = self - .s3_client - .list_objects(DELTA_LAKE_BUCKET, s3_prefix, continuation_token) - .await?; - - if let Some(common_prefixes) = file_list.common_prefixes { - let keys: Vec = common_prefixes - .into_iter() - .map(|o| o.prefix.unwrap()) - .collect(); - results.extend(keys); - } - - if let Some(objects) = file_list.contents { - let keys: Vec = objects.into_iter().map(|o| o.key.unwrap()).collect(); - results.extend(keys); - } - - if file_list.next_continuation_token.is_some() { - continuation_token = file_list.next_continuation_token; - counter += 1; - if counter > MAX_S3_LIST_REQUESTS { - anyhow::bail!("Exceeded internal limit of {MAX_S3_LIST_REQUESTS}") - } - } else { - break; - } - } - - Ok(results) - } - fn storage_path_for_account(&self, account: &str) -> String { let mut folders = account.split('.').collect::>(); folders.reverse(); @@ -92,12 +54,20 @@ where let path = self.storage_path_for_account(&pattern); let folders = self - .list_s3_bucket_by_prefix(&format!("{}/{}/", INDEXED_ACTIONS_PREFIX, path)) + .s3_client + .list_all_objects( + DELTA_LAKE_BUCKET, + &format!("{}/{}/", INDEXED_ACTIONS_PREFIX, path), + ) .await?; // for each matching folder list files let mut results = vec![]; for folder in folders { - results.extend(self.list_s3_bucket_by_prefix(&folder).await?); + results.extend( + self.s3_client + .list_all_objects(DELTA_LAKE_BUCKET, &folder) + .await?, + ); } Ok(results) } @@ -119,12 +89,16 @@ where results.extend(self.list_index_files_by_wildcard(&account).await?); } else { results.extend( - self.list_s3_bucket_by_prefix(&format!( - "{}/{}/", - INDEXED_ACTIONS_PREFIX, - self.storage_path_for_account(account) - )) - .await?, + self.s3_client + .list_all_objects( + DELTA_LAKE_BUCKET, + &format!( + "{}/{}/", + INDEXED_ACTIONS_PREFIX, + self.storage_path_for_account(account) + ), + ) + .await?, ); }; } @@ -133,12 +107,16 @@ where } pattern if pattern.contains('*') => self.list_index_files_by_wildcard(&pattern).await, pattern => { - self.list_s3_bucket_by_prefix(&format!( - "{}/{}/", - INDEXED_ACTIONS_PREFIX, - self.storage_path_for_account(pattern), - )) - .await + self.s3_client + .list_all_objects( + DELTA_LAKE_BUCKET, + &format!( + "{}/{}/", + INDEXED_ACTIONS_PREFIX, + self.storage_path_for_account(pattern), + ), + ) + .await } } } @@ -194,7 +172,7 @@ where } #[cfg(test)] -mod test { +mod tests { use super::*; use mockall::predicate::*; @@ -204,7 +182,7 @@ mod test { mock_s3_client .expect_get_text_file() - .with(eq(DELTA_LAKE_BUCKET.to_string()), eq(LATEST_BLOCK_METADATA_KEY.to_string())) + .with(eq(DELTA_LAKE_BUCKET), eq(LATEST_BLOCK_METADATA_KEY)) .returning(|_bucket, _prefix| Box::pin(async move { Ok("{ \"last_indexed_block\": \"106309326\", \"first_indexed_block\": \"106164983\", \"last_indexed_block_date\": \"2023-11-22\", \"first_indexed_block_date\": \"2023-11-21\", \"processed_at_utc\": \"2023-11-22 23:06:24.358000\" }".to_string()) })); let delta_lake_client = DeltaLakeClient::new(mock_s3_client); diff --git a/block-streamer/src/s3_client.rs b/block-streamer/src/s3_client.rs index c16f21dc7..8234b40e5 100644 --- a/block-streamer/src/s3_client.rs +++ b/block-streamer/src/s3_client.rs @@ -1,3 +1,5 @@ +const MAX_S3_LIST_REQUESTS: usize = 1000; + #[async_trait::async_trait] #[mockall::automock] pub trait S3ClientTrait { @@ -21,6 +23,8 @@ pub trait S3ClientTrait { >; async fn get_text_file(&self, bucket: &str, prefix: &str) -> anyhow::Result; + + async fn list_all_objects(&self, bucket: &str, prefix: &str) -> anyhow::Result>; } #[derive(Clone, Debug)] @@ -84,4 +88,47 @@ impl S3ClientTrait for S3Client { Ok(String::from_utf8(bytes.to_vec())?) } + + async fn list_all_objects(&self, bucket: &str, prefix: &str) -> anyhow::Result> { + let mut results = vec![]; + let mut continuation_token: Option = None; + + let mut counter = 0; + loop { + if counter > MAX_S3_LIST_REQUESTS { + anyhow::bail!("Exceeded internal limit of {MAX_S3_LIST_REQUESTS}") + } + + let list = self + .list_objects(bucket, prefix, continuation_token) + .await?; + + if let Some(common_prefixes) = list.common_prefixes { + let keys: Vec = common_prefixes + .into_iter() + .filter_map(|common_prefix| common_prefix.prefix) + .collect(); + + results.extend(keys); + } + + if let Some(objects) = list.contents { + let keys: Vec = objects + .into_iter() + .filter_map(|object| object.key) + .collect(); + + results.extend(keys); + } + + if list.next_continuation_token.is_some() { + continuation_token = list.next_continuation_token; + counter += 1; + } else { + break; + } + } + + Ok(results) + } } From e48ba77ae86a16dd9691b73ddbd32ef061683ca0 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 10:54:30 +1300 Subject: [PATCH 29/40] test: Test `DeltaLakeClient` listing multi/wildcard --- block-streamer/src/delta_lake_client.rs | 212 +++++++++++++++++++++++- block-streamer/src/s3_client.rs | 2 +- 2 files changed, 210 insertions(+), 4 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 1b50905b7..e949e2a11 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -174,7 +174,7 @@ where #[cfg(test)] mod tests { use super::*; - use mockall::predicate::*; + use mockall::predicate; #[tokio::test] async fn fetches_metadata_from_s3() { @@ -182,8 +182,8 @@ mod tests { mock_s3_client .expect_get_text_file() - .with(eq(DELTA_LAKE_BUCKET), eq(LATEST_BLOCK_METADATA_KEY)) - .returning(|_bucket, _prefix| Box::pin(async move { Ok("{ \"last_indexed_block\": \"106309326\", \"first_indexed_block\": \"106164983\", \"last_indexed_block_date\": \"2023-11-22\", \"first_indexed_block_date\": \"2023-11-21\", \"processed_at_utc\": \"2023-11-22 23:06:24.358000\" }".to_string()) })); + .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq(LATEST_BLOCK_METADATA_KEY)) + .returning(|_bucket, _prefix| Ok("{ \"last_indexed_block\": \"106309326\", \"first_indexed_block\": \"106164983\", \"last_indexed_block_date\": \"2023-11-22\", \"first_indexed_block_date\": \"2023-11-21\", \"processed_at_utc\": \"2023-11-22 23:06:24.358000\" }".to_string())); let delta_lake_client = DeltaLakeClient::new(mock_s3_client); @@ -200,4 +200,210 @@ mod tests { } ) } + + #[tokio::test] + async fn lists_block_heights_for_single_account() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_list_all_objects() + .returning(|_bucket, _prefix| { + Ok(vec![ + "silver/accounts/action_receipt_actions/metadata/near/dataplatform/queryapi/2023-05-15.json".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/dataplatform/queryapi/2023-05-17.json".to_string(), + ]) + }); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/dataplatform/queryapi/2023-05-15.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[91940840,91942989],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[91942989,91940840]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/dataplatform/queryapi/2023-05-17.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[92080299,92080344],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92080344,92080299]}]}".to_string())); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_heights = delta_lake_client + .list_matching_block_heights( + NaiveDate::from_ymd_opt(2023, 5, 16) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(), + "queryapi.dataplatform.near", + ) + .await + .unwrap(); + + assert_eq!( + block_heights, + vec![ + "{\"heights\":[92080299,92080344],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92080344,92080299]}]}" + ] + ) + } + + #[tokio::test] + async fn lists_block_heights_for_multiple_accounts() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_list_all_objects() + // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case + // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) + .returning(|_bucket, prefix| { + let objects = match prefix { + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string() + ], + "silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/2023-05-18.json".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/2023-05-30.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/sputnik-dao/hackathon/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/sputnik-dao/hackathon/2022-05-27.json".to_string() + ], + _ => panic!("Unexpected prefix: {}", prefix) + }; + + Ok(objects) + }); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[45894617,45894627,45894628,45894712,45898413,45898423,45898424],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[45898423,45894627]},{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[45894712]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[45894628,45894617,45898424,45898413]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[45898423,45894627]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/2023-05-18.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[92167977,92168200,92168293,92168338,92168535,92168870,92168871,92168922,92168923,92168939,92168971,92169330],\"actions\":[{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[92168200,92168338]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[92168535,92167977]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[92167977]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92168922,92168971,92168870]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[92168871,92168923,92169330,92168293,92168939,92167977]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/2023-05-30.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[93065811,93067570,93067619,93067631,93067726,93067737,93067770,93067889,93067920,93067926,93067936,93073935,93073944,93073954],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[93073954,93067770,93067726,93065811,93067619,93073935,93067889,93067737,93067570,93067926,93073944,93067920,93067631,93067936]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/sputnik-dao/hackathon/2022-05-27.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[66494954],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[66494954]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[66494954]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[66494954]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[66494954]}]}".to_string())); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_heights = delta_lake_client + .list_matching_block_heights( + NaiveDate::from_ymd_opt(2022, 5, 26) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(), + "hackathon.agency.near, hackathon.aurora-silo-dev.near, hackathon.sputnik-dao.near", + ) + .await + .unwrap(); + + assert_eq!( + block_heights, + vec![ + "{\"heights\":[92167977,92168200,92168293,92168338,92168535,92168870,92168871,92168922,92168923,92168939,92168971,92169330],\"actions\":[{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[92168200,92168338]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[92168535,92167977]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[92167977]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92168922,92168971,92168870]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[92168871,92168923,92169330,92168293,92168939,92167977]}]}", + "{\"heights\":[93065811,93067570,93067619,93067631,93067726,93067737,93067770,93067889,93067920,93067926,93067936,93073935,93073944,93073954],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[93073954,93067770,93067726,93065811,93067619,93073935,93067889,93067737,93067570,93067926,93073944,93067920,93067631,93067936]}]}", + "{\"heights\":[66494954],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[66494954]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[66494954]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[66494954]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[66494954]}]}" + ] + ) + } + + #[tokio::test] + async fn lists_block_heights_for_wildcard() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_list_all_objects() + // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case + // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) + .returning(|_bucket, prefix| { + let objects = match prefix { + "silver/accounts/action_receipt_actions/metadata/near/keypom/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/keypom/nft/".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-23.json".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/nft/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/nft/2023-09-26.json".to_string(), + ], + + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-23.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-23.json".to_string() + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string() + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15json".to_string() + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/nft/2023-09-26.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/nft/2023-09-26.json".to_string() + ], + _ => panic!("Unexpected prefix: {}", prefix) + }; + + Ok(objects) + }); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[78516467,78516476,78516489,78516511,78516512],\"actions\":[{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[78516467]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[78516476]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[78516476,78516512]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[78516476]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[78516511]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[78516489]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/nft/2023-09-26.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[102025554],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[102025554]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-23.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[104045849,104047967,104047968],\"actions\":[{\"action_kind\":\"TRANSFER\",\"block_heights\":[104047968,104045849,104047967]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}".to_string())); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_heights = delta_lake_client + .list_matching_block_heights( + NaiveDate::from_ymd_opt(2023, 5, 26) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(), + "*.keypom.near", + ) + .await + .unwrap(); + + assert_eq!( + block_heights, + vec![ + "{\"heights\":[102025554],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[102025554]}]}", + "{\"heights\":[104045849,104047967,104047968],\"actions\":[{\"action_kind\":\"TRANSFER\",\"block_heights\":[104047968,104045849,104047967]}]}", + "{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}" + ] + ) + } + + // #[tokio::test] + // #[ignore] + // async fn list_with_csv_and_wildcard_contracts() { + // let aws_config = aws_config::from_env().load().await; + // let s3_client = crate::s3_client::S3Client::new(&aws_config); + // + // let delta_lake_client = DeltaLakeClient::new(s3_client); + // + // let list = delta_lake_client + // .list_matching_index_files("*.keypom.near, hackathon.agency.near, *.nearcrowd.near") + // .await + // .unwrap(); + // + // assert!(list.len() >= 1370); + // } } diff --git a/block-streamer/src/s3_client.rs b/block-streamer/src/s3_client.rs index 8234b40e5..f3c72e436 100644 --- a/block-streamer/src/s3_client.rs +++ b/block-streamer/src/s3_client.rs @@ -1,7 +1,7 @@ const MAX_S3_LIST_REQUESTS: usize = 1000; -#[async_trait::async_trait] #[mockall::automock] +#[async_trait::async_trait] pub trait S3ClientTrait { async fn get_object( &self, From cb318fcb7eb14380921270798763bd3a40066fd4 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 11:34:48 +1300 Subject: [PATCH 30/40] refactor: Rename functions to add clarity --- block-streamer/src/delta_lake_client.rs | 100 ++++++++++-------------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index e949e2a11..f28048082 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -2,7 +2,6 @@ use anyhow::Context; use chrono::{DateTime, NaiveDate, Utc}; use futures::future::try_join_all; -const MAX_S3_LIST_REQUESTS: usize = 1000; const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; const INDEXED_ACTIONS_PREFIX: &str = "silver/accounts/action_receipt_actions/metadata"; const LATEST_BLOCK_METADATA_KEY: &str = @@ -42,33 +41,36 @@ where .context("Unable to parse Metadata") } - fn storage_path_for_account(&self, account: &str) -> String { - let mut folders = account.split('.').collect::>(); + fn account_id_to_s3_prefix(&self, account: &str) -> String { + let mut folders = account.split('.').collect::>(); folders.reverse(); - folders.join("/") + + format!("{}/{}/", INDEXED_ACTIONS_PREFIX, folders.join("/")) } - async fn list_index_files_by_wildcard(&self, pattern: &&str) -> anyhow::Result> { - // remove sub-account wildcard from pattern - let pattern = pattern.replace("*.", ""); - let path = self.storage_path_for_account(&pattern); + async fn list_objects_recursive( + &self, + prefix: &str, + depth: u32, + ) -> anyhow::Result> { + if depth > 1 { + unimplemented!("Recursive list with depth > 1 not supported") + } - let folders = self + let objects = self .s3_client - .list_all_objects( - DELTA_LAKE_BUCKET, - &format!("{}/{}/", INDEXED_ACTIONS_PREFIX, path), - ) + .list_all_objects(DELTA_LAKE_BUCKET, prefix) .await?; - // for each matching folder list files + let mut results = vec![]; - for folder in folders { + for object in objects { results.extend( self.s3_client - .list_all_objects(DELTA_LAKE_BUCKET, &folder) + .list_all_objects(DELTA_LAKE_BUCKET, &object) .await?, ); } + Ok(results) } @@ -86,17 +88,17 @@ where let account = account.trim(); if account.contains('*') { - results.extend(self.list_index_files_by_wildcard(&account).await?); + let pattern = account.replace("*.", ""); + results.extend( + self.list_objects_recursive(&self.account_id_to_s3_prefix(&pattern), 1) + .await?, + ); } else { results.extend( self.s3_client .list_all_objects( DELTA_LAKE_BUCKET, - &format!( - "{}/{}/", - INDEXED_ACTIONS_PREFIX, - self.storage_path_for_account(account) - ), + &self.account_id_to_s3_prefix(account), ) .await?, ); @@ -105,37 +107,23 @@ where Ok(results) } - pattern if pattern.contains('*') => self.list_index_files_by_wildcard(&pattern).await, + pattern if pattern.contains('*') => { + let pattern = pattern.replace("*.", ""); + self.list_objects_recursive(&self.account_id_to_s3_prefix(&pattern), 1) + .await + } pattern => { self.s3_client - .list_all_objects( - DELTA_LAKE_BUCKET, - &format!( - "{}/{}/", - INDEXED_ACTIONS_PREFIX, - self.storage_path_for_account(pattern), - ), - ) + .list_all_objects(DELTA_LAKE_BUCKET, &self.account_id_to_s3_prefix(pattern)) .await } } } - fn file_name_date_after(&self, start_date: DateTime, file_name: &str) -> bool { - let file_name_date = file_name.split('/').last().unwrap().replace(".json", ""); - let file_name_date = NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d"); - match file_name_date { - Ok(file_name_date) => file_name_date >= start_date.date_naive(), - Err(e) => { - // if we can't parse the date assume a file this code is not meant to handle - tracing::debug!( - target: crate::LOG_TARGET, - "Error parsing file name date: {:?}", - e - ); - false - } - } + fn date_from_s3_path(&self, path: &str) -> Option { + let file_name_date = path.split('/').last()?.replace(".json", ""); + + NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d").ok() } pub async fn list_matching_block_heights( @@ -150,24 +138,22 @@ where contract_pattern, ); - let fetch_and_parse_tasks = file_list + let futures = file_list .into_iter() - .filter(|index_file_listing| self.file_name_date_after(start_date, index_file_listing)) - .map(|key| { - async move { - // Fetch the file - self.s3_client.get_text_file(DELTA_LAKE_BUCKET, &key).await - } + // TODO use `start_after` in the request to S3 to avoid this filter + .filter(|file_path| { + self.date_from_s3_path(file_path) + .map_or(false, |file_date| file_date >= start_date.date_naive()) }) + .map(|key| async move { self.s3_client.get_text_file(DELTA_LAKE_BUCKET, &key).await }) .collect::>(); - // Execute all tasks in parallel and wait for completion - let file_content_list = try_join_all(fetch_and_parse_tasks).await?; + let file_content_list = try_join_all(futures).await?; Ok(file_content_list .into_iter() - .filter(|file_contents| !file_contents.is_empty()) - .collect::>()) + .filter(|content| !content.is_empty()) + .collect()) } } From 5431708b00a46f41bae19e776d66bd25abb0c6a9 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 13:22:04 +1300 Subject: [PATCH 31/40] test: Listing index files for multiple acccounts and wildcard --- block-streamer/src/delta_lake_client.rs | 84 ++++++++++++++++++++----- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index f28048082..d6f0dc97f 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -143,6 +143,7 @@ where // TODO use `start_after` in the request to S3 to avoid this filter .filter(|file_path| { self.date_from_s3_path(file_path) + // Ignore invalid paths, i.e. sub-folders, by default .map_or(false, |file_date| file_date >= start_date.date_naive()) }) .map(|key| async move { self.s3_client.get_text_file(DELTA_LAKE_BUCKET, &key).await }) @@ -377,19 +378,72 @@ mod tests { ) } - // #[tokio::test] - // #[ignore] - // async fn list_with_csv_and_wildcard_contracts() { - // let aws_config = aws_config::from_env().load().await; - // let s3_client = crate::s3_client::S3Client::new(&aws_config); - // - // let delta_lake_client = DeltaLakeClient::new(s3_client); - // - // let list = delta_lake_client - // .list_matching_index_files("*.keypom.near, hackathon.agency.near, *.nearcrowd.near") - // .await - // .unwrap(); - // - // assert!(list.len() >= 1370); - // } + #[tokio::test] + async fn lists_block_heights_for_multiple_accounts_and_wildcard() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_list_all_objects() + // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case + // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) + .returning(|_bucket, prefix| { + let objects = match prefix { + "silver/accounts/action_receipt_actions/metadata/near/keypom/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/".to_string(), + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string() + ], + + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string() + ], + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15json".to_string() + ], + _ => panic!("Unexpected prefix: {}", prefix) + }; + + Ok(objects) + }); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[45894617,45894627,45894628,45894712,45898413,45898423,45898424],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[45898423,45894627]},{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[45894712]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[45894628,45894617,45898424,45898413]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[45898423,45894627]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/beta/2022-11-15.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[78516467,78516476,78516489,78516511,78516512],\"actions\":[{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[78516467]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[78516476]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[78516476,78516512]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[78516476]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[78516511]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[78516489]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}".to_string())); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_heights = delta_lake_client + .list_matching_block_heights( + NaiveDate::from_ymd_opt(2021, 5, 26) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(), + "*.keypom.near, hackathon.agency.near", + ) + .await + .unwrap(); + + assert_eq!( + block_heights, + vec![ + "{\"heights\":[78516467,78516476,78516489,78516511,78516512],\"actions\":[{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[78516467]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[78516476]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[78516476,78516512]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[78516476]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[78516511]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[78516489]}]}", + "{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}", + "{\"heights\":[45894617,45894627,45894628,45894712,45898413,45898423,45898424],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[45898423,45894627]},{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[45894712]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[45894628,45894617,45898424,45898413]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[45898423,45894627]}]}", + ] + ) + } } From cfdfb7a4291e75bdf5f5ed0be825399aa42f96cc Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 13:47:50 +1300 Subject: [PATCH 32/40] refactor: Use consistent naming for contract_id --- block-streamer/src/delta_lake_client.rs | 35 ++++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index d6f0dc97f..8ef0b1a5f 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -41,8 +41,8 @@ where .context("Unable to parse Metadata") } - fn account_id_to_s3_prefix(&self, account: &str) -> String { - let mut folders = account.split('.').collect::>(); + fn s3_prefix_from_contract_id(&self, contract_id: &str) -> String { + let mut folders = contract_id.split('.').collect::>(); folders.reverse(); format!("{}/{}/", INDEXED_ACTIONS_PREFIX, folders.join("/")) @@ -80,25 +80,28 @@ where ) -> anyhow::Result> { match contract_pattern { pattern if pattern.contains(',') => { - let accounts = pattern.split(','); + let contract_ids = pattern.split(','); let mut results = vec![]; - for account in accounts { - let account = account.trim(); + for contract_id in contract_ids { + let contract_id = contract_id.trim(); - if account.contains('*') { - let pattern = account.replace("*.", ""); + if contract_id.contains('*') { + let pattern = contract_id.replace("*.", ""); results.extend( - self.list_objects_recursive(&self.account_id_to_s3_prefix(&pattern), 1) - .await?, + self.list_objects_recursive( + &self.s3_prefix_from_contract_id(&pattern), + 1, + ) + .await?, ); } else { results.extend( self.s3_client .list_all_objects( DELTA_LAKE_BUCKET, - &self.account_id_to_s3_prefix(account), + &self.s3_prefix_from_contract_id(contract_id), ) .await?, ); @@ -108,13 +111,13 @@ where Ok(results) } pattern if pattern.contains('*') => { - let pattern = pattern.replace("*.", ""); - self.list_objects_recursive(&self.account_id_to_s3_prefix(&pattern), 1) + let contract_id = pattern.replace("*.", ""); + self.list_objects_recursive(&self.s3_prefix_from_contract_id(&contract_id), 1) .await } pattern => { self.s3_client - .list_all_objects(DELTA_LAKE_BUCKET, &self.account_id_to_s3_prefix(pattern)) + .list_all_objects(DELTA_LAKE_BUCKET, &self.s3_prefix_from_contract_id(pattern)) .await } } @@ -189,7 +192,7 @@ mod tests { } #[tokio::test] - async fn lists_block_heights_for_single_account() { + async fn lists_block_heights_for_single_contract() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); mock_s3_client @@ -232,7 +235,7 @@ mod tests { } #[tokio::test] - async fn lists_block_heights_for_multiple_accounts() { + async fn lists_block_heights_for_multiple_contracts() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); mock_s3_client @@ -379,7 +382,7 @@ mod tests { } #[tokio::test] - async fn lists_block_heights_for_multiple_accounts_and_wildcard() { + async fn lists_block_heights_for_multiple_contracts_and_wildcard() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); mock_s3_client From d9100a713ed5f90eb67747b6febe163e2af11b98 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 14:25:17 +1300 Subject: [PATCH 33/40] refactor: Return `BlockHeight` instead of raw index files --- block-streamer/src/block_streamer.rs | 107 ++++----------------- block-streamer/src/delta_lake_client.rs | 121 ++++++++++++++++++------ 2 files changed, 109 insertions(+), 119 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index 776592173..d19eacc5f 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -1,13 +1,13 @@ use crate::indexer_config::IndexerConfig; use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::MatchingRule; -use crate::s3; use anyhow::{bail, Context}; use aws_sdk_s3::Client as S3Client; use chrono::{DateTime, LocalResult, TimeZone, Utc}; use near_jsonrpc_client::JsonRpcClient; use near_jsonrpc_primitives::types::blocks::RpcBlockRequest; use near_lake_framework::near_indexer_primitives::types::{BlockHeight, BlockId, BlockReference}; +use near_lake_framework::near_indexer_primitives::views::BlockView; use tokio::task::JoinHandle; pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; @@ -120,13 +120,22 @@ pub(crate) async fn start_block_stream( let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await?; let last_indexed_block = latest_block_metadata.last_indexed_block.parse::()?; - let blocks_from_index = filter_matching_blocks_from_index_files( - start_block_height, - &indexer, - delta_lake_client, - start_date, - ) - .await?; + let blocks_from_index = match &indexer.indexer_rule.matching_rule { + MatchingRule::ActionAny { + affected_account_id, + .. + } => { + delta_lake_client + .list_matching_block_heights(start_date, affected_account_id) + .await + } + MatchingRule::ActionFunctionCall { .. } => { + bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer.account_id, indexer.function_name); + } + MatchingRule::Event { .. } => { + bail!("Event matching rule not yet supported for historical processing, function {:?} {:?}", indexer.account_id, indexer.function_name); + } + }?; tracing::info!( "Flushing {} block heights from index files to historical Stream for indexer: {}", @@ -200,88 +209,6 @@ pub(crate) async fn start_block_stream( Ok(()) } -pub(crate) async fn filter_matching_blocks_from_index_files( - start_block_height: BlockHeight, - indexer: &IndexerConfig, - delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, - start_date: DateTime, -) -> anyhow::Result> { - let mut needs_dedupe_and_sort = false; - let indexer_rule = &indexer.indexer_rule; - - let index_files_content = match &indexer_rule.matching_rule { - MatchingRule::ActionAny { - affected_account_id, - .. - } => { - if affected_account_id.contains('*') || affected_account_id.contains(',') { - needs_dedupe_and_sort = true; - } - delta_lake_client - .list_matching_block_heights(start_date, affected_account_id) - .await - } - MatchingRule::ActionFunctionCall { .. } => { - bail!("ActionFunctionCall matching rule not yet supported for historical processing, function: {:?} {:?}", indexer.account_id, indexer.function_name); - } - MatchingRule::Event { .. } => { - bail!("Event matching rule not yet supported for historical processing, function {:?} {:?}", indexer.account_id, indexer.function_name); - } - }?; - - tracing::info!( - "Found {file_count} index files for function {:?} {:?} with matching rule {indexer_rule:?}", - indexer.account_id, - indexer.function_name, - file_count = index_files_content.len() - ); - let mut blocks_to_process: Vec = - parse_blocks_from_index_files(index_files_content, start_block_height); - if needs_dedupe_and_sort { - blocks_to_process.sort(); - blocks_to_process.dedup(); - } - tracing::info!( - "Found {block_count} indexed blocks to process for function {:?} {:?}", - indexer.account_id, - indexer.function_name, - block_count = blocks_to_process.len() - ); - - Ok(blocks_to_process) -} - -fn parse_blocks_from_index_files( - index_files_content: Vec, - start_block_height: u64, -) -> Vec { - index_files_content - .into_iter() - .flat_map(|file_content| { - if let Ok(file_json) = serde_json::from_str::(&file_content) { - if let Some(block_heights) = file_json["heights"].as_array() { - block_heights - .iter() - .map(|block_height| block_height.as_u64().unwrap()) - .collect::>() - .into_iter() - .filter(|block_height| block_height >= &start_block_height) - .collect() - } else { - tracing::error!( - "Unable to parse index file, no heights found: {:?}", - file_content - ); - vec![] - } - } else { - tracing::error!("Unable to parse index file: {:?}", file_content); - vec![] - } - }) - .collect::>() -} - // if block does not exist, try next block, up to MAX_RPC_BLOCKS_TO_PROCESS (20) blocks pub async fn lookup_block_date_or_next_block_date( block_height: u64, diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 8ef0b1a5f..67d91d72d 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -1,6 +1,7 @@ use anyhow::Context; use chrono::{DateTime, NaiveDate, Utc}; use futures::future::try_join_all; +use near_lake_framework::near_indexer_primitives; const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; const INDEXED_ACTIONS_PREFIX: &str = "silver/accounts/action_receipt_actions/metadata"; @@ -16,6 +17,18 @@ pub struct LatestBlockMetadata { pub processed_at_utc: String, } +#[derive(serde::Deserialize, Debug, Eq, PartialEq)] +pub struct IndexFileAction { + pub action_kind: String, + pub block_heights: Vec, +} + +#[derive(serde::Deserialize, Debug, Eq, PartialEq)] +pub struct IndexFile { + pub heights: Vec, + pub actions: Vec, +} + pub struct DeltaLakeClient where T: crate::s3_client::S3ClientTrait, @@ -63,6 +76,8 @@ where .await?; let mut results = vec![]; + // TODO do in parallel? + // TODO only list objects without .json extension for object in objects { results.extend( self.s3_client @@ -133,7 +148,7 @@ where &self, start_date: DateTime, contract_pattern: &str, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let file_list = self.list_matching_index_files(contract_pattern).await?; tracing::debug!( "Found {} index files matching {}", @@ -154,10 +169,25 @@ where let file_content_list = try_join_all(futures).await?; - Ok(file_content_list + let mut block_heights: Vec<_> = file_content_list .into_iter() - .filter(|content| !content.is_empty()) - .collect()) + .filter_map(|content| { + if content.is_empty() { + None + } else { + serde_json::from_str::(&content).ok() + } + }) + .flat_map(|index_file| index_file.heights) + .collect(); + + let pattern_has_multiple_contracts = contract_pattern.chars().any(|c| c == ',' || c == '*'); + if pattern_has_multiple_contracts { + block_heights.sort(); + block_heights.dedup(); + } + + Ok(block_heights) } } @@ -226,12 +256,7 @@ mod tests { .await .unwrap(); - assert_eq!( - block_heights, - vec![ - "{\"heights\":[92080299,92080344],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92080344,92080299]}]}" - ] - ) + assert_eq!(block_heights, vec![92080299, 92080344]) } #[tokio::test] @@ -240,8 +265,6 @@ mod tests { mock_s3_client .expect_list_all_objects() - // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case - // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) .returning(|_bucket, prefix| { let objects = match prefix { "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/" => vec![ @@ -270,7 +293,7 @@ mod tests { mock_s3_client .expect_get_text_file() .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/aurora-silo-dev/hackathon/2023-05-30.json".to_string())) - .returning(|_bucket, _prefix| Ok("{\"heights\":[93065811,93067570,93067619,93067631,93067726,93067737,93067770,93067889,93067920,93067926,93067936,93073935,93073944,93073954],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[93073954,93067770,93067726,93065811,93067619,93073935,93067889,93067737,93067570,93067926,93073944,93067920,93067631,93067936]}]}".to_string())); + .returning(|_bucket, _prefix| Ok("{\"heights\":[92167977,93067570,93067619,93067631,93067726,93067737,93067770,93067889,93067920,93067926,93067936,93073935,93073944,93073954],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[93073954,93067770,93067726,93065811,93067619,93073935,93067889,93067737,93067570,93067926,93073944,93067920,93067631,93067936]}]}".to_string())); mock_s3_client .expect_get_text_file() .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/sputnik-dao/hackathon/2022-05-27.json".to_string())) @@ -293,9 +316,10 @@ mod tests { assert_eq!( block_heights, vec![ - "{\"heights\":[92167977,92168200,92168293,92168338,92168535,92168870,92168871,92168922,92168923,92168939,92168971,92169330],\"actions\":[{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[92168200,92168338]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[92168535,92167977]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[92167977]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[92168922,92168971,92168870]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[92168871,92168923,92169330,92168293,92168939,92167977]}]}", - "{\"heights\":[93065811,93067570,93067619,93067631,93067726,93067737,93067770,93067889,93067920,93067926,93067936,93073935,93073944,93073954],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[93073954,93067770,93067726,93065811,93067619,93073935,93067889,93067737,93067570,93067926,93073944,93067920,93067631,93067936]}]}", - "{\"heights\":[66494954],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[66494954]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[66494954]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[66494954]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[66494954]}]}" + 66494954, 92167977, 92168200, 92168293, 92168338, 92168535, 92168870, 92168871, + 92168922, 92168923, 92168939, 92168971, 92169330, 93067570, 93067619, 93067631, + 93067726, 93067737, 93067770, 93067889, 93067920, 93067926, 93067936, 93073935, + 93073944, 93073954 ] ) } @@ -306,8 +330,6 @@ mod tests { mock_s3_client .expect_list_all_objects() - // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case - // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) .returning(|_bucket, prefix| { let objects = match prefix { "silver/accounts/action_receipt_actions/metadata/near/keypom/" => vec![ @@ -373,11 +395,7 @@ mod tests { assert_eq!( block_heights, - vec![ - "{\"heights\":[102025554],\"actions\":[{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[102025554]}]}", - "{\"heights\":[104045849,104047967,104047968],\"actions\":[{\"action_kind\":\"TRANSFER\",\"block_heights\":[104047968,104045849,104047967]}]}", - "{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}" - ] + vec![102025554, 104045849, 104047967, 104047968, 104616819] ) } @@ -387,8 +405,6 @@ mod tests { mock_s3_client .expect_list_all_objects() - // FIX: This syntax is preferable as it will assert the use of the arguments - but it causes a compiler error in this specific case - // .with(predicate::eq(DELTA_LAKE_BUCKET), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/".to_string())) .returning(|_bucket, prefix| { let objects = match prefix { "silver/accounts/action_receipt_actions/metadata/near/keypom/" => vec![ @@ -401,7 +417,6 @@ mod tests { "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/" => vec![ "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string() ], - "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json" => vec![ "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string() ], @@ -443,10 +458,58 @@ mod tests { assert_eq!( block_heights, vec![ - "{\"heights\":[78516467,78516476,78516489,78516511,78516512],\"actions\":[{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[78516467]},{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[78516476]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[78516476,78516512]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[78516476]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[78516511]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[78516489]}]}", - "{\"heights\":[104616819],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}", - "{\"heights\":[45894617,45894627,45894628,45894712,45898413,45898423,45898424],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[45898423,45894627]},{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[45894712]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[45894628,45894617,45898424,45898413]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[45898423,45894627]}]}", + 45894617, 45894627, 45894628, 45894712, 45898413, 45898423, 45898424, 78516467, + 78516476, 78516489, 78516511, 78516512, 104616819 ] ) } + + #[tokio::test] + async fn sorts_and_removes_duplicates_for_multiple_contracts() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_list_all_objects() + .returning(|_bucket, prefix| { + let objects = match prefix { + "silver/accounts/action_receipt_actions/metadata/near/keypom/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string(), + ], + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/" => vec![ + "silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string() + ], + + _ => panic!("Unexpected prefix: {}", prefix) + }; + + Ok(objects) + }); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/agency/hackathon/2021-08-22.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[45894628,45894617,45898413,45894627,45894712,45898423,45898424],\"actions\":[{\"action_kind\":\"CREATE_ACCOUNT\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"FUNCTION_CALL\",\"block_heights\":[45898423,45894627]},{\"action_kind\":\"DELETE_ACCOUNT\",\"block_heights\":[45894712]},{\"action_kind\":\"ADD_KEY\",\"block_heights\":[45894617,45898413]},{\"action_kind\":\"TRANSFER\",\"block_heights\":[45894628,45894617,45898424,45898413]},{\"action_kind\":\"DEPLOY_CONTRACT\",\"block_heights\":[45898423,45894627]}]}".to_string())); + mock_s3_client + .expect_get_text_file() + .with(predicate::eq(DELTA_LAKE_BUCKET.to_string()), predicate::eq("silver/accounts/action_receipt_actions/metadata/near/keypom/2023-10-31.json".to_string())) + .returning(|_bucket, _prefix| Ok("{\"heights\":[45898424,45898423,45898413,45894712],\"actions\":[{\"action_kind\":\"ADD_KEY\",\"block_heights\":[104616819]}]}".to_string())); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_heights = delta_lake_client + .list_matching_block_heights( + NaiveDate::from_ymd_opt(2021, 5, 26) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(), + "keypom.near, hackathon.agency.near", + ) + .await + .unwrap(); + + assert_eq!( + block_heights, + vec![45894617, 45894627, 45894628, 45894712, 45898413, 45898423, 45898424] + ) + } } From 242a997d9fe949fa773c8aca4bab3c641da7a71e Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 15:50:32 +1300 Subject: [PATCH 34/40] refactor: Fetch start block date from s3 instead of rpc --- block-streamer/Cargo.lock | 965 +-------------------------- block-streamer/Cargo.toml | 12 +- block-streamer/src/block_streamer.rs | 256 +++++-- block-streamer/src/main.rs | 10 +- 4 files changed, 243 insertions(+), 1000 deletions(-) diff --git a/block-streamer/Cargo.lock b/block-streamer/Cargo.lock index 2621a191b..b96492ead 100644 --- a/block-streamer/Cargo.lock +++ b/block-streamer/Cargo.lock @@ -2,62 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "actix" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cba56612922b907719d4a01cf11c8d5b458e7d3dba946d0435f20f58d6795ed2" -dependencies = [ - "actix-macros", - "actix-rt", - "actix_derive", - "bitflags 2.4.1", - "bytes", - "crossbeam-channel", - "futures-core", - "futures-sink", - "futures-task", - "futures-util", - "log", - "once_cell", - "parking_lot", - "pin-project-lite", - "smallvec", - "tokio", - "tokio-util 0.7.10", -] - -[[package]] -name = "actix-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" -dependencies = [ - "quote", - "syn 2.0.39", -] - -[[package]] -name = "actix-rt" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" -dependencies = [ - "futures-core", - "tokio", -] - -[[package]] -name = "actix_derive" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "ahash" version = "0.8.6" @@ -94,54 +38,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" - -[[package]] -name = "anstyle-parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" -dependencies = [ - "anstyle", - "windows-sys", -] - [[package]] name = "anyhow" version = "1.0.75" @@ -196,17 +92,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -747,7 +632,7 @@ dependencies = [ "pin-project-lite", "pin-utils", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tracing", ] @@ -899,7 +784,7 @@ dependencies = [ "serde", "time", "tokio", - "tokio-util 0.7.10", + "tokio-util", ] [[package]] @@ -957,12 +842,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.5" @@ -1000,12 +879,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - [[package]] name = "blake2" version = "0.9.2" @@ -1050,15 +923,13 @@ dependencies = [ "chrono", "futures", "mockall", - "near-jsonrpc-client", - "near-jsonrpc-primitives", "near-lake-framework", "redis", "serde", "serde_json", "tokio", "tokio-stream", - "tokio-util 0.7.10", + "tokio-util", "tracing", "tracing-subscriber", "wildmatch", @@ -1201,52 +1072,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "clap" -version = "4.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 2.0.39", -] - -[[package]] -name = "clap_lex" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" - -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - [[package]] name = "combine" version = "4.6.6" @@ -1258,7 +1083,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.10", + "tokio-util", ] [[package]] @@ -1316,25 +1141,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - [[package]] name = "crunchy" version = "0.2.2" @@ -1640,15 +1446,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - [[package]] name = "enum-map" version = "2.7.2" @@ -1675,16 +1472,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "errno" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" -dependencies = [ - "libc", - "windows-sys", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -1719,12 +1506,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "float-cmp" version = "0.9.0" @@ -1740,21 +1521,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1927,7 +1693,7 @@ dependencies = [ "indexmap 2.1.0", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util", "tracing", ] @@ -1952,30 +1718,12 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.3" @@ -2000,15 +1748,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys", -] - [[package]] name = "http" version = "0.2.11" @@ -2098,31 +1837,6 @@ dependencies = [ "tokio-rustls 0.24.1", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.58" @@ -2193,12 +1907,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - [[package]] name = "itertools" version = "0.10.5" @@ -2241,22 +1949,6 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" -[[package]] -name = "linux-raw-sys" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.20" @@ -2288,12 +1980,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "mio" version = "0.8.9" @@ -2332,30 +2018,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "near-account-id" version = "0.17.0" @@ -2366,28 +2028,6 @@ dependencies = [ "serde", ] -[[package]] -name = "near-chain-configs" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f9a1c805846237d56f99b328ba6ab77e5d43ef59aaaf8d2a41d42fdc708a7b" -dependencies = [ - "anyhow", - "chrono", - "derive_more", - "near-config-utils", - "near-crypto", - "near-o11y", - "near-primitives", - "num-rational", - "once_cell", - "serde", - "serde_json", - "sha2 0.10.8", - "smart-default", - "tracing", -] - [[package]] name = "near-config-utils" version = "0.17.0" @@ -2448,56 +2088,21 @@ dependencies = [ ] [[package]] -name = "near-jsonrpc-client" -version = "0.6.0" +name = "near-lake-framework" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118f44c02ad211db805c1370ad3ff26576af6ff554093c9fece1b835d29d233a" +checksum = "5fbfc3d7c294aa144c3a1817452931e91ce045ee1cf2e34b7b439d4695073634" dependencies = [ - "borsh", - "lazy_static", - "log", - "near-chain-configs", - "near-crypto", - "near-jsonrpc-primitives", - "near-primitives", - "reqwest", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "near-jsonrpc-primitives" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b2934b5ab243e25e951c984525ba0aff0e719ed915c988c5195405aa0f6987" -dependencies = [ - "arbitrary", - "near-chain-configs", - "near-crypto", - "near-primitives", - "near-rpc-error-macro", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "near-lake-framework" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbfc3d7c294aa144c3a1817452931e91ce045ee1cf2e34b7b439d4695073634" -dependencies = [ - "anyhow", - "async-stream", - "async-trait", - "aws-config 0.53.0", - "aws-credential-types 0.53.0", - "aws-sdk-s3 0.23.0", - "aws-types 0.53.0", - "derive_builder", - "futures", - "near-indexer-primitives", + "anyhow", + "async-stream", + "async-trait", + "aws-config 0.53.0", + "aws-credential-types 0.53.0", + "aws-sdk-s3 0.23.0", + "aws-types 0.53.0", + "derive_builder", + "futures", + "near-indexer-primitives", "serde", "serde_json", "thiserror", @@ -2506,32 +2111,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "near-o11y" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7d35397b02b131c188c72f3885e97daeccab134ec2fc8cc0073a94cf1cfe19" -dependencies = [ - "actix", - "atty", - "clap", - "near-crypto", - "near-primitives-core", - "once_cell", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "prometheus", - "serde", - "strum", - "thiserror", - "tokio", - "tracing", - "tracing-appender", - "tracing-opentelemetry", - "tracing-subscriber", -] - [[package]] name = "near-primitives" version = "0.17.0" @@ -2576,7 +2155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775fec19ef51a341abdbf792a9dda5b4cb89f488f681b2fd689b9321d24db47b" dependencies = [ "arbitrary", - "base64 0.21.5", + "base64", "borsh", "bs58", "derive_more", @@ -2699,7 +2278,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi", "libc", ] @@ -2715,98 +2294,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl" -version = "0.10.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "opentelemetry" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "js-sys", - "lazy_static", - "percent-encoding", - "pin-project", - "rand 0.8.5", - "thiserror", - "tokio", - "tokio-stream", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a" -dependencies = [ - "async-trait", - "futures", - "futures-util", - "http", - "opentelemetry", - "prost", - "thiserror", - "tokio", - "tonic", - "tonic-build", -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" -dependencies = [ - "opentelemetry", -] - [[package]] name = "outref" version = "0.1.0" @@ -2836,45 +2329,12 @@ dependencies = [ "sha2 0.10.8", ] -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.1.0", -] - [[package]] name = "pin-project" version = "1.1.3" @@ -2917,12 +2377,6 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" - [[package]] name = "powerfmt" version = "0.2.0" @@ -2993,80 +2447,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prometheus" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "memchr", - "parking_lot", - "protobuf", - "thiserror", -] - -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" -dependencies = [ - "bytes", - "heck 0.3.3", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost", - "prost-types", - "regex", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost", -] - -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - [[package]] name = "quote" version = "1.0.33" @@ -3165,19 +2545,10 @@ dependencies = [ "ryu", "sha1 0.6.1", "tokio", - "tokio-util 0.7.10", + "tokio-util", "url", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "reed-solomon-erasure" version = "4.0.2" @@ -3231,44 +2602,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" -[[package]] -name = "reqwest" -version = "0.11.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" -dependencies = [ - "base64 0.21.5", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "rfc6979" version = "0.3.1" @@ -3318,19 +2651,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" -dependencies = [ - "bitflags 2.4.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - [[package]] name = "rustls" version = "0.20.9" @@ -3373,7 +2693,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64", ] [[package]] @@ -3407,12 +2727,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "sct" version = "0.7.1" @@ -3462,7 +2776,7 @@ version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -3527,25 +2841,13 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_with" version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ - "base64 0.21.5", + "base64", "chrono", "hex", "indexmap 1.9.3", @@ -3753,7 +3055,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "rustversion", @@ -3788,40 +3090,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" -dependencies = [ - "cfg-if", - "fastrand 2.0.1", - "redox_syscall", - "rustix", - "windows-sys", -] - [[package]] name = "termtree" version = "0.4.1" @@ -3913,7 +3181,6 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3921,16 +3188,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -3942,16 +3199,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -3984,20 +3231,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.10" @@ -4021,49 +3254,6 @@ dependencies = [ "serde", ] -[[package]] -name = "tonic" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.13.1", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util 0.6.10", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic-build" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" -dependencies = [ - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - [[package]] name = "tower" version = "0.4.13" @@ -4072,13 +3262,9 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", - "slab", "tokio", - "tokio-util 0.7.10", "tower-layer", "tower-service", "tracing", @@ -4108,18 +3294,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" -dependencies = [ - "crossbeam-channel", - "thiserror", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.27" @@ -4141,27 +3315,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "tracing-log" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -4173,20 +3326,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.17.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" -dependencies = [ - "once_cell", - "opentelemetry", - "tracing", - "tracing-core", - "tracing-log 0.1.4", - "tracing-subscriber", -] - [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -4202,7 +3341,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log 0.2.0", + "tracing-log", ] [[package]] @@ -4250,12 +3389,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - [[package]] name = "unsafe-libyaml" version = "0.2.9" @@ -4291,12 +3424,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - [[package]] name = "uuid" version = "1.6.1" @@ -4309,12 +3436,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" @@ -4373,18 +3494,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.88" @@ -4434,18 +3543,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "wildmatch" version = "2.1.1" @@ -4549,16 +3646,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys", -] - [[package]] name = "xmlparser" version = "0.13.6" diff --git a/block-streamer/Cargo.toml b/block-streamer/Cargo.toml index 9cbc9a893..2335b81f5 100644 --- a/block-streamer/Cargo.toml +++ b/block-streamer/Cargo.toml @@ -3,15 +3,14 @@ name = "block-streamer" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = "1.0.57" async-trait = "0.1.74" +aws-config = { version = "1.0.0", features = ["behavior-version-latest"]} aws-smithy-http = "0.60.0" aws-smithy-types = "1.0.1" aws-sdk-s3 = "0.39.1" -aws-config = { version = "1.0.0", features = ["behavior-version-latest"]} +aws-types = "1.0.0" borsh = "0.10.2" chrono = "0.4.25" futures = "0.3.5" @@ -20,13 +19,10 @@ redis = { version = "0.21.5", features = ["tokio-comp", "connection-manager"] } serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" tracing = "0.1.40" -tracing-subscriber = "0.3.18" -tokio = "1.28.0" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tokio = { version = "1.28.0", features = ["rt-multi-thread"]} tokio-util = "0.7.10" tokio-stream = "0.1.14" wildmatch = "2.1.1" near-lake-framework = "0.7.1" -near-jsonrpc-client = "0.6.0" -near-jsonrpc-primitives = "0.17.0" -aws-types = "1.0.0" diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index d19eacc5f..dade0b6ec 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -2,15 +2,11 @@ use crate::indexer_config::IndexerConfig; use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::MatchingRule; use anyhow::{bail, Context}; -use aws_sdk_s3::Client as S3Client; -use chrono::{DateTime, LocalResult, TimeZone, Utc}; -use near_jsonrpc_client::JsonRpcClient; -use near_jsonrpc_primitives::types::blocks::RpcBlockRequest; -use near_lake_framework::near_indexer_primitives::types::{BlockHeight, BlockId, BlockReference}; -use near_lake_framework::near_indexer_primitives::views::BlockView; +use chrono::TimeZone; +use near_lake_framework::near_indexer_primitives; use tokio::task::JoinHandle; -pub const MAX_RPC_BLOCKS_TO_PROCESS: u8 = 20; +pub const MAX_S3_RETRY_COUNT: u8 = 20; pub struct Task { handle: JoinHandle>, @@ -28,13 +24,11 @@ impl BlockStreamer { pub fn start( &mut self, - start_block_height: BlockHeight, + start_block_height: near_indexer_primitives::types::BlockHeight, indexer: IndexerConfig, redis_connection_manager: crate::redis::ConnectionManager, - s3_client: S3Client, + s3_client: crate::s3_client::S3Client, chain_id: ChainId, - json_rpc_client: JsonRpcClient, - delta_lake_client: crate::delta_lake_client::DeltaLakeClient, ) -> anyhow::Result<()> { if self.task.is_some() { return Err(anyhow::anyhow!("BlockStreamer has already been started",)); @@ -57,10 +51,8 @@ impl BlockStreamer { start_block_height, indexer.clone(), &redis_connection_manager, - &s3_client, + s3_client, &chain_id, - &json_rpc_client, - &delta_lake_client, ) => { result.map_err(|err| { tracing::error!( @@ -101,21 +93,20 @@ impl BlockStreamer { } pub(crate) async fn start_block_stream( - start_block_height: BlockHeight, + start_block_height: near_indexer_primitives::types::BlockHeight, indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, - s3_client: &S3Client, + s3_client: crate::s3_client::S3Client, chain_id: &ChainId, - json_rpc_client: &JsonRpcClient, - delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, ) -> anyhow::Result<()> { tracing::info!( "Starting block stream from {start_block_height} for indexer: {}", indexer.get_full_name(), ); - let start_date = - lookup_block_date_or_next_block_date(start_block_height, json_rpc_client).await?; + let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new(s3_client.clone()); + + let start_date = get_nearest_block_date(&s3_client, start_block_height, chain_id).await?; let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await?; let last_indexed_block = latest_block_metadata.last_indexed_block.parse::()?; @@ -209,36 +200,213 @@ pub(crate) async fn start_block_stream( Ok(()) } -// if block does not exist, try next block, up to MAX_RPC_BLOCKS_TO_PROCESS (20) blocks -pub async fn lookup_block_date_or_next_block_date( +pub async fn get_nearest_block_date( + s3_client: &impl crate::s3_client::S3ClientTrait, block_height: u64, - client: &JsonRpcClient, -) -> anyhow::Result> { + chain_id: &ChainId, +) -> anyhow::Result> { + let bucket = match chain_id { + ChainId::Mainnet => "near-lake-data-mainnet", + ChainId::Testnet => "near-lake-data-testnet", + }; + let mut current_block_height = block_height; - let mut retry_count = 0; + let mut retry_count = 1; loop { - let request = RpcBlockRequest { - block_reference: BlockReference::BlockId(BlockId::Height(current_block_height)), - }; - - match client.call(request).await { - Ok(response) => { - let header = response.header; - let timestamp_nanosec = header.timestamp_nanosec; - return match Utc.timestamp_opt((timestamp_nanosec / 1000000000) as i64, 0) { - LocalResult::Single(date) => Ok(date), - LocalResult::Ambiguous(date, _) => Ok(date), - LocalResult::None => Err(anyhow::anyhow!("Unable to get block timestamp")), - }; + let block_key = format!("{:0>12}/block.json", current_block_height); + match s3_client.get_text_file(bucket, &block_key).await { + Ok(text) => { + let block: near_indexer_primitives::views::BlockView = serde_json::from_str(&text)?; + return Ok(chrono::Utc.timestamp_nanos(block.header.timestamp_nanosec as i64)); } - Err(_) => { - tracing::debug!("RPC failed to get block: {:?}", current_block_height); - retry_count += 1; - if retry_count > MAX_RPC_BLOCKS_TO_PROCESS { - return Err(anyhow::anyhow!("Unable to get block")); + + Err(e) => { + if e.root_cause() + .downcast_ref::() + .is_some() + { + retry_count += 1; + if retry_count > MAX_S3_RETRY_COUNT { + anyhow::bail!("Exceeded maximum retries to fetch block from S3"); + } + + tracing::debug!( + "Block {} not found on S3, attempting to fetch next block", + current_block_height + ); + current_block_height += 1; + continue; } - current_block_height += 1; + + return Err(e).context("Failed to fetch block from S3"); } } } } + +#[cfg(test)] +mod tests { + use super::*; + use mockall::predicate; + + mod get_near_block_date { + use super::*; + + #[tokio::test] + async fn gets_the_date_of_the_closest_block() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397175/block.json"), + ) + .times(1) + .returning(|_, _| { + Ok(r#"{ + "author": "someone", + "header": { + "approvals": [], + "block_merkle_root": "ERiC7AJ2zbVz1HJHThR5NWDDN9vByhwdjcVfivmpY5B", + "block_ordinal": 92102682, + "challenges_result": [], + "challenges_root": "11111111111111111111111111111111", + "chunk_headers_root": "MDiJxDyvUQaZRKmUwa5jgQuV6XjwVvnm4tDrajCxwvz", + "chunk_mask": [], + "chunk_receipts_root": "n84wEo7kTKTCJsyqBZ2jndhjrAMeJAXMwKvnJR7vCuy", + "chunk_tx_root": "D8j64GMKBMvUfvnuHtWUyDtMHM5mJ2pA4G5VmYYJvo5G", + "chunks_included": 4, + "epoch_id": "2RMQiomr6CSSwUWpmB62YohxHbfadrHfcsaa3FVb4J9x", + "epoch_sync_data_hash": null, + "gas_price": "100000000", + "hash": "FA1z9RVm9fX3g3mgP3NToZGwWeeXYn8bvZs4nwwTgCpD", + "height": 102162333, + "last_ds_final_block": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "last_final_block": "8xkwjn6Lb6UhMBhxcbVQBf3318GafkdaXoHA8Jako1nn", + "latest_protocol_version": 62, + "next_bp_hash": "dmW84aEj2iVJMLwJodJwTfAyeA1LJaHEthvnoAsvTPt", + "next_epoch_id": "C9TDDYthANoduoTBZS7WYDsBSe9XCm4M2F9hRoVXVXWY", + "outcome_root": "6WxzWLVp4b4bFbxHzu18apVfXLvHGKY7CHoqD2Eq3TFJ", + "prev_hash": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "prev_height": 102162332, + "prev_state_root": "Aq2ndkyDiwroUWN69Ema9hHtnr6dPHoEBRNyfmd8v4gB", + "random_value": "7ruuMyDhGtTkYaCGYMy7PirPiM79DXa8GhVzQW1pHRoz", + "rent_paid": "0", + "signature": "ed25519:5gYYaWHkAEK5etB8tDpw7fmehkoYSprUxKPygaNqmhVDFCMkA1n379AtL1BBkQswLAPxWs1BZvypFnnLvBtHRknm", + "timestamp": 1695921400989555700, + "timestamp_nanosec": "1695921400989555820", + "total_supply": "1155783047679681223245725102954966", + "validator_proposals": [], + "validator_reward": "0" + }, + "chunks": [] + }"# + .to_string()) + }); + + let block_date = get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet) + .await + .unwrap(); + + assert_eq!( + block_date, + chrono::Utc.timestamp_nanos(1695921400989555820 as i64) + ); + } + + #[tokio::test] + async fn retires_if_a_block_doesnt_exist() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397175/block.json"), + ) + .times(1) + .returning(|_, _| { + Err(anyhow::anyhow!( + aws_sdk_s3::types::error::NoSuchKey::builder().build() + )) + }); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397176/block.json"), + ) + .times(1) + .returning(|_, _| { + Ok(r#"{ + "author": "someone", + "header": { + "approvals": [], + "block_merkle_root": "ERiC7AJ2zbVz1HJHThR5NWDDN9vByhwdjcVfivmpY5B", + "block_ordinal": 92102682, + "challenges_result": [], + "challenges_root": "11111111111111111111111111111111", + "chunk_headers_root": "MDiJxDyvUQaZRKmUwa5jgQuV6XjwVvnm4tDrajCxwvz", + "chunk_mask": [], + "chunk_receipts_root": "n84wEo7kTKTCJsyqBZ2jndhjrAMeJAXMwKvnJR7vCuy", + "chunk_tx_root": "D8j64GMKBMvUfvnuHtWUyDtMHM5mJ2pA4G5VmYYJvo5G", + "chunks_included": 4, + "epoch_id": "2RMQiomr6CSSwUWpmB62YohxHbfadrHfcsaa3FVb4J9x", + "epoch_sync_data_hash": null, + "gas_price": "100000000", + "hash": "FA1z9RVm9fX3g3mgP3NToZGwWeeXYn8bvZs4nwwTgCpD", + "height": 102162333, + "last_ds_final_block": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "last_final_block": "8xkwjn6Lb6UhMBhxcbVQBf3318GafkdaXoHA8Jako1nn", + "latest_protocol_version": 62, + "next_bp_hash": "dmW84aEj2iVJMLwJodJwTfAyeA1LJaHEthvnoAsvTPt", + "next_epoch_id": "C9TDDYthANoduoTBZS7WYDsBSe9XCm4M2F9hRoVXVXWY", + "outcome_root": "6WxzWLVp4b4bFbxHzu18apVfXLvHGKY7CHoqD2Eq3TFJ", + "prev_hash": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "prev_height": 102162332, + "prev_state_root": "Aq2ndkyDiwroUWN69Ema9hHtnr6dPHoEBRNyfmd8v4gB", + "random_value": "7ruuMyDhGtTkYaCGYMy7PirPiM79DXa8GhVzQW1pHRoz", + "rent_paid": "0", + "signature": "ed25519:5gYYaWHkAEK5etB8tDpw7fmehkoYSprUxKPygaNqmhVDFCMkA1n379AtL1BBkQswLAPxWs1BZvypFnnLvBtHRknm", + "timestamp": 1695921400989555700, + "timestamp_nanosec": "1695921400989555820", + "total_supply": "1155783047679681223245725102954966", + "validator_proposals": [], + "validator_reward": "0" + }, + "chunks": [] + }"# + .to_string()) + }); + + let block_date = get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet) + .await + .unwrap(); + + assert_eq!( + block_date, + chrono::Utc.timestamp_nanos(1695921400989555820 as i64) + ); + } + + #[tokio::test] + async fn exits_if_maximum_retries_exceeded() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .times(MAX_S3_RETRY_COUNT as usize) + .returning(|_, _| { + Err(anyhow::anyhow!( + aws_sdk_s3::types::error::NoSuchKey::builder().build() + )) + }); + + let result = + get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet).await; + + assert!(result.is_err()); + } + } +} diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index ace06c664..d5da2a759 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -1,4 +1,3 @@ -use near_jsonrpc_client::JsonRpcClient; use tracing_subscriber::prelude::*; use crate::indexer_config::IndexerConfig; @@ -24,15 +23,10 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Starting {}", crate::LOG_TARGET); - let json_rpc_client = JsonRpcClient::connect("https://archival-rpc.mainnet.near.org"); let redis_connection_manager = redis::connect("redis://127.0.0.1").await?; let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new( - crate::s3_client::S3Client::new(&aws_config), - ); + let s3_client = crate::s3_client::S3Client::new(&aws_config); let contract = "queryapi.dataplatform.near"; let matching_rule = MatchingRule::ActionAny { @@ -63,8 +57,6 @@ async fn main() -> anyhow::Result<()> { redis_connection_manager, s3_client, ChainId::Mainnet, - json_rpc_client, - delta_lake_client, )?; streamer.take_handle().unwrap().await??; From 187d1873de886d20213abac205157724d1e480e3 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 15:53:30 +1300 Subject: [PATCH 35/40] chore: Remove unused code --- block-streamer/src/indexer_rule.rs | 54 ----- block-streamer/src/s3.rs | 342 ----------------------------- 2 files changed, 396 deletions(-) delete mode 100644 block-streamer/src/indexer_rule.rs delete mode 100644 block-streamer/src/s3.rs diff --git a/block-streamer/src/indexer_rule.rs b/block-streamer/src/indexer_rule.rs deleted file mode 100644 index 5ae310e28..000000000 --- a/block-streamer/src/indexer_rule.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(not(feature = "near-sdk"))] -use borsh::{self, BorshDeserialize, BorshSerialize}; -#[cfg(not(feature = "near-sdk"))] -use serde::{Deserialize, Serialize}; - -#[cfg(feature = "near-sdk")] -use near_sdk::borsh::{self, BorshDeserialize, BorshSerialize}; -#[cfg(feature = "near-sdk")] -use near_sdk::serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -pub struct IndexerRule { - pub indexer_rule_kind: IndexerRuleKind, - pub matching_rule: MatchingRule, - pub id: Option, - pub name: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -pub enum IndexerRuleKind { - Action, - Event, - AnyBlock, - Shard, -} -// future: ComposedRuleKind for multiple actions or events - -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE")] -pub enum Status { - Any, - Success, - Fail, -} - -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, PartialEq, Eq)] -#[serde(tag = "rule", rename_all = "SCREAMING_SNAKE_CASE")] -pub enum MatchingRule { - ActionAny { - affected_account_id: String, - status: Status, - }, - ActionFunctionCall { - affected_account_id: String, - status: Status, - function: String, - }, - Event { - contract_account_id: String, - standard: String, - version: String, - event: String, - }, -} diff --git a/block-streamer/src/s3.rs b/block-streamer/src/s3.rs deleted file mode 100644 index 0356184db..000000000 --- a/block-streamer/src/s3.rs +++ /dev/null @@ -1,342 +0,0 @@ -use anyhow::{bail, Context, Result}; -use aws_sdk_s3::Client as S3Client; -use chrono::{DateTime, NaiveDate, Utc}; -use futures::future::try_join_all; - -// Sanity check, if we hit this we have 1M S3 results. -// Currently that would be either 2,700 years of FunctionCall data or 1M contract folders. -// If we hit 1M contracts we should build an index to support efficient wildcard contract matching. -const MAX_S3_LIST_REQUESTS: usize = 1000; -pub const INDEXED_DATA_FILES_BUCKET: &str = "near-delta-lake"; -pub const INDEXED_ACTIONS_FILES_FOLDER: &str = "silver/accounts/action_receipt_actions/metadata"; - -fn storage_path_for_account(account: &str) -> String { - let mut folders = account.split('.').collect::>(); - folders.reverse(); - folders.join("/") -} - -pub async fn find_index_files_by_pattern( - s3_client: &S3Client, - s3_bucket: &str, - s3_folder: &str, - pattern: &str, -) -> Result> { - Ok(match pattern { - x if x.contains(',') => { - let account_array = x.split(','); - let mut results = vec![]; - for account in account_array { - let account = account.trim(); - let sub_results = if account.contains('*') { - list_index_files_by_wildcard(s3_client, s3_bucket, s3_folder, &account).await? - } else { - list_s3_bucket_by_prefix( - s3_client, - s3_bucket, - &format!("{}/{}/", s3_folder, storage_path_for_account(account)), - ) - .await? - }; - results.extend(sub_results); - } - results - } - x if x.contains('*') => { - list_index_files_by_wildcard(s3_client, s3_bucket, s3_folder, &x).await? - } - _ => { - list_s3_bucket_by_prefix( - s3_client, - s3_bucket, - &format!("{}/{}/", s3_folder, storage_path_for_account(pattern),), - ) - .await? - } - }) -} - -async fn list_index_files_by_wildcard( - s3_client: &S3Client, - s3_bucket: &str, - s3_folder: &str, - pattern: &&str, -) -> Result> { - // remove sub-account wildcard from pattern - let pattern = pattern.replace("*.", ""); - let path = storage_path_for_account(&pattern); - - let folders = - list_s3_bucket_by_prefix(s3_client, s3_bucket, &format!("{}/{}/", s3_folder, path)).await?; - // for each matching folder list files - let mut results = vec![]; - for folder in folders { - results.extend(list_s3_bucket_by_prefix(s3_client, s3_bucket, &folder).await?); - } - Ok(results) -} - -async fn list_s3_bucket_by_prefix( - s3_client: &S3Client, - s3_bucket: &str, - s3_prefix: &str, -) -> Result> { - let mut results = vec![]; - let mut continuation_token: Option = None; - - let mut counter = 0; - loop { - let mut configured_client = s3_client - .list_objects_v2() - .bucket(s3_bucket) - .prefix(s3_prefix) - .delimiter("/"); - - if continuation_token.is_some() { - configured_client = configured_client.continuation_token(continuation_token.unwrap()); - } - - let file_list = configured_client.send().await?; - if let Some(common_prefixes) = file_list.common_prefixes { - let keys: Vec = common_prefixes - .into_iter() - .map(|o| o.prefix.unwrap()) - .collect(); - results.extend(keys); - } - if let Some(objects) = file_list.contents { - let keys: Vec = objects.into_iter().map(|o| o.key.unwrap()).collect(); - results.extend(keys); - } - if file_list.next_continuation_token.is_some() { - continuation_token = file_list.next_continuation_token; - counter += 1; - if counter > MAX_S3_LIST_REQUESTS { - bail!("Exceeded internal limit of {MAX_S3_LIST_REQUESTS}") - } - } else { - break; - } - } - Ok(results) -} - -pub async fn fetch_contract_index_files( - s3_client: &S3Client, - s3_bucket: &str, - s3_folder: &str, - start_date: DateTime, - contract_pattern: &str, -) -> Result> { - // list all index files - let file_list = - find_index_files_by_pattern(s3_client, s3_bucket, s3_folder, contract_pattern).await?; - - let fetch_and_parse_tasks = file_list - .into_iter() - .filter(|index_file_listing| file_name_date_after(start_date, index_file_listing)) - .map(|key| { - let s3_client = s3_client.clone(); - async move { - // Fetch the file - fetch_text_file_from_s3(s3_bucket, key, &s3_client).await - } - }) - .collect::>(); - - // Execute all tasks in parallel and wait for completion - let file_contents: Vec = try_join_all(fetch_and_parse_tasks).await?; - Ok(file_contents - .into_iter() - .filter(|file_contents| !file_contents.is_empty()) - .collect::>()) -} - -pub async fn fetch_text_file_from_s3( - s3_bucket: &str, - key: String, - s3_client: &S3Client, -) -> Result { - // todo: can we retry if this fails like the lake s3_fetcher fn does? - // If so, can we differentiate between a file not existing (block height does not exist) and a network error? - let get_object_output = s3_client - .get_object() - .bucket(s3_bucket) - .key(key.clone()) - .send() - .await - .with_context(|| format!("Error fetching index file {key}"))?; - - let bytes = get_object_output - .body - .collect() - .await - .with_context(|| format!("Error reading bytes of index file {key}"))?; - String::from_utf8(bytes.to_vec()).with_context(|| format!("Error parsing index file {key}")) -} - -/// check whether the filename is a date after the start date -/// filename is in format 2022-10-03.json -fn file_name_date_after(start_date: DateTime, file_name: &str) -> bool { - let file_name_date = file_name.split('/').last().unwrap().replace(".json", ""); - let file_name_date = NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d"); - match file_name_date { - Ok(file_name_date) => file_name_date >= start_date.date_naive(), - Err(e) => { - // if we can't parse the date assume a file this code is not meant to handle - tracing::debug!( - target: crate::LOG_TARGET, - "Error parsing file name date: {:?}", - e - ); - false - } - } -} - -#[cfg(test)] -mod tests { - use crate::s3::{ - fetch_text_file_from_s3, find_index_files_by_pattern, list_s3_bucket_by_prefix, - INDEXED_ACTIONS_FILES_FOLDER, INDEXED_DATA_FILES_BUCKET, - }; - - /// Parses env vars from .env, Run with - /// cargo test s3::tests::list_delta_bucket -- mainnet from-latest; - #[tokio::test] - #[ignore] - async fn list_delta_bucket() { - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let list = list_s3_bucket_by_prefix( - &s3_client, - INDEXED_DATA_FILES_BUCKET, - &format!("{}/", INDEXED_ACTIONS_FILES_FOLDER), - ) - .await - .unwrap(); - assert_eq!(list.len(), 4); - } - - /// cargo test s3::tests::list_with_single_contract -- mainnet from-latest - #[tokio::test] - #[ignore] - async fn list_with_single_contract() { - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let list = find_index_files_by_pattern( - &s3_client, - INDEXED_DATA_FILES_BUCKET, - INDEXED_ACTIONS_FILES_FOLDER, - "hackathon.agency.near", - ) - .await - .unwrap(); - assert_eq!(list.len(), 1); - } - - /// cargo test s3::tests::list_with_csv_contracts -- mainnet from-latest - #[tokio::test] - #[ignore] - async fn list_with_csv_contracts() { - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let list = find_index_files_by_pattern( - &s3_client, - INDEXED_DATA_FILES_BUCKET, - INDEXED_ACTIONS_FILES_FOLDER, - "hackathon.agency.near, hackathon.aurora-silo-dev.near, hackathon.sputnik-dao.near", - ) - .await - .unwrap(); - assert!(list.len() >= 15); // expecting 15 but these contracts could get randomly called sometime - } - - /// cargo test s3::tests::list_with_wildcard_contracts -- mainnet from-latest - #[tokio::test] - #[ignore] - async fn list_with_wildcard_contracts() { - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let list = find_index_files_by_pattern( - &s3_client, - INDEXED_DATA_FILES_BUCKET, - INDEXED_ACTIONS_FILES_FOLDER, - "*.keypom.near", - ) - .await - .unwrap(); - assert!(list.len() >= 550); - } - - /// cargo test s3::tests::list_with_csv_and_wildcard_contracts -- mainnet from-latest - #[tokio::test] - #[ignore] - async fn list_with_csv_and_wildcard_contracts() { - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let list = find_index_files_by_pattern( - &s3_client, - INDEXED_DATA_FILES_BUCKET, - INDEXED_ACTIONS_FILES_FOLDER, - "*.keypom.near, hackathon.agency.near, *.nearcrowd.near", - ) - .await - .unwrap(); - assert!(list.len() > 1370); - } - - #[test] - fn storage_path_for_account_splits_and_reverses_into_folders() { - let account = "buildnear.testnet"; - let expected = "testnet/buildnear"; - let actual = super::storage_path_for_account(account); - assert_eq!(expected, actual); - - let account = "v2.keypom.near"; - let expected = "near/keypom/v2"; - let actual = super::storage_path_for_account(account); - assert_eq!(expected, actual); - - let account = "0.app5.hipodev.near"; - let expected = "near/hipodev/app5/0"; - let actual = super::storage_path_for_account(account); - assert_eq!(expected, actual); - } - - #[tokio::test] - #[ignore] - async fn handle_key_404() { - let mut success = false; - - let aws_config = aws_config::from_env().load().await; - let s3_client = aws_sdk_s3::Client::new(&aws_config); - - let s3_result = fetch_text_file_from_s3( - "near-lake-data-mainnet", - "does_not_exist/block.json".to_string(), - &s3_client, - ) - .await; - - if s3_result.is_err() { - let wrapped_error = s3_result.err().unwrap(); - let error = wrapped_error.root_cause(); - if error - .downcast_ref::() - .is_some() - { - success = true; - } else { - println!("Failed to downcast error: {:?}", error); - } - } - - assert!(success); - } -} From a621487ea68ea664daace90e5cd65d131a85e229 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 15:54:46 +1300 Subject: [PATCH 36/40] refactor: Make unused methods private --- block-streamer/src/delta_lake_client.rs | 2 +- block-streamer/src/main.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 67d91d72d..c7c894630 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -89,7 +89,7 @@ where Ok(results) } - pub async fn list_matching_index_files( + async fn list_matching_index_files( &self, contract_pattern: &str, ) -> anyhow::Result> { diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index d5da2a759..4402e4a9e 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -9,7 +9,6 @@ mod delta_lake_client; mod indexer_config; mod redis; mod rules; -mod s3; mod s3_client; pub(crate) const LOG_TARGET: &str = "block_streamer"; From 95697874f6916c9b40eabe00555d12b870a76510 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 16:18:55 +1300 Subject: [PATCH 37/40] chore: Add more logging --- block-streamer/src/block_streamer.rs | 30 ++++++++++++-------- block-streamer/src/delta_lake_client.rs | 13 +++++++++ block-streamer/src/rules/outcomes_reducer.rs | 6 ++-- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_streamer.rs index dade0b6ec..1efb62918 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_streamer.rs @@ -100,12 +100,13 @@ pub(crate) async fn start_block_stream( chain_id: &ChainId, ) -> anyhow::Result<()> { tracing::info!( - "Starting block stream from {start_block_height} for indexer: {}", + "Starting block stream at {start_block_height} for indexer: {}", indexer.get_full_name(), ); let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new(s3_client.clone()); + // TODO move to DeltaLakeClient let start_date = get_nearest_block_date(&s3_client, start_block_height, chain_id).await?; let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await?; @@ -116,6 +117,12 @@ pub(crate) async fn start_block_stream( affected_account_id, .. } => { + tracing::debug!( + "Fetching block heights starting from {} from delta lake for indexer: {}", + start_date.date_naive(), + indexer.get_full_name() + ); + // TODO Remove all block heights after start_block_height delta_lake_client .list_matching_block_heights(start_date, affected_account_id) .await @@ -128,7 +135,7 @@ pub(crate) async fn start_block_stream( } }?; - tracing::info!( + tracing::debug!( "Flushing {} block heights from index files to historical Stream for indexer: {}", blocks_from_index.len(), indexer.get_full_name(), @@ -137,6 +144,7 @@ pub(crate) async fn start_block_stream( for block in &blocks_from_index { crate::redis::xadd( redis_connection_manager, + // TODO make configurable crate::redis::generate_historical_stream_key(&indexer.get_full_name()), &[("block_height", block)], ) @@ -144,7 +152,7 @@ pub(crate) async fn start_block_stream( .context("Failed to add block to Redis Stream")?; } - let last_indexed_block = + let mut last_indexed_block = blocks_from_index .last() .map_or(last_indexed_block, |&last_block_in_index| { @@ -152,7 +160,7 @@ pub(crate) async fn start_block_stream( std::cmp::max(last_block_in_index, last_indexed_block) }); - tracing::info!( + tracing::debug!( "Starting near-lake-framework from {last_indexed_block} for indexer: {}", indexer.get_full_name(), ); @@ -167,9 +175,9 @@ pub(crate) async fn start_block_stream( let (sender, mut stream) = near_lake_framework::streamer(lake_config); - let mut filtered_block_count = 0; while let Some(streamer_message) = stream.recv().await { let block_height = streamer_message.block.header.height; + last_indexed_block = block_height; let matches = crate::rules::reduce_indexer_rule_matches( &indexer.indexer_rule, @@ -178,8 +186,6 @@ pub(crate) async fn start_block_stream( ); if !matches.is_empty() { - filtered_block_count += 1; - crate::redis::xadd( redis_connection_manager, crate::redis::generate_historical_stream_key(&indexer.get_full_name()), @@ -191,9 +197,9 @@ pub(crate) async fn start_block_stream( drop(sender); - tracing::info!( - "Flushed {} unindexed block heights to historical Stream for indexer: {}", - filtered_block_count, + tracing::debug!( + "Stopped block stream at {} for indexer: {}", + last_indexed_block, indexer.get_full_name(), ); @@ -311,7 +317,7 @@ mod tests { assert_eq!( block_date, - chrono::Utc.timestamp_nanos(1695921400989555820 as i64) + chrono::Utc.timestamp_nanos(1695921400989555820_i64) ); } @@ -386,7 +392,7 @@ mod tests { assert_eq!( block_date, - chrono::Utc.timestamp_nanos(1695921400989555820 as i64) + chrono::Utc.timestamp_nanos(1695921400989555820_i64) ); } diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index c7c894630..635a2c3c1 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -167,6 +167,13 @@ where .map(|key| async move { self.s3_client.get_text_file(DELTA_LAKE_BUCKET, &key).await }) .collect::>(); + tracing::debug!( + "Found {} index files matching {} after date {}", + futures.len(), + contract_pattern, + start_date + ); + let file_content_list = try_join_all(futures).await?; let mut block_heights: Vec<_> = file_content_list @@ -187,6 +194,12 @@ where block_heights.dedup(); } + tracing::debug!( + "Found {} matching block heights matching {}", + block_heights.len(), + contract_pattern, + ); + Ok(block_heights) } } diff --git a/block-streamer/src/rules/outcomes_reducer.rs b/block-streamer/src/rules/outcomes_reducer.rs index 72433c9b7..84d62ca11 100644 --- a/block-streamer/src/rules/outcomes_reducer.rs +++ b/block-streamer/src/rules/outcomes_reducer.rs @@ -43,7 +43,7 @@ fn build_indexer_rule_match( chain_id: ChainId, ) -> IndexerRuleMatch { IndexerRuleMatch { - chain_id: chain_id.clone(), + chain_id, indexer_rule_id: indexer_rule.id, indexer_rule_name: indexer_rule.name.clone(), payload: build_indexer_rule_match_payload( @@ -67,7 +67,7 @@ fn build_indexer_rule_match_payload( match &indexer_rule.matching_rule { MatchingRule::ActionAny { .. } | MatchingRule::ActionFunctionCall { .. } => { IndexerRuleMatchPayload::Actions { - block_hash: block_header_hash.to_string(), + block_hash: block_header_hash, receipt_id: receipt_execution_outcome.receipt.receipt_id.to_string(), transaction_hash, } @@ -101,7 +101,7 @@ fn build_indexer_rule_match_payload( .clone(); IndexerRuleMatchPayload::Events { - block_hash: block_header_hash.to_string(), + block_hash: block_header_hash, receipt_id: receipt_execution_outcome.receipt.receipt_id.to_string(), transaction_hash, event: event.event.clone(), From 9574e5451cbcd14035f36bc367c6c7f3836d48b3 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 16:40:51 +1300 Subject: [PATCH 38/40] refactor: Rename `BlockStreamer` to `BlockStream` --- block-streamer/src/{block_streamer.rs => block_stream.rs} | 4 ++-- block-streamer/src/main.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename block-streamer/src/{block_streamer.rs => block_stream.rs} (99%) diff --git a/block-streamer/src/block_streamer.rs b/block-streamer/src/block_stream.rs similarity index 99% rename from block-streamer/src/block_streamer.rs rename to block-streamer/src/block_stream.rs index 1efb62918..6dd818136 100644 --- a/block-streamer/src/block_streamer.rs +++ b/block-streamer/src/block_stream.rs @@ -13,11 +13,11 @@ pub struct Task { cancellation_token: tokio_util::sync::CancellationToken, } -pub struct BlockStreamer { +pub struct BlockStream { task: Option, } -impl BlockStreamer { +impl BlockStream { pub fn new() -> Self { Self { task: None } } diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 4402e4a9e..15da8ebad 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -4,7 +4,7 @@ use crate::indexer_config::IndexerConfig; use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; -mod block_streamer; +mod block_stream; mod delta_lake_client; mod indexer_config; mod redis; @@ -48,7 +48,7 @@ async fn main() -> anyhow::Result<()> { indexer_rule: filter_rule, }; - let mut streamer = block_streamer::BlockStreamer::new(); + let mut streamer = block_stream::BlockStream::new(); streamer.start( 106000000, From 0531b5ff9a5d7eb1ff243dbdc122097252a98f88 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Fri, 24 Nov 2023 21:05:42 +1300 Subject: [PATCH 39/40] refactor: Encapsulate block date logic within `DeltaLakeClient` --- block-streamer/src/block_stream.rs | 232 +------------------- block-streamer/src/delta_lake_client.rs | 276 ++++++++++++++++++++---- block-streamer/src/main.rs | 4 +- 3 files changed, 246 insertions(+), 266 deletions(-) diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs index 6dd818136..56794ba46 100644 --- a/block-streamer/src/block_stream.rs +++ b/block-streamer/src/block_stream.rs @@ -27,7 +27,7 @@ impl BlockStream { start_block_height: near_indexer_primitives::types::BlockHeight, indexer: IndexerConfig, redis_connection_manager: crate::redis::ConnectionManager, - s3_client: crate::s3_client::S3Client, + delta_lake_client: crate::delta_lake_client::DeltaLakeClient, chain_id: ChainId, ) -> anyhow::Result<()> { if self.task.is_some() { @@ -51,7 +51,7 @@ impl BlockStream { start_block_height, indexer.clone(), &redis_connection_manager, - s3_client, + &delta_lake_client, &chain_id, ) => { result.map_err(|err| { @@ -96,7 +96,7 @@ pub(crate) async fn start_block_stream( start_block_height: near_indexer_primitives::types::BlockHeight, indexer: IndexerConfig, redis_connection_manager: &crate::redis::ConnectionManager, - s3_client: crate::s3_client::S3Client, + delta_lake_client: &crate::delta_lake_client::DeltaLakeClient, chain_id: &ChainId, ) -> anyhow::Result<()> { tracing::info!( @@ -104,13 +104,10 @@ pub(crate) async fn start_block_stream( indexer.get_full_name(), ); - let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new(s3_client.clone()); - - // TODO move to DeltaLakeClient - let start_date = get_nearest_block_date(&s3_client, start_block_height, chain_id).await?; - let latest_block_metadata = delta_lake_client.get_latest_block_metadata().await?; - let last_indexed_block = latest_block_metadata.last_indexed_block.parse::()?; + let last_indexed_block = latest_block_metadata + .last_indexed_block + .parse::()?; let blocks_from_index = match &indexer.indexer_rule.matching_rule { MatchingRule::ActionAny { @@ -119,12 +116,12 @@ pub(crate) async fn start_block_stream( } => { tracing::debug!( "Fetching block heights starting from {} from delta lake for indexer: {}", - start_date.date_naive(), + start_block_height, indexer.get_full_name() ); - // TODO Remove all block heights after start_block_height + delta_lake_client - .list_matching_block_heights(start_date, affected_account_id) + .list_matching_block_heights(start_block_height, affected_account_id) .await } MatchingRule::ActionFunctionCall { .. } => { @@ -205,214 +202,3 @@ pub(crate) async fn start_block_stream( Ok(()) } - -pub async fn get_nearest_block_date( - s3_client: &impl crate::s3_client::S3ClientTrait, - block_height: u64, - chain_id: &ChainId, -) -> anyhow::Result> { - let bucket = match chain_id { - ChainId::Mainnet => "near-lake-data-mainnet", - ChainId::Testnet => "near-lake-data-testnet", - }; - - let mut current_block_height = block_height; - let mut retry_count = 1; - loop { - let block_key = format!("{:0>12}/block.json", current_block_height); - match s3_client.get_text_file(bucket, &block_key).await { - Ok(text) => { - let block: near_indexer_primitives::views::BlockView = serde_json::from_str(&text)?; - return Ok(chrono::Utc.timestamp_nanos(block.header.timestamp_nanosec as i64)); - } - - Err(e) => { - if e.root_cause() - .downcast_ref::() - .is_some() - { - retry_count += 1; - if retry_count > MAX_S3_RETRY_COUNT { - anyhow::bail!("Exceeded maximum retries to fetch block from S3"); - } - - tracing::debug!( - "Block {} not found on S3, attempting to fetch next block", - current_block_height - ); - current_block_height += 1; - continue; - } - - return Err(e).context("Failed to fetch block from S3"); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use mockall::predicate; - - mod get_near_block_date { - use super::*; - - #[tokio::test] - async fn gets_the_date_of_the_closest_block() { - let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); - - mock_s3_client - .expect_get_text_file() - .with( - predicate::eq("near-lake-data-mainnet"), - predicate::eq("000106397175/block.json"), - ) - .times(1) - .returning(|_, _| { - Ok(r#"{ - "author": "someone", - "header": { - "approvals": [], - "block_merkle_root": "ERiC7AJ2zbVz1HJHThR5NWDDN9vByhwdjcVfivmpY5B", - "block_ordinal": 92102682, - "challenges_result": [], - "challenges_root": "11111111111111111111111111111111", - "chunk_headers_root": "MDiJxDyvUQaZRKmUwa5jgQuV6XjwVvnm4tDrajCxwvz", - "chunk_mask": [], - "chunk_receipts_root": "n84wEo7kTKTCJsyqBZ2jndhjrAMeJAXMwKvnJR7vCuy", - "chunk_tx_root": "D8j64GMKBMvUfvnuHtWUyDtMHM5mJ2pA4G5VmYYJvo5G", - "chunks_included": 4, - "epoch_id": "2RMQiomr6CSSwUWpmB62YohxHbfadrHfcsaa3FVb4J9x", - "epoch_sync_data_hash": null, - "gas_price": "100000000", - "hash": "FA1z9RVm9fX3g3mgP3NToZGwWeeXYn8bvZs4nwwTgCpD", - "height": 102162333, - "last_ds_final_block": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", - "last_final_block": "8xkwjn6Lb6UhMBhxcbVQBf3318GafkdaXoHA8Jako1nn", - "latest_protocol_version": 62, - "next_bp_hash": "dmW84aEj2iVJMLwJodJwTfAyeA1LJaHEthvnoAsvTPt", - "next_epoch_id": "C9TDDYthANoduoTBZS7WYDsBSe9XCm4M2F9hRoVXVXWY", - "outcome_root": "6WxzWLVp4b4bFbxHzu18apVfXLvHGKY7CHoqD2Eq3TFJ", - "prev_hash": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", - "prev_height": 102162332, - "prev_state_root": "Aq2ndkyDiwroUWN69Ema9hHtnr6dPHoEBRNyfmd8v4gB", - "random_value": "7ruuMyDhGtTkYaCGYMy7PirPiM79DXa8GhVzQW1pHRoz", - "rent_paid": "0", - "signature": "ed25519:5gYYaWHkAEK5etB8tDpw7fmehkoYSprUxKPygaNqmhVDFCMkA1n379AtL1BBkQswLAPxWs1BZvypFnnLvBtHRknm", - "timestamp": 1695921400989555700, - "timestamp_nanosec": "1695921400989555820", - "total_supply": "1155783047679681223245725102954966", - "validator_proposals": [], - "validator_reward": "0" - }, - "chunks": [] - }"# - .to_string()) - }); - - let block_date = get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet) - .await - .unwrap(); - - assert_eq!( - block_date, - chrono::Utc.timestamp_nanos(1695921400989555820_i64) - ); - } - - #[tokio::test] - async fn retires_if_a_block_doesnt_exist() { - let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); - - mock_s3_client - .expect_get_text_file() - .with( - predicate::eq("near-lake-data-mainnet"), - predicate::eq("000106397175/block.json"), - ) - .times(1) - .returning(|_, _| { - Err(anyhow::anyhow!( - aws_sdk_s3::types::error::NoSuchKey::builder().build() - )) - }); - mock_s3_client - .expect_get_text_file() - .with( - predicate::eq("near-lake-data-mainnet"), - predicate::eq("000106397176/block.json"), - ) - .times(1) - .returning(|_, _| { - Ok(r#"{ - "author": "someone", - "header": { - "approvals": [], - "block_merkle_root": "ERiC7AJ2zbVz1HJHThR5NWDDN9vByhwdjcVfivmpY5B", - "block_ordinal": 92102682, - "challenges_result": [], - "challenges_root": "11111111111111111111111111111111", - "chunk_headers_root": "MDiJxDyvUQaZRKmUwa5jgQuV6XjwVvnm4tDrajCxwvz", - "chunk_mask": [], - "chunk_receipts_root": "n84wEo7kTKTCJsyqBZ2jndhjrAMeJAXMwKvnJR7vCuy", - "chunk_tx_root": "D8j64GMKBMvUfvnuHtWUyDtMHM5mJ2pA4G5VmYYJvo5G", - "chunks_included": 4, - "epoch_id": "2RMQiomr6CSSwUWpmB62YohxHbfadrHfcsaa3FVb4J9x", - "epoch_sync_data_hash": null, - "gas_price": "100000000", - "hash": "FA1z9RVm9fX3g3mgP3NToZGwWeeXYn8bvZs4nwwTgCpD", - "height": 102162333, - "last_ds_final_block": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", - "last_final_block": "8xkwjn6Lb6UhMBhxcbVQBf3318GafkdaXoHA8Jako1nn", - "latest_protocol_version": 62, - "next_bp_hash": "dmW84aEj2iVJMLwJodJwTfAyeA1LJaHEthvnoAsvTPt", - "next_epoch_id": "C9TDDYthANoduoTBZS7WYDsBSe9XCm4M2F9hRoVXVXWY", - "outcome_root": "6WxzWLVp4b4bFbxHzu18apVfXLvHGKY7CHoqD2Eq3TFJ", - "prev_hash": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", - "prev_height": 102162332, - "prev_state_root": "Aq2ndkyDiwroUWN69Ema9hHtnr6dPHoEBRNyfmd8v4gB", - "random_value": "7ruuMyDhGtTkYaCGYMy7PirPiM79DXa8GhVzQW1pHRoz", - "rent_paid": "0", - "signature": "ed25519:5gYYaWHkAEK5etB8tDpw7fmehkoYSprUxKPygaNqmhVDFCMkA1n379AtL1BBkQswLAPxWs1BZvypFnnLvBtHRknm", - "timestamp": 1695921400989555700, - "timestamp_nanosec": "1695921400989555820", - "total_supply": "1155783047679681223245725102954966", - "validator_proposals": [], - "validator_reward": "0" - }, - "chunks": [] - }"# - .to_string()) - }); - - let block_date = get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet) - .await - .unwrap(); - - assert_eq!( - block_date, - chrono::Utc.timestamp_nanos(1695921400989555820_i64) - ); - } - - #[tokio::test] - async fn exits_if_maximum_retries_exceeded() { - let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); - - mock_s3_client - .expect_get_text_file() - .times(MAX_S3_RETRY_COUNT as usize) - .returning(|_, _| { - Err(anyhow::anyhow!( - aws_sdk_s3::types::error::NoSuchKey::builder().build() - )) - }); - - let result = - get_nearest_block_date(&mock_s3_client, 106397175, &ChainId::Mainnet).await; - - assert!(result.is_err()); - } - } -} diff --git a/block-streamer/src/delta_lake_client.rs b/block-streamer/src/delta_lake_client.rs index 635a2c3c1..26fd2b144 100644 --- a/block-streamer/src/delta_lake_client.rs +++ b/block-streamer/src/delta_lake_client.rs @@ -1,9 +1,11 @@ +use crate::rules::types::indexer_rule_match::ChainId; use anyhow::Context; -use chrono::{DateTime, NaiveDate, Utc}; +use chrono::TimeZone; use futures::future::try_join_all; use near_lake_framework::near_indexer_primitives; const DELTA_LAKE_BUCKET: &str = "near-delta-lake"; +const MAX_S3_RETRY_COUNT: u8 = 20; const INDEXED_ACTIONS_PREFIX: &str = "silver/accounts/action_receipt_actions/metadata"; const LATEST_BLOCK_METADATA_KEY: &str = "silver/accounts/action_receipt_actions/metadata/latest_block.json"; @@ -34,6 +36,7 @@ where T: crate::s3_client::S3ClientTrait, { s3_client: T, + chain_id: ChainId, } impl DeltaLakeClient @@ -41,7 +44,11 @@ where T: crate::s3_client::S3ClientTrait, { pub fn new(s3_client: T) -> Self { - DeltaLakeClient { s3_client } + DeltaLakeClient { + s3_client, + // hardcode to mainnet for + chain_id: ChainId::Mainnet, + } } pub async fn get_latest_block_metadata(&self) -> anyhow::Result { @@ -54,6 +61,56 @@ where .context("Unable to parse Metadata") } + fn get_lake_bucket(&self) -> String { + match self.chain_id { + ChainId::Mainnet => "near-lake-data-mainnet".to_string(), + ChainId::Testnet => "near-lake-data-testnet".to_string(), + } + } + + pub async fn get_nearest_block_date( + &self, + block_height: u64, + ) -> anyhow::Result> { + let mut current_block_height = block_height; + let mut retry_count = 1; + loop { + let block_key = format!("{:0>12}/block.json", current_block_height); + match self + .s3_client + .get_text_file(&self.get_lake_bucket(), &block_key) + .await + { + Ok(text) => { + let block: near_indexer_primitives::views::BlockView = + serde_json::from_str(&text)?; + return Ok(chrono::Utc.timestamp_nanos(block.header.timestamp_nanosec as i64)); + } + + Err(e) => { + if e.root_cause() + .downcast_ref::() + .is_some() + { + retry_count += 1; + if retry_count > MAX_S3_RETRY_COUNT { + anyhow::bail!("Exceeded maximum retries to fetch block from S3"); + } + + tracing::debug!( + "Block {} not found on S3, attempting to fetch next block", + current_block_height + ); + current_block_height += 1; + continue; + } + + return Err(e).context("Failed to fetch block from S3"); + } + } + } + } + fn s3_prefix_from_contract_id(&self, contract_id: &str) -> String { let mut folders = contract_id.split('.').collect::>(); folders.reverse(); @@ -138,17 +195,19 @@ where } } - fn date_from_s3_path(&self, path: &str) -> Option { + fn date_from_s3_path(&self, path: &str) -> Option { let file_name_date = path.split('/').last()?.replace(".json", ""); - NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d").ok() + chrono::NaiveDate::parse_from_str(&file_name_date, "%Y-%m-%d").ok() } pub async fn list_matching_block_heights( &self, - start_date: DateTime, + start_block_height: near_indexer_primitives::types::BlockHeight, contract_pattern: &str, ) -> anyhow::Result> { + let start_date = self.get_nearest_block_date(start_block_height).await?; + let file_list = self.list_matching_index_files(contract_pattern).await?; tracing::debug!( "Found {} index files matching {}", @@ -200,6 +259,7 @@ where contract_pattern, ); + // TODO Remove all block heights after start_block_height Ok(block_heights) } } @@ -209,6 +269,57 @@ mod tests { use super::*; use mockall::predicate; + fn generate_block_with_timestamp(date: &str) -> String { + let naive_date = chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d") + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap(); + + let date_time_utc = chrono::Utc.from_utc_datetime(&naive_date).timestamp() * 1_000_000_000; + + format!( + r#"{{ + "author": "someone", + "header": {{ + "approvals": [], + "block_merkle_root": "ERiC7AJ2zbVz1HJHThR5NWDDN9vByhwdjcVfivmpY5B", + "block_ordinal": 92102682, + "challenges_result": [], + "challenges_root": "11111111111111111111111111111111", + "chunk_headers_root": "MDiJxDyvUQaZRKmUwa5jgQuV6XjwVvnm4tDrajCxwvz", + "chunk_mask": [], + "chunk_receipts_root": "n84wEo7kTKTCJsyqBZ2jndhjrAMeJAXMwKvnJR7vCuy", + "chunk_tx_root": "D8j64GMKBMvUfvnuHtWUyDtMHM5mJ2pA4G5VmYYJvo5G", + "chunks_included": 4, + "epoch_id": "2RMQiomr6CSSwUWpmB62YohxHbfadrHfcsaa3FVb4J9x", + "epoch_sync_data_hash": null, + "gas_price": "100000000", + "hash": "FA1z9RVm9fX3g3mgP3NToZGwWeeXYn8bvZs4nwwTgCpD", + "height": 102162333, + "last_ds_final_block": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "last_final_block": "8xkwjn6Lb6UhMBhxcbVQBf3318GafkdaXoHA8Jako1nn", + "latest_protocol_version": 62, + "next_bp_hash": "dmW84aEj2iVJMLwJodJwTfAyeA1LJaHEthvnoAsvTPt", + "next_epoch_id": "C9TDDYthANoduoTBZS7WYDsBSe9XCm4M2F9hRoVXVXWY", + "outcome_root": "6WxzWLVp4b4bFbxHzu18apVfXLvHGKY7CHoqD2Eq3TFJ", + "prev_hash": "Ax2a3MSYuv2hgybnCbpNJMdYmPrHDHdA2hHTUrBkD915", + "prev_height": 102162332, + "prev_state_root": "Aq2ndkyDiwroUWN69Ema9hHtnr6dPHoEBRNyfmd8v4gB", + "random_value": "7ruuMyDhGtTkYaCGYMy7PirPiM79DXa8GhVzQW1pHRoz", + "rent_paid": "0", + "signature": "ed25519:5gYYaWHkAEK5etB8tDpw7fmehkoYSprUxKPygaNqmhVDFCMkA1n379AtL1BBkQswLAPxWs1BZvypFnnLvBtHRknm", + "timestamp": 1695921400989555700, + "timestamp_nanosec": "{}", + "total_supply": "1155783047679681223245725102954966", + "validator_proposals": [], + "validator_reward": "0" + }}, + "chunks": [] + }}"#, + date_time_utc + ) + } + #[tokio::test] async fn fetches_metadata_from_s3() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); @@ -238,6 +349,13 @@ mod tests { async fn lists_block_heights_for_single_contract() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000091940840/block.json"), + ) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2023-05-16"))); mock_s3_client .expect_list_all_objects() .returning(|_bucket, _prefix| { @@ -258,14 +376,7 @@ mod tests { let delta_lake_client = DeltaLakeClient::new(mock_s3_client); let block_heights = delta_lake_client - .list_matching_block_heights( - NaiveDate::from_ymd_opt(2023, 5, 16) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .and_utc(), - "queryapi.dataplatform.near", - ) + .list_matching_block_heights(91940840, "queryapi.dataplatform.near") .await .unwrap(); @@ -276,6 +387,13 @@ mod tests { async fn lists_block_heights_for_multiple_contracts() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000045894617/block.json"), + ) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2022-05-26"))); mock_s3_client .expect_list_all_objects() .returning(|_bucket, prefix| { @@ -316,11 +434,7 @@ mod tests { let block_heights = delta_lake_client .list_matching_block_heights( - NaiveDate::from_ymd_opt(2022, 5, 26) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .and_utc(), + 45894617, "hackathon.agency.near, hackathon.aurora-silo-dev.near, hackathon.sputnik-dao.near", ) .await @@ -341,6 +455,13 @@ mod tests { async fn lists_block_heights_for_wildcard() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000078516467/block.json"), + ) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2023-05-26"))); mock_s3_client .expect_list_all_objects() .returning(|_bucket, prefix| { @@ -395,14 +516,7 @@ mod tests { let delta_lake_client = DeltaLakeClient::new(mock_s3_client); let block_heights = delta_lake_client - .list_matching_block_heights( - NaiveDate::from_ymd_opt(2023, 5, 26) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .and_utc(), - "*.keypom.near", - ) + .list_matching_block_heights(78516467, "*.keypom.near") .await .unwrap(); @@ -416,6 +530,13 @@ mod tests { async fn lists_block_heights_for_multiple_contracts_and_wildcard() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000045894617/block.json"), + ) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2021-05-26"))); mock_s3_client .expect_list_all_objects() .returning(|_bucket, prefix| { @@ -457,14 +578,7 @@ mod tests { let delta_lake_client = DeltaLakeClient::new(mock_s3_client); let block_heights = delta_lake_client - .list_matching_block_heights( - NaiveDate::from_ymd_opt(2021, 5, 26) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .and_utc(), - "*.keypom.near, hackathon.agency.near", - ) + .list_matching_block_heights(45894617, "*.keypom.near, hackathon.agency.near") .await .unwrap(); @@ -481,6 +595,13 @@ mod tests { async fn sorts_and_removes_duplicates_for_multiple_contracts() { let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000045894628/block.json"), + ) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2021-05-26"))); mock_s3_client .expect_list_all_objects() .returning(|_bucket, prefix| { @@ -509,14 +630,7 @@ mod tests { let delta_lake_client = DeltaLakeClient::new(mock_s3_client); let block_heights = delta_lake_client - .list_matching_block_heights( - NaiveDate::from_ymd_opt(2021, 5, 26) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap() - .and_utc(), - "keypom.near, hackathon.agency.near", - ) + .list_matching_block_heights(45894628, "keypom.near, hackathon.agency.near") .await .unwrap(); @@ -525,4 +639,82 @@ mod tests { vec![45894617, 45894627, 45894628, 45894712, 45898413, 45898423, 45898424] ) } + + #[tokio::test] + async fn gets_the_date_of_the_closest_block() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397175/block.json"), + ) + .times(1) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2021-05-26"))); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_date = delta_lake_client + .get_nearest_block_date(106397175) + .await + .unwrap(); + + assert_eq!(block_date, chrono::Utc.timestamp_nanos(1621987200000000000)); + } + + #[tokio::test] + async fn retires_if_a_block_doesnt_exist() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397175/block.json"), + ) + .times(1) + .returning(|_, _| { + Err(anyhow::anyhow!( + aws_sdk_s3::types::error::NoSuchKey::builder().build() + )) + }); + mock_s3_client + .expect_get_text_file() + .with( + predicate::eq("near-lake-data-mainnet"), + predicate::eq("000106397176/block.json"), + ) + .times(1) + .returning(|_bucket, _prefix| Ok(generate_block_with_timestamp("2021-05-26"))); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let block_date = delta_lake_client + .get_nearest_block_date(106397175) + .await + .unwrap(); + + assert_eq!(block_date, chrono::Utc.timestamp_nanos(1621987200000000000)); + } + + #[tokio::test] + async fn exits_if_maximum_retries_exceeded() { + let mut mock_s3_client = crate::s3_client::MockS3ClientTrait::new(); + + mock_s3_client + .expect_get_text_file() + .times(MAX_S3_RETRY_COUNT as usize) + .returning(|_, _| { + Err(anyhow::anyhow!( + aws_sdk_s3::types::error::NoSuchKey::builder().build() + )) + }); + + let delta_lake_client = DeltaLakeClient::new(mock_s3_client); + + let result = delta_lake_client.get_nearest_block_date(106397175).await; + + assert!(result.is_err()); + } } diff --git a/block-streamer/src/main.rs b/block-streamer/src/main.rs index 15da8ebad..c6efd3b51 100644 --- a/block-streamer/src/main.rs +++ b/block-streamer/src/main.rs @@ -27,6 +27,8 @@ async fn main() -> anyhow::Result<()> { let aws_config = aws_config::from_env().load().await; let s3_client = crate::s3_client::S3Client::new(&aws_config); + let delta_lake_client = crate::delta_lake_client::DeltaLakeClient::new(s3_client); + let contract = "queryapi.dataplatform.near"; let matching_rule = MatchingRule::ActionAny { affected_account_id: contract.to_string(), @@ -54,7 +56,7 @@ async fn main() -> anyhow::Result<()> { 106000000, indexer, redis_connection_manager, - s3_client, + delta_lake_client, ChainId::Mainnet, )?; From 21706bdbb7399e00a1663779e64bae89fc056f58 Mon Sep 17 00:00:00 2001 From: Morgan Mccauley Date: Mon, 27 Nov 2023 09:10:10 +1300 Subject: [PATCH 40/40] chore: Remove unused code --- block-streamer/src/block_stream.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs index 56794ba46..39bb2b4e6 100644 --- a/block-streamer/src/block_stream.rs +++ b/block-streamer/src/block_stream.rs @@ -2,12 +2,9 @@ use crate::indexer_config::IndexerConfig; use crate::rules::types::indexer_rule_match::ChainId; use crate::rules::MatchingRule; use anyhow::{bail, Context}; -use chrono::TimeZone; use near_lake_framework::near_indexer_primitives; use tokio::task::JoinHandle; -pub const MAX_S3_RETRY_COUNT: u8 = 20; - pub struct Task { handle: JoinHandle>, cancellation_token: tokio_util::sync::CancellationToken,