From b492d87ec4457beeab6b6017e91ea2ce727bcb20 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Tue, 23 Apr 2024 09:12:44 +0200 Subject: [PATCH 1/3] chore: version bump to v0.12.0 --- CHANGELOG.md | 8 ++++++++ Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff353ddfb..57236c864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.12.0] - 2024-04-23 + ### Added - Pathfinder now supports storing only the latest state of the Merkle tries. This can be enabled by specifying '--storage.prune-state-tries true' on the command line when creating a new database. @@ -16,11 +18,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pruned merkle tries take significantly less disk space than full ones. - Pathfinder stores this setting in its database and defaults to using that. - Once set pruning cannot be enabled/disabled for non-empty databases. + - Pruning achieves a ~75% reduction in overall storage compared to archive. ### Removed - Support for Goerli testnet +### Changed + +- Improved compactness of state diff and transaction data, resulting in noticeable storage savings. + - Roughly a 20% reduction for a state trie archive node, or a 45% reduction for a pruned variant. + ## [0.11.6] - 2024-04-10 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 6f0fba3f1..201d43150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6007,7 +6007,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "p2p" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "async-stream", @@ -6043,7 +6043,7 @@ dependencies = [ [[package]] name = "p2p_proto" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "fake", @@ -6062,7 +6062,7 @@ dependencies = [ [[package]] name = "p2p_proto_derive" -version = "0.11.6" +version = "0.12.0" dependencies = [ "proc-macro2", "quote", @@ -6071,7 +6071,7 @@ dependencies = [ [[package]] name = "p2p_stream" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "async-trait", @@ -6188,7 +6188,7 @@ checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "pathfinder" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "assert_matches", @@ -6253,7 +6253,7 @@ dependencies = [ [[package]] name = "pathfinder-common" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "bitvec", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "pathfinder-compiler" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "cairo-lang-starknet 1.0.0-alpha.6", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "pathfinder-crypto" -version = "0.11.6" +version = "0.12.0" dependencies = [ "ark-ff", "assert_matches", @@ -6310,7 +6310,7 @@ dependencies = [ [[package]] name = "pathfinder-ethereum" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "async-trait", @@ -6330,7 +6330,7 @@ dependencies = [ [[package]] name = "pathfinder-executor" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "blockifier", @@ -6350,7 +6350,7 @@ dependencies = [ [[package]] name = "pathfinder-merkle-tree" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "bitvec", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "pathfinder-retry" -version = "0.11.6" +version = "0.12.0" dependencies = [ "tokio", "tokio-retry", @@ -6374,7 +6374,7 @@ dependencies = [ [[package]] name = "pathfinder-rpc" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "assert_matches", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "pathfinder-serde" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "num-bigint 0.4.4", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "pathfinder-storage" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "assert_matches", @@ -8083,7 +8083,7 @@ dependencies = [ [[package]] name = "starknet-gateway-client" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "assert_matches", @@ -8117,7 +8117,7 @@ dependencies = [ [[package]] name = "starknet-gateway-test-fixtures" -version = "0.11.6" +version = "0.12.0" dependencies = [ "pathfinder-common", "pathfinder-crypto", @@ -8125,7 +8125,7 @@ dependencies = [ [[package]] name = "starknet-gateway-types" -version = "0.11.6" +version = "0.12.0" dependencies = [ "anyhow", "assert_matches", diff --git a/Cargo.toml b/Cargo.toml index ba79ee926..3c49a7c93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ lto = true opt-level = 3 [workspace.package] -version = "0.11.6" +version = "0.12.0" edition = "2021" license = "MIT OR Apache-2.0" rust-version = "1.76" From 9f370cb778588112f411f870c4ac1a6af0aca171 Mon Sep 17 00:00:00 2001 From: Mirko von Leipzig Date: Tue, 23 Apr 2024 09:28:40 +0200 Subject: [PATCH 2/3] docs(readme): document state trie pruning --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index f6299b936..a75781a7f 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,38 @@ sudo docker run --rm eqlabs/pathfinder:latest --help Block times on `mainnet` can be prohibitively long for certain applications. As a workaround, Starknet added the concept of a `pending` block which is the block currently under construction. This is supported by pathfinder, and usage is documented in the [JSON-RPC API](#json-rpc-api) with various methods accepting `"block_id"="pending"`. +### State trie pruning + +Pathfinder allows you to control the number of blocks of state trie history to preserve. You can choose between archive: + +``` +--storage.state-tries = archive +``` + +which stores all of history, or to keep only the last `k+1` blocks: + +``` +--storage.state-tries = k +``` + +The latest block is always stored, though in the future we plan an option to disable this entirely. Currently at least +one block is required to trustlessly verify Starknet's state update. + +State trie data consumes a massive amount of storage space. You can expect an overall storage reduction of ~75% when going +from archive to pruned mode. + +The downside to pruning this data is that storage proofs are only available for blocks that are not pruned i.e. with +`--storage.state-tries = k` you can only serve storage proofs for the latest `k+1` blocks. + +Note that this only impacts storage proofs - for all other considerations pathfinder is still an archive mode and no +data is dropped. + +Also note that you cannot switch between archive and pruned modes. You may however change `k` between different runs of +pathfinder. + +If you don't care about storage proofs, you can maximise storage savings by setting `--storage.state-tries = 0`, which +will only store the latest block's state trie. + ### Logging Logging can be configured using the `RUST_LOG` environment variable. From e730dc80b46f2f6050bdbec9f87a953463960278 Mon Sep 17 00:00:00 2001 From: Krisztian Kovacs Date: Tue, 23 Apr 2024 13:00:31 +0200 Subject: [PATCH 3/3] chore(README): add new 0.12.0 snapshots --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a75781a7f..9eced69d1 100644 --- a/README.md +++ b/README.md @@ -175,10 +175,14 @@ This produces uncompressed database file `testnet-sepolia.sqlite` that can then ### Available database snapshots -| Network | Block | Pathfinder version required | Filename | Download URL | Compressed size | SHA2-256 checksum of compressed file | -| --------------- | ------ | --------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------------------------ | -| Sepolia testnet | 47191 | >= 0.11.0 | `sepolia-testnet_0.11.0_47191.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/sepolia-testnet_0.11.0_47191.sqlite.zst) | 1.91 GB | `82704d8382bac460550c3d31dd3c1f4397c4c43a90fb0e38110b0cd07cd94831` | -| mainnet | 595424 | >= 0.11.0 | `mainnet_0.11.0_595424.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/mainnet_0.11.0_595424.sqlite.zst) | 469.63 GB | `e42bae71c97c1a403116a7362f15f5180b19e8cc647efb1357f1ae8924dce654` | +| Network | Block | Pathfinder version required | Mode | Filename | Download URL | Compressed size | SHA2-256 checksum of compressed file | +| --------------- | ------ | --------------------------- | ------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------------------------------------------------ | +| Sepolia testnet | 47191 | >= 0.11.0 | archive | `sepolia-testnet_0.11.0_47191.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/sepolia-testnet_0.11.0_47191.sqlite.zst) | 1.91 GB | `82704d8382bac460550c3d31dd3c1f4397c4c43a90fb0e38110b0cd07cd94831` | +| Sepolia testnet | 61322 | >= 0.12.0 | archive | `sepolia-testnet_0.12.0_61322_archive.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/sepolia-testnet_0.12.0_61322_archive.sqlite.zst) | 3.56 GB | `d25aa259ce62bb4b2e3ff49d243217799c99cd8b7e594a7bb24d4c091d980828` | +| Sepolia testnet | 61322 | >= 0.12.0 | pruned | `sepolia-testnet_0.12.0_61322_pruned.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/sepolia-testnet_0.12.0_61322_pruned.sqlite.zst) | 1.26 GB | `f2da766a8f8be93170997b3e5f268c0146aec1147c8ec569d0d6fdd5cd9bc3f1` | +| Mainnet | 595424 | >= 0.11.0 | archive | `mainnet_0.11.0_595424.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/mainnet_0.11.0_595424.sqlite.zst) | 469.63 GB | `e42bae71c97c1a403116a7362f15f5180b19e8cc647efb1357f1ae8924dce654` | +| Mainnet | 635054 | >= 0.12.0 | archive | `mainnet_0.12.0_635054_archive.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/mainnet_0.12.0_635054_archive.sqlite.zst) | 383.86 GB | `d401902684cecaae4a88d6c0219498a0da1bbdb3334ea5b91e3a16212db9ee43` | +| Mainnet | 635054 | >= 0.12.0 | pruned | `mainnet_0.12.0_635054_pruned.sqlite.zst` | [Download](https://pub-1fac64c3c0334cda85b45bcc02635c32.r2.dev/mainnet_0.12.0_635054_pruned.sqlite.zst) | 59.89 GB | `1d854423278611b414130ac05f486c66ef475f47a1c930c2af5296c9906f9ae0` | ## Configuration