From 05bbf3dacd2729d3a8f78e04e35e966ed74f4adf Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Thu, 9 Jan 2025 12:19:25 +0800 Subject: [PATCH 1/3] feat(storage): switch to eviction time disk cache insertion Signed-off-by: MrCroxx --- Cargo.lock | 25 ++++++++----------------- Cargo.toml | 12 ++++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2547112a44e3..94f365c92d058 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4930,9 +4930,8 @@ dependencies = [ [[package]] name = "foyer" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c63beb108769d28b042829164139a5245359ccfdb4a8face928ac154e2c9ed" +version = "0.14.0-dev" +source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" dependencies = [ "ahash 0.8.11", "anyhow", @@ -4943,22 +4942,19 @@ dependencies = [ "foyer-storage", "futures", "madsim-tokio", - "pin-project", "tracing", ] [[package]] name = "foyer-common" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68be583b3f51bcbc6b72f40a5aea6fa6aaff10692649c08b20458bb6703f79f0" +version = "0.14.0-dev" +source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" dependencies = [ "ahash 0.8.11", "bytes", "cfg-if", "fastrace", "futures", - "hashbrown 0.15.0", "itertools 0.13.0", "madsim-tokio", "parking_lot 0.12.1", @@ -4978,9 +4974,8 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a01e64de1452409977219fe014caa427874677aacd647bec9207d98d018bcd8" +version = "0.14.0-dev" +source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" dependencies = [ "ahash 0.8.11", "bitflags 2.6.0", @@ -4994,7 +4989,6 @@ dependencies = [ "itertools 0.13.0", "madsim-tokio", "parking_lot 0.12.1", - "paste", "pin-project", "serde", "thiserror 2.0.3", @@ -5003,9 +4997,8 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb89bdcdf355e8cb2b29eaa0158839f4f7c6974c6cc86f6e54e4a48a2afa5b8" +version = "0.14.0-dev" +source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" dependencies = [ "ahash 0.8.11", "allocator-api2", @@ -5014,7 +5007,6 @@ dependencies = [ "async-channel 2.2.1", "auto_enums", "bincode 1.3.3", - "bitflags 2.6.0", "bytes", "clap", "either", @@ -5025,7 +5017,6 @@ dependencies = [ "foyer-memory", "fs4", "futures", - "hashbrown 0.15.0", "itertools 0.13.0", "libc", "lz4", diff --git a/Cargo.toml b/Cargo.toml index 771601e38e8a0..1954eb305a657 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,12 @@ normal = ["workspace-hack"] development = ["expect-test", "pretty_assertions"] [workspace.dependencies] -foyer = { version = "0.13.1", features = ["tracing", "nightly", "prometheus"] } +# foyer = { version = "0.13.1", features = ["tracing", "nightly", "prometheus"] } +foyer = { git = "https://github.com/foyer-rs/foyer", rev = "0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96", features = [ + "tracing", + "nightly", + "prometheus", +] } apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "25113ba88234a9ae23296e981d8302c290fdaa4b", features = [ "snappy", "zstandard", @@ -154,7 +159,10 @@ icelake = { git = "https://github.com/risingwavelabs/icelake.git", rev = "0ec44f "prometheus", ] } # branch dev_rebase_main_20241230 -iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "683fb89edeaf8d1baae69e1f376d68b92be1d496", features = ["storage-s3", "storage-gcs"] } +iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "683fb89edeaf8d1baae69e1f376d68b92be1d496", features = [ + "storage-s3", + "storage-gcs", +] } iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "683fb89edeaf8d1baae69e1f376d68b92be1d496" } iceberg-catalog-glue = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "683fb89edeaf8d1baae69e1f376d68b92be1d496" } opendal = "0.49" From cf938c1c2c128befdfecd0c682fc0ee9c316f24e Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Fri, 10 Jan 2025 14:59:49 +0800 Subject: [PATCH 2/3] perf: don't iter evicted entries without disk cache Signed-off-by: MrCroxx --- Cargo.lock | 9 +++++---- Cargo.toml | 2 +- src/storage/src/store_impl.rs | 7 +++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94f365c92d058..d2014b60e9d2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4931,7 +4931,7 @@ dependencies = [ [[package]] name = "foyer" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" +source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" dependencies = [ "ahash 0.8.11", "anyhow", @@ -4948,7 +4948,7 @@ dependencies = [ [[package]] name = "foyer-common" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" +source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" dependencies = [ "ahash 0.8.11", "bytes", @@ -4975,9 +4975,10 @@ dependencies = [ [[package]] name = "foyer-memory" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" +source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" dependencies = [ "ahash 0.8.11", + "arc-swap", "bitflags 2.6.0", "cmsketch", "equivalent", @@ -4998,7 +4999,7 @@ dependencies = [ [[package]] name = "foyer-storage" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96#0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96" +source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" dependencies = [ "ahash 0.8.11", "allocator-api2", diff --git a/Cargo.toml b/Cargo.toml index 1954eb305a657..d09b6916a86ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ development = ["expect-test", "pretty_assertions"] [workspace.dependencies] # foyer = { version = "0.13.1", features = ["tracing", "nightly", "prometheus"] } -foyer = { git = "https://github.com/foyer-rs/foyer", rev = "0ae7820bfc626d67a62d19a36e6c1b46dcbb0e96", features = [ +foyer = { git = "https://github.com/foyer-rs/foyer", rev = "719ab39", features = [ "tracing", "nightly", "prometheus", diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs index 6488e9c3f23b6..aa6b1ad144864 100644 --- a/src/storage/src/store_impl.rs +++ b/src/storage/src/store_impl.rs @@ -42,8 +42,11 @@ use crate::monitor::{ use crate::opts::StorageOpts; use crate::StateStore; -static FOYER_METRICS_REGISTRY: LazyLock = - LazyLock::new(|| PrometheusMetricsRegistry::new(GLOBAL_METRICS_REGISTRY.clone())); +static FOYER_METRICS_REGISTRY: LazyLock> = LazyLock::new(|| { + Box::new(PrometheusMetricsRegistry::new( + GLOBAL_METRICS_REGISTRY.clone(), + )) +}); mod opaque_type { use super::*; From 2e3776ed4b92b39581b30e19eb657e2de7c8190d Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Tue, 14 Jan 2025 15:10:04 +0800 Subject: [PATCH 3/3] chore: switch to a unreleased version on main Signed-off-by: MrCroxx --- Cargo.lock | 8 ++++---- Cargo.toml | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9433351e0a635..9b2a9e28e535d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4931,7 +4931,7 @@ dependencies = [ [[package]] name = "foyer" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" +source = "git+https://github.com/foyer-rs/foyer?rev=bdf484622d487df7ac08533fcc93ab745ed418cf#bdf484622d487df7ac08533fcc93ab745ed418cf" dependencies = [ "ahash 0.8.11", "anyhow", @@ -4948,7 +4948,7 @@ dependencies = [ [[package]] name = "foyer-common" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" +source = "git+https://github.com/foyer-rs/foyer?rev=bdf484622d487df7ac08533fcc93ab745ed418cf#bdf484622d487df7ac08533fcc93ab745ed418cf" dependencies = [ "ahash 0.8.11", "bytes", @@ -4975,7 +4975,7 @@ dependencies = [ [[package]] name = "foyer-memory" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" +source = "git+https://github.com/foyer-rs/foyer?rev=bdf484622d487df7ac08533fcc93ab745ed418cf#bdf484622d487df7ac08533fcc93ab745ed418cf" dependencies = [ "ahash 0.8.11", "arc-swap", @@ -4999,7 +4999,7 @@ dependencies = [ [[package]] name = "foyer-storage" version = "0.14.0-dev" -source = "git+https://github.com/foyer-rs/foyer?rev=719ab39#719ab390e866293101566698e1a5747eac34d444" +source = "git+https://github.com/foyer-rs/foyer?rev=bdf484622d487df7ac08533fcc93ab745ed418cf#bdf484622d487df7ac08533fcc93ab745ed418cf" dependencies = [ "ahash 0.8.11", "allocator-api2", diff --git a/Cargo.toml b/Cargo.toml index d09b6916a86ab..5da97a509c3ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,8 +96,9 @@ normal = ["workspace-hack"] development = ["expect-test", "pretty_assertions"] [workspace.dependencies] +# Switch to a released foyer version after the next release. # foyer = { version = "0.13.1", features = ["tracing", "nightly", "prometheus"] } -foyer = { git = "https://github.com/foyer-rs/foyer", rev = "719ab39", features = [ +foyer = { git = "https://github.com/foyer-rs/foyer", rev = "bdf484622d487df7ac08533fcc93ab745ed418cf", features = [ "tracing", "nightly", "prometheus",