diff --git a/Cargo.lock b/Cargo.lock index 997d4cd986..c70b1b55b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,60 +197,60 @@ checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "apalis" -version = "0.5.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3661d27ed090fb120a887a8416f648343a8e6e864791b36f6175a72b2ab3df39" +checksum = "78bbaeebf00817d5aa561515b313ef0d280bf4b92592e4709b21925c1233f613" dependencies = [ "apalis-core", "apalis-cron", "apalis-redis", "apalis-sql", - "futures", - "pin-project-lite", - "serde", - "thiserror", - "tokio", - "tower", - "tracing", - "tracing-futures", ] [[package]] name = "apalis-core" -version = "0.5.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82227972a1bb6f5f5c4444b8228aaed79e28d6ad411e5f88ad46dc04cf066bb" +checksum = "1deb48475efcdece1f23a0553209ee842f264c2a5e9bcc4928bfa6a15a044cde" dependencies = [ - "async-oneshot", - "async-timer 1.0.0-beta.13", + "async-stream", + "async-trait", + "chrono", "futures", + "graceful-shutdown", + "http 1.1.0", + "log", "pin-project-lite", "serde", - "serde_json", + "strum 0.25.0", "thiserror", + "tokio", "tower", + "tracing", + "tracing-futures", "ulid", ] [[package]] name = "apalis-cron" -version = "0.5.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11c4150f1088c1237cfde2d5cd3b045c17b3ed605c52bb3346641e18f2e1f77" +checksum = "43310b7e0132f9520b09224fb6faafb32eec82a672aa79c09e46b5b488ed505b" dependencies = [ "apalis-core", "async-stream", "chrono", "cron", "futures", + "tokio", "tower", ] [[package]] name = "apalis-redis" -version = "0.5.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f0968397ad66d4628a3c8022e201d3edc58eb44a522b5c76b5efd334b9fdd" +checksum = "2abee8225fd875e57b530abbcf2d9c3122c1a2cce905367b67c6410b6f9654d7" dependencies = [ "apalis-core", "async-stream", @@ -260,20 +260,23 @@ dependencies = [ "log", "redis", "serde", + "serde_json", "tokio", ] [[package]] name = "apalis-sql" -version = "0.5.1" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99eaea6cf256a5d0fce59c68608ba16e3ea9f01cb4a45e5c7fa5709ea44dacd1" +checksum = "5899bfd124e460f1449ffab643f6bac0dc417e7ca234f34c732c923c6a3addbf" dependencies = [ "apalis-core", "async-stream", + "async-trait", + "chrono", + "debounced", "futures", "futures-lite 2.3.0", - "log", "serde", "serde_json", "sqlx", @@ -450,7 +453,7 @@ checksum = "3188809947798ea6db736715a60cf645ba3b87ea031c710130e1476b48e45967" dependencies = [ "Inflector", "async-graphql-parser", - "darling", + "darling 0.20.8", "proc-macro-crate 1.3.1", "proc-macro2", "quote", @@ -559,15 +562,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-oneshot" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae47de2a02d543205f3f5457a90b6ecbc9494db70557bd29590ec8f1ddff5463" -dependencies = [ - "futures-micro", -] - [[package]] name = "async-std" version = "1.12.0" @@ -622,28 +616,6 @@ version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" -[[package]] -name = "async-timer" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5fa6ed76cb2aa820707b4eb9ec46f42da9ce70b0eafab5e5e34942b38a44d5" -dependencies = [ - "libc", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "async-timer" -version = "1.0.0-beta.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a18932baa05100f01c9980d03e330f95a8f2dee1a7576969fa507bdce3b568" -dependencies = [ - "error-code", - "libc", - "wasm-bindgen", -] - [[package]] name = "async-tls" version = "0.10.0" @@ -1258,6 +1230,43 @@ dependencies = [ "either", ] +[[package]] +name = "cached" +version = "0.49.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8e463fceca5674287f32d252fb1d94083758b8709c160efae66d263e5f4eba" +dependencies = [ + "ahash 0.8.11", + "cached_proc_macro", + "cached_proc_macro_types", + "directories", + "hashbrown 0.14.3", + "instant", + "once_cell", + "rmp-serde", + "serde", + "sled", + "thiserror", +] + +[[package]] +name = "cached_proc_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9f16c0d84de31a2ab7fdf5f7783c14631f7075cf464eb3bb43119f61c9cb2a" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" + [[package]] name = "castaway" version = "0.2.2" @@ -1576,6 +1585,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1716,14 +1734,38 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.8", + "darling_macro 0.20.8", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -1740,13 +1782,24 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ - "darling_core", + "darling_core 0.20.8", "quote", "syn 2.0.58", ] @@ -1761,7 +1814,7 @@ dependencies = [ "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.9", ] [[package]] @@ -1799,6 +1852,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "debounced" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d8b0346b9fa0aa01a3fa4bcce48d62f8738e9c2956e92f275bbf6cf9d6fab5" +dependencies = [ + "futures-timer", + "futures-util", +] + [[package]] name = "der" version = "0.6.1" @@ -1899,13 +1962,22 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" +dependencies = [ + "dirs-sys 0.3.7", +] + [[package]] name = "dirs" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", ] [[package]] @@ -1918,6 +1990,17 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dirs-sys" version = "0.4.1" @@ -2146,12 +2229,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "error-code" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0474425d51df81997e2f90a21591180b38eccf27292d755f3e30750225c175b" - [[package]] name = "etcetera" version = "0.8.0" @@ -2303,6 +2380,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -2369,7 +2456,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -2417,15 +2504,6 @@ dependencies = [ "syn 2.0.58", ] -[[package]] -name = "futures-micro" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b460264b3593d68b16a7bc35f7bc226ddfebdf9a1c8db1ed95d5cc6b7168c826" -dependencies = [ - "pin-project-lite", -] - [[package]] name = "futures-sink" version = "0.3.30" @@ -2438,6 +2516,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.30" @@ -2568,6 +2652,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "graceful-shutdown" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3effbaf774a1da3462925bb182ccf975c284cf46edca5569ea93420a657af484" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "graphql-introspection-query" version = "0.2.0" @@ -3277,7 +3372,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.5.0", "libc", - "redox_syscall", + "redox_syscall 0.4.1", ] [[package]] @@ -3785,6 +3880,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -3792,7 +3898,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -3803,7 +3923,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -4333,9 +4453,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.25.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6472825949c09872e8f2c50bde59fcefc17748b6be5c90fd67cd8b4daca73bfd" +checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" dependencies = [ "arc-swap", "async-trait", @@ -4354,6 +4474,15 @@ dependencies = [ "url", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -4510,18 +4639,6 @@ dependencies = [ "winreg 0.52.0", ] -[[package]] -name = "retainer" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8c01a8276c14d0f8d51ebcf8a48f0748f9f73f5f6b29e688126e6a52bcb145" -dependencies = [ - "async-lock 2.8.0", - "async-timer 0.7.4", - "log", - "rand 0.8.5", -] - [[package]] name = "rfc6979" version = "0.3.1" @@ -4602,6 +4719,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938a142ab806f18b88a97b0dea523d39e0fd730a064b035726adcfc58a8a5188" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "rs-utils" version = "0.0.0" @@ -4824,7 +4963,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryot" -version = "5.0.7" +version = "5.0.8" dependencies = [ "anyhow", "apalis", @@ -4835,6 +4974,7 @@ dependencies = [ "aws-sdk-s3", "axum", "boilermates", + "cached", "chrono", "chrono-tz", "config 0.1.0", @@ -4865,7 +5005,6 @@ dependencies = [ "paginate", "rand 0.8.5", "regex", - "retainer", "rs-utils", "rust_decimal", "rust_decimal_macros", @@ -4936,7 +5075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "220fe894642e59af0239378fb9d6d045db36933958449d66f05f6e7ddaf05086" dependencies = [ "convert_case", - "darling", + "darling 0.20.8", "proc-macro2", "quote", "syn 2.0.58", @@ -5374,7 +5513,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ - "darling", + "darling 0.20.8", "proc-macro2", "quote", "syn 2.0.58", @@ -5511,6 +5650,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "slug" version = "0.1.5" @@ -5909,7 +6064,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "phf_shared 0.10.0", "precomputed-hash", "serde", @@ -5955,6 +6110,9 @@ name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros 0.25.3", +] [[package]] name = "strum" @@ -5962,7 +6120,20 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros", + "strum_macros 0.26.2", +] + +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.58", ] [[package]] @@ -6259,7 +6430,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.5.6", @@ -6888,7 +7059,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "redox_syscall", + "redox_syscall 0.4.1", "wasite", ] diff --git a/Cargo.toml b/Cargo.toml index d28cdcc712..393c0e8d09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,8 +40,3 @@ serde = { version = "1.0.198", features = ["derive"] } serde_json = "1.0.116" strum = { version = "0.26.2", features = ["derive"] } tracing = { version = "0.1.40", features = ["attributes"] } - -[profile.release] -opt-level = 's' -strip = true -lto = "thin" diff --git a/apps/backend/Cargo.toml b/apps/backend/Cargo.toml index e0359a9ca5..98c2a6751b 100644 --- a/apps/backend/Cargo.toml +++ b/apps/backend/Cargo.toml @@ -1,13 +1,19 @@ [package] name = "ryot" -version = "5.0.7" +version = "5.0.8" edition = "2021" repository = "https://github.com/IgnisDa/ryot" license = "GPL-3.0" [dependencies] anyhow = { workspace = true } -apalis = { version = "0.5.1", features = ["cron", "limit", "sqlite"] } +# TODO: Upgrade when https://github.com/geofmureithi/apalis/issues/301 is fixed +apalis = { version = "0.4.9", features = [ + "cron", + "extensions", + "limit", + "sqlite", +] } argon2 = "0.5.3" async-graphql = { workspace = true } async-graphql-axum = "7.0.3" @@ -15,6 +21,7 @@ async-trait = "0.1.80" aws-sdk-s3 = { version = "1.23.0", features = ["behavior-version-latest"] } axum = { version = "0.7.5", features = ["macros", "multipart"] } boilermates = "0.3.0" +cached = { version = "0.49.3", features = ["disk_store"] } chrono = { workspace = true } chrono-tz = "0.9.0" config = { path = "../../libs/config" } @@ -52,7 +59,6 @@ openidconnect = "3.5.0" paginate = "1.1.11" rand = "0.8.5" regex = "1.10.4" -retainer = "0.3.0" rs-utils = { path = "../../libs/rs-utils" } rust_decimal = "1.35.0" rust_decimal_macros = "1.34.2" diff --git a/apps/backend/src/background.rs b/apps/backend/src/background.rs index e8fd87a12c..a92d6be93f 100644 --- a/apps/backend/src/background.rs +++ b/apps/backend/src/background.rs @@ -33,11 +33,9 @@ impl Job for ScheduledJob { const NAME: &'static str = "apalis::ScheduledJob"; } -pub async fn media_jobs( - _information: ScheduledJob, - misc_service: Data>, - importer_service: Data>, -) -> Result<(), Error> { +pub async fn media_jobs(_information: ScheduledJob, ctx: JobContext) -> Result<(), JobError> { + let misc_service = ctx.data::>().unwrap(); + let importer_service = ctx.data::>().unwrap(); if env::var("DISABLE_INVALIDATE_IMPORT_JOBS").is_err() { tracing::trace!("Invalidating invalid media import jobs"); importer_service.invalidate_import_jobs().await.unwrap(); @@ -74,26 +72,25 @@ pub async fn media_jobs( Ok(()) } -pub async fn user_jobs( - _information: ScheduledJob, - service: Data>, -) -> Result<(), Error> { +pub async fn user_jobs(_information: ScheduledJob, ctx: JobContext) -> Result<(), JobError> { + let misc_service = ctx.data::>().unwrap(); tracing::trace!("Cleaning up user and metadata association"); - service + misc_service .cleanup_user_and_metadata_association() .await .unwrap(); tracing::trace!("Removing old user summaries and regenerating them"); - service.regenerate_user_summaries().await.unwrap(); + misc_service.regenerate_user_summaries().await.unwrap(); Ok(()) } pub async fn yank_integrations_data( _information: ScheduledJob, - service: Data>, -) -> Result<(), Error> { + ctx: JobContext, +) -> Result<(), JobError> { + let misc_service = ctx.data::>().unwrap(); tracing::trace!("Getting data from yanked integrations for all users"); - service.yank_integrations_data().await.unwrap(); + misc_service.yank_integrations_data().await.unwrap(); Ok(()) } @@ -112,8 +109,9 @@ impl Job for CoreApplicationJob { pub async fn perform_core_application_job( information: CoreApplicationJob, - misc_service: Data>, -) -> Result<(), Error> { + ctx: JobContext, +) -> Result<(), JobError> { + let misc_service = ctx.data::>().unwrap(); let name = information.to_string(); tracing::trace!("Started job: {:#?}", name); let start = Instant::now(); @@ -157,11 +155,12 @@ impl Job for ApplicationJob { pub async fn perform_application_job( information: ApplicationJob, - misc_service: Data>, - importer_service: Data>, - exporter_service: Data>, - exercise_service: Data>, -) -> Result<(), Error> { + ctx: JobContext, +) -> Result<(), JobError> { + let importer_service = ctx.data::>().unwrap(); + let exporter_service = ctx.data::>().unwrap(); + let misc_service = ctx.data::>().unwrap(); + let exercise_service = ctx.data::>().unwrap(); let name = information.to_string(); tracing::trace!("Started job: {:#?}", name); let start = Instant::now(); diff --git a/apps/backend/src/importer/media_tracker.rs b/apps/backend/src/importer/media_tracker.rs index 2778e029ec..eef903dc28 100644 --- a/apps/backend/src/importer/media_tracker.rs +++ b/apps/backend/src/importer/media_tracker.rs @@ -139,12 +139,13 @@ struct ItemDetails { } pub async fn import(input: DeployMediaTrackerImportInput) -> Result { + let api_url = input.api_url.trim_end_matches('/'); let client: Client = Config::new() .add_header(USER_AGENT, USER_AGENT_STR) .unwrap() .add_header("Access-Token", input.api_key) .unwrap() - .set_base_url(Url::parse(&format!("{}/api/", input.api_url)).unwrap()) + .set_base_url(Url::parse(&format!("{}/api/", api_url)).unwrap()) .try_into() .unwrap(); diff --git a/apps/backend/src/importer/mod.rs b/apps/backend/src/importer/mod.rs index 7ecd03b035..4d65804077 100644 --- a/apps/backend/src/importer/mod.rs +++ b/apps/backend/src/importer/mod.rs @@ -227,22 +227,19 @@ impl ImporterService { pub async fn deploy_import_job( &self, user_id: i32, - mut input: DeployImportJobInput, + input: DeployImportJobInput, ) -> Result { - if let Some(s) = input.media_tracker.as_mut() { - s.api_url = s.api_url.trim_end_matches('/').to_owned() - } - let job = self + let job = ApplicationJob::ImportFromExternalSource(user_id, Box::new(input)); + let task = self .media_service .perform_application_job .clone() - .push(ApplicationJob::ImportFromExternalSource( - user_id, - Box::new(input), - )) + .push(job) .await .unwrap(); - Ok(job.to_string()) + let job_id = task.to_string(); + tracing::debug!("Deployed import job with id = {id}", id = job_id); + Ok(job_id) } pub async fn invalidate_import_jobs(&self) -> Result<()> { diff --git a/apps/backend/src/main.rs b/apps/backend/src/main.rs index e42fe5cd88..1340e2cafd 100644 --- a/apps/backend/src/main.rs +++ b/apps/backend/src/main.rs @@ -11,9 +11,10 @@ use anyhow::{bail, Result}; use apalis::{ cron::{CronStream, Schedule}, layers::{ - limit::RateLimitLayer as ApalisRateLimitLayer, tracing::TraceLayer as ApalisTraceLayer, + Extension as ApalisExtension, RateLimitLayer as ApalisRateLimitLayer, + TraceLayer as ApalisTraceLayer, }, - prelude::{Job as ApalisJob, *}, + prelude::{timer::TokioTimer as SleepTimer, Job as ApalisJob, *}, sqlite::SqliteStorage, }; use aws_sdk_s3::config::Region; @@ -34,7 +35,6 @@ use sea_orm_migration::MigratorTrait; use serde::{de::DeserializeOwned, Serialize}; use sqlx::{pool::PoolOptions, SqlitePool}; use tokio::{join, net::TcpListener}; -use tower::buffer::BufferLayer; use tower_http::{ catch_panic::CatchPanicLayer as TowerCatchPanicLayer, cors::CorsLayer as TowerCorsLayer, trace::TraceLayer as TowerTraceLayer, @@ -256,77 +256,70 @@ async fn main() -> Result<()> { let exercise_service_1 = app_services.exercise_service.clone(); let monitor = async { - Monitor::::new() + Monitor::new() // cron jobs - .register_with_count(1, { - WorkerBuilder::new("general_user_cleanup") + .register_with_count(1, move |c| { + WorkerBuilder::new(format!("general_user_cleanup-{c}")) .stream( - CronStream::new_with_timezone( + CronStream::new( Schedule::from_str(&format!("0 0 */{} ? * *", user_cleanup_every)) .unwrap(), - tz, ) - .into_stream(), + .timer(SleepTimer) + .to_stream_with_timezone(tz), ) .layer(ApalisTraceLayer::new()) - .data(media_service_1.clone()) + .layer(ApalisExtension(media_service_1.clone())) .build_fn(user_jobs) }) - .register_with_count(1, { - WorkerBuilder::new("general_media_cleanup_job") + .register_with_count(1, move |c| { + WorkerBuilder::new(format!("general_media_cleanup_job-{c}")) .stream( // every day - CronStream::new_with_timezone( - Schedule::from_str("0 0 0 * * *").unwrap(), - tz, - ) - .into_stream(), + CronStream::new(Schedule::from_str("0 0 0 * * *").unwrap()) + .timer(SleepTimer) + .to_stream_with_timezone(tz), ) .layer(ApalisTraceLayer::new()) - .data(importer_service_2.clone()) - .data(media_service_2.clone()) + .layer(ApalisExtension(importer_service_2.clone())) + .layer(ApalisExtension(media_service_2.clone())) .build_fn(media_jobs) }) - .register_with_count(1, { - WorkerBuilder::new("yank_integrations_data") + .register_with_count(1, move |c| { + WorkerBuilder::new(format!("yank_integrations_data-{c}")) .stream( - CronStream::new_with_timezone( + CronStream::new( Schedule::from_str(&format!("0 0 */{} ? * *", pull_every)).unwrap(), - tz, ) - .into_stream(), + .timer(SleepTimer) + .to_stream_with_timezone(tz), ) .layer(ApalisTraceLayer::new()) - .data(media_service_3.clone()) + .layer(ApalisExtension(media_service_3.clone())) .build_fn(yank_integrations_data) }) // application jobs - .register_with_count(1, { - WorkerBuilder::new("perform_core_application_job") + .register_with_count(1, move |c| { + WorkerBuilder::new(format!("perform_core_application_job-{c}")) .layer(ApalisTraceLayer::new()) - .data(media_service_5.clone()) + .layer(ApalisExtension(media_service_5.clone())) .with_storage(perform_core_application_job_storage.clone()) .build_fn(perform_core_application_job) }) - .register_with_count( - 1, - WorkerBuilder::new("perform_application_job") - .data(importer_service_1.clone()) - .data(exporter_service_1.clone()) - .data(media_service_4.clone()) - .data(exercise_service_1.clone()) + .register_with_count(3, move |c| { + WorkerBuilder::new(format!("perform_application_job-{c}")) + .layer(ApalisTraceLayer::new()) + .layer(ApalisRateLimitLayer::new( + rate_limit_count, + Duration::new(5, 0), + )) + .layer(ApalisExtension(importer_service_1.clone())) + .layer(ApalisExtension(exporter_service_1.clone())) + .layer(ApalisExtension(media_service_4.clone())) + .layer(ApalisExtension(exercise_service_1.clone())) .with_storage(perform_application_job_storage.clone()) - // DEV: Had to do this fuckery because of https://github.com/geofmureithi/apalis/issues/297 - .chain(|s| { - s.layer(BufferLayer::new(1024)) - .layer(ApalisRateLimitLayer::new( - rate_limit_count, - Duration::new(5, 0), - )) - .layer(ApalisTraceLayer::new()) - }) - .build_fn(perform_application_job), - ) + .build_fn(perform_application_job) + }) .run() .await .unwrap(); @@ -350,8 +343,9 @@ async fn main() -> Result<()> { async fn create_storage( pool: SqlitePool, ) -> SqliteStorage { - SqliteStorage::setup(&pool).await.unwrap(); - SqliteStorage::new(pool) + let st = SqliteStorage::new(pool); + st.setup().await.unwrap(); + st } fn init_tracing() -> Result<()> { diff --git a/apps/backend/src/miscellaneous/resolver.rs b/apps/backend/src/miscellaneous/resolver.rs index 4f3969d9cf..fedc38db21 100644 --- a/apps/backend/src/miscellaneous/resolver.rs +++ b/apps/backend/src/miscellaneous/resolver.rs @@ -1,7 +1,9 @@ use std::{ collections::{HashMap, HashSet}, + fmt, fs::File, iter::zip, + path::PathBuf, str::FromStr, sync::Arc, }; @@ -12,6 +14,7 @@ use argon2::{Argon2, PasswordHash, PasswordVerifier}; use async_graphql::{ Context, Enum, Error, InputObject, InputType, Object, OneofObject, Result, SimpleObject, Union, }; +use cached::{DiskCache, IOCached}; use chrono::{Datelike, Days, Duration as ChronoDuration, NaiveDate, Utc}; use database::{ AliasedExercise, AliasedMetadata, AliasedMetadataGroup, AliasedMetadataToGenre, AliasedPerson, @@ -32,7 +35,6 @@ use openidconnect::{ reqwest::async_http_client, AuthenticationFlow, AuthorizationCode, CsrfToken, Nonce, Scope, TokenResponse, }; -use retainer::Cache; use rs_utils::{convert_naive_to_utc, get_first_and_last_day_of_month, IsFeatureEnabled}; use rust_decimal::{ prelude::{FromPrimitive, ToPrimitive}, @@ -129,7 +131,7 @@ use crate::{ utils::{ add_entity_to_collection, associate_user_with_entity, entity_in_collections, get_current_date, get_stored_asset, get_user_to_entity_association, ilike_sql, - partial_user_by_id, user_by_id, user_id_from_token, AUTHOR, + partial_user_by_id, user_by_id, user_id_from_token, AUTHOR, TEMP_DIR, }, }; @@ -579,7 +581,7 @@ struct CoreDetails { oidc_enabled: bool, } -#[derive(Debug, Ord, PartialEq, Eq, PartialOrd, Clone)] +#[derive(Debug, Ord, PartialEq, Eq, PartialOrd, Clone, Hash)] struct ProgressUpdateCache { user_id: i32, metadata_id: i32, @@ -590,6 +592,12 @@ struct ProgressUpdateCache { manga_chapter_number: Option, } +impl fmt::Display for ProgressUpdateCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#?}", self) + } +} + #[derive(SimpleObject)] struct UserPersonDetails { reviews: Vec, @@ -1463,9 +1471,9 @@ pub struct MiscellaneousService { pub perform_core_application_job: SqliteStorage, timezone: Arc, file_storage_service: Arc, - seen_progress_cache: Arc>, config: Arc, oidc_client: Arc>, + seen_progress_cache: DiskCache, } impl AuthProvider for MiscellaneousService {} @@ -1480,21 +1488,29 @@ impl MiscellaneousService { timezone: Arc, oidc_client: Arc>, ) -> Self { - let seen_progress_cache = Arc::new(Cache::new()); - let seen_progress_cache_clone = seen_progress_cache.clone(); - - let frequency = ChronoDuration::try_minutes(3).unwrap().to_std().unwrap(); - tokio::spawn(async move { seen_progress_cache_clone.monitor(4, 0.25, frequency).await }); + let cache_name = "seen_progress_cache"; + let path = PathBuf::new().join(TEMP_DIR); + let seen_progress_cache = DiskCache::new(cache_name) + .set_lifespan( + ChronoDuration::try_hours(config.server.progress_update_threshold) + .unwrap() + .num_seconds() + .try_into() + .unwrap(), + ) + .set_disk_directory(path) + .build() + .unwrap(); Self { db: db.clone(), config, timezone, file_storage_service, - seen_progress_cache, perform_application_job: perform_application_job.clone(), perform_core_application_job: perform_core_application_job.clone(), oidc_client, + seen_progress_cache, } } } @@ -2495,7 +2511,8 @@ impl MiscellaneousService { anime_episode_number: input.anime_episode_number, manga_chapter_number: input.manga_chapter_number, }; - if respect_cache && self.seen_progress_cache.get(&cache).await.is_some() { + let in_cache = self.seen_progress_cache.cache_get(&cache).unwrap(); + if respect_cache && in_cache.is_some() { return Ok(ProgressUpdateResultUnion::Error(ProgressUpdateError { error: ProgressUpdateErrorVariant::AlreadySeen, })); @@ -2675,16 +2692,7 @@ impl MiscellaneousService { tracing::debug!("Progress update = {:?}", seen); let id = seen.id; if seen.state == SeenState::Completed && respect_cache { - self.seen_progress_cache - .insert( - cache, - (), - ChronoDuration::try_hours(self.config.server.progress_update_threshold) - .unwrap() - .to_std() - .unwrap(), - ) - .await; + self.seen_progress_cache.cache_set(cache, ()).unwrap(); } self.after_media_seen_tasks(seen).await?; Ok(ProgressUpdateResultUnion::Ok(IdObject { id })) @@ -4715,7 +4723,7 @@ impl MiscellaneousService { anime_episode_number: aen, manga_chapter_number: mcn, }; - self.seen_progress_cache.remove(&cache).await; + self.seen_progress_cache.cache_remove(&cache).unwrap(); let seen_id = si.id; let progress = si.progress; let metadata_id = si.metadata_id; @@ -6032,6 +6040,7 @@ impl MiscellaneousService { .all(&self.db) .await?; for user_id in users_with_integrations { + tracing::debug!("Yanking integrations data for user {}", user_id); self.yank_integrations_data_for_user(user_id).await?; } Ok(()) diff --git a/apps/frontend/app/routes/_dashboard.collections.list.yours.tsx b/apps/frontend/app/routes/_dashboard.collections.list.yours.tsx index 42121c5738..51bb6b2df2 100644 --- a/apps/frontend/app/routes/_dashboard.collections.list.yours.tsx +++ b/apps/frontend/app/routes/_dashboard.collections.list.yours.tsx @@ -150,10 +150,13 @@ export default function Page() { const [toUpdateCollection, setToUpdateCollection] = useState(); - const [opened, { open, close }] = useDisclosure(false); + const [ + createOrUpdateModalOpened, + { open: createOrUpdateModalOpen, close: createOrUpdateModalClose }, + ] = useDisclosure(false); useEffect(() => { if (transition.state !== "submitting") { - close(); + createOrUpdateModalClose(); setToUpdateCollection(undefined); } }, [transition.state]); @@ -169,7 +172,7 @@ export default function Page() { variant="outline" onClick={() => { setToUpdateCollection(undefined); - open(); + createOrUpdateModalOpen(); }} > @@ -181,12 +184,18 @@ export default function Page() { key={c.id} collection={c} setToUpdateCollection={setToUpdateCollection} + openModal={createOrUpdateModalOpen} /> ))} - + @@ -251,6 +260,7 @@ type Collection = UserCollectionsListQuery["userCollectionsList"][number]; const DisplayCollection = (props: { collection: Collection; setToUpdateCollection: (c: UpdateCollectionInput) => void; + openModal: () => void; }) => { const fetcher = useFetcher(); const deleteFormRef = useRef<HTMLFormElement>(null); @@ -285,7 +295,7 @@ const DisplayCollection = (props: { description: props.collection.description, visibility: props.collection.visibility, }); - open(); + props.openModal(); }} > <IconEdit size={18} />