From d1927a40fa7568747c4f4f6be48cb550beb55e53 Mon Sep 17 00:00:00 2001 From: Louis <836250617@qq.com> Date: Tue, 18 Jun 2024 20:20:27 +0800 Subject: [PATCH 1/2] feature/from_archives: MVP --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- fav_cli/src/bili/action.rs | 11 +++++++++-- fav_cli/src/bili/mod.rs | 16 +++++++++++----- fav_utils/Cargo.toml | 2 +- fav_utils/build.rs | 2 +- fav_utils/proto/bili.proto | 2 ++ fav_utils/src/bili/api.rs | 12 ++++++++++++ fav_utils/src/bili/ops.rs | 35 +++++++++++++++++++++++++++++++---- fav_utils/src/bili/res.rs | 34 +++++++++++++++++++++++----------- 10 files changed, 96 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5261e6b..404f68c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -382,7 +382,7 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fav" -version = "0.2.28" +version = "0.2.29" dependencies = [ "fav_cli", "tokio", @@ -392,7 +392,7 @@ dependencies = [ [[package]] name = "fav_cli" -version = "0.2.28" +version = "0.2.29" dependencies = [ "chrono", "clap", @@ -440,7 +440,7 @@ dependencies = [ [[package]] name = "fav_utils" -version = "0.0.11" +version = "0.0.12" dependencies = [ "fav_core", "futures", diff --git a/Cargo.toml b/Cargo.toml index 9e5adb7..4ee62c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace.package] -version = "0.2.28" +version = "0.2.29" authors = ["Louis <836250617@qq.com>"] description = "Back up your favorite online resources with CLI." license = "MIT" @@ -16,8 +16,8 @@ documentation = "" [workspace.dependencies] fav_core = { path = "fav_core", version = "0.1.3" } fav_derive = { path = "fav_derive", version = "0.0.2" } -fav_utils = { path = "fav_utils", version = "0.0.11" } -fav_cli = { path = "fav_cli", version = "0.2.28" } +fav_utils = { path = "fav_utils", version = "0.0.12" } +fav_cli = { path = "fav_cli", version = "0.2.29" } [profile.release] lto = "fat" diff --git a/fav_cli/src/bili/action.rs b/fav_cli/src/bili/action.rs index f1bfa51..6e27d14 100644 --- a/fav_cli/src/bili/action.rs +++ b/fav_cli/src/bili/action.rs @@ -62,9 +62,15 @@ pub(super) fn status_all( show_sets: bool, show_res: bool, only_track: bool, + show_all: bool, ) -> FavCoreResult<()> { if show_sets { - let sub = sets.subset(|s| s.check_status(StatusFlags::TRACK) | !only_track); + let sub = sets.subset(|s| { + (s.check_status(StatusFlags::TRACK) | !only_track) + & ((s.upper.mid == 0 // self-created set's mid is 0 + && !s.check_status(StatusFlags::EXPIRED)) + | show_all) + }); sub.table(); } if show_res { @@ -79,7 +85,8 @@ pub(super) fn status_all( pub(super) async fn fetch(sets: &mut BiliSets) -> FavCoreResult<()> { let bili = Bili::read()?; bili.fetch_sets(sets).await?; - let mut sub = sets.subset(|s| s.check_status(StatusFlags::TRACK)); + let mut sub = + sets.subset(|s| s.check_status(StatusFlags::TRACK) & !s.check_status(StatusFlags::EXPIRED)); bili.batch_fetch_set(&mut sub, 8).await?; for set in sub.iter_mut() { let mut sub = set.subset(|r| { diff --git a/fav_cli/src/bili/mod.rs b/fav_cli/src/bili/mod.rs index bc56e57..2891666 100644 --- a/fav_cli/src/bili/mod.rs +++ b/fav_cli/src/bili/mod.rs @@ -53,6 +53,9 @@ enum Commands { /// Show tracked only #[arg(long, short)] track: bool, + /// Show sets including AchivesSets(合集) + #[arg(long, short)] + all: bool, }, /// Track a remote source Track { @@ -114,7 +117,7 @@ impl Cli { clap_complete::generate(shell, &mut cmd, "fav", &mut std::io::stdout()); } subcmd => { - let mut sets = BiliSets::read()?; + let mut sets = BiliSets::read().unwrap_or_default(); let res = match subcmd { Commands::Auth { subcmd: authcmd } => { match authcmd { @@ -132,21 +135,24 @@ impl Cli { sets: show_sets, res: show_res, track: only_track, + all: show_all, } => match id { Some(id) => { - if show_sets | show_res | only_track { + if show_sets | show_res | only_track | show_all { Cli::command() .error( ErrorKind::ArgumentConflict, - "The id to 'fav status' does not take -s, -r, -t, options.", + "The id to 'fav status' does not take -s, -r, -t, -a, options.", ) .exit(); } status(&mut sets, id) } None => match (show_sets, show_res) { - (false, false) => status_all(&mut sets, true, false, only_track), - _ => status_all(&mut sets, show_sets, show_res, only_track), + (false, false) => { + status_all(&mut sets, true, false, only_track, show_all) + } + _ => status_all(&mut sets, show_sets, show_res, only_track, show_all), }, }, Commands::Fetch => fetch(&mut sets).await, diff --git a/fav_utils/Cargo.toml b/fav_utils/Cargo.toml index 0b8d234..1674005 100644 --- a/fav_utils/Cargo.toml +++ b/fav_utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fav_utils" -version = "0.0.11" +version = "0.0.12" authors.workspace = true description = "Fav's utils crate; A collection of utilities and data structures for the fav project" license.workspace = true diff --git a/fav_utils/build.rs b/fav_utils/build.rs index a8f3682..55ee9de 100644 --- a/fav_utils/build.rs +++ b/fav_utils/build.rs @@ -14,7 +14,7 @@ fn main() -> Result<(), Box> { let gen = std::fs::read_to_string(&path)?; let processed = gen.replace("#!", "//").replace("//!", "//"); std::fs::write(path, processed)?; - println!("cargo:return-if-changed=proto"); + println!("cargo:return-if-changed=proto/bili.proto"); println!("cargo:return-if-changed=build.rs"); Ok(()) } diff --git a/fav_utils/proto/bili.proto b/fav_utils/proto/bili.proto index 2346256..a879ce2 100644 --- a/fav_utils/proto/bili.proto +++ b/fav_utils/proto/bili.proto @@ -33,6 +33,8 @@ message BiliSet { int32 media_count = 4; Upper upper = 14; repeated BiliRes medias = 15; + repeated BiliRes archives = 16; + bool is_archives_list = 17; } message BiliSets { diff --git a/fav_utils/src/bili/api.rs b/fav_utils/src/bili/api.rs index b9c192b..73aa894 100644 --- a/fav_utils/src/bili/api.rs +++ b/fav_utils/src/bili/api.rs @@ -11,7 +11,9 @@ impl ApiProvider for Bili { ApiKind::QrPoll => &QrPollApi, ApiKind::Logout => &LogoutApi, ApiKind::FetchSets => &SetsApi, + ApiKind::FetchAchivesSets => &AchivesSetsApi, ApiKind::FetchSet => &SetApi, + ApiKind::FetchAchivesSet => &AchivesSetApi, ApiKind::FetchRes => &ResApi, ApiKind::Wbi => &WbiApi, ApiKind::Pull => &PullApi, @@ -25,7 +27,9 @@ pub enum ApiKind { QrPoll, Logout, FetchSets, + FetchAchivesSets, FetchSet, + FetchAchivesSet, FetchRes, Wbi, Pull, @@ -49,10 +53,18 @@ struct LogoutApi; #[api(endpoint("https://api.bilibili.com/x/v3/fav/folder/created/list-all"), params(&["up_mid"]), cookies(&["SESSDATA"]))] struct SetsApi; +#[derive(Api)] +#[api(endpoint("https://api.bilibili.com/x/v3/fav/folder/collected/list"), params(&["up_mid", "pn", "ps", "platform"]), cookies(&["SESSDATA"]))] +struct AchivesSetsApi; + #[derive(Api)] #[api(endpoint("https://api.bilibili.com/x/v3/fav/resource/list"), params(&["media_id", "pn", "ps"]), cookies(&["SESSDATA"]))] struct SetApi; +#[derive(Api)] +#[api(endpoint("https://api.bilibili.com/x/polymer/web-space/seasons_archives_list"), params(&["mid", "season_id", "page_num", "page_size"]), cookies(&["SESSDATA"]))] +struct AchivesSetApi; + #[derive(Api)] #[api(endpoint("https://api.bilibili.com/x/web-interface/view"), params(&["bvid"]), cookies(&["SESSDATA"]))] struct ResApi; diff --git a/fav_utils/src/bili/ops.rs b/fav_utils/src/bili/ops.rs index 07eaeb9..08f05ea 100644 --- a/fav_utils/src/bili/ops.rs +++ b/fav_utils/src/bili/ops.rs @@ -58,6 +58,25 @@ impl SetsOps for Bili { .request_proto(ApiKind::FetchSets, params, "/data") .await?; info!("Fetch sets successfully."); + let mut pn = 1; + loop { + let params = vec![ + self.cookies().get("DedeUserID").expect(HINT).to_owned(), + pn.to_string(), + "20".to_string(), + "web".to_string(), + ]; + let resp = self.request(ApiKind::FetchAchivesSets, params).await?; + let json: serde_json::Value = fav_core::ops::resp2json(resp, "/data").await?; + let mut new: BiliSets = fav_core::ops::json2proto(&json)?; + new.iter_mut().for_each(|set| set.is_archives_list = true); + *sets |= new; + if !json["has_more"].as_bool().unwrap() { + break; + } + pn += 1; + } + info!("Fetch archives sets successfully."); Ok(()) } } @@ -71,20 +90,28 @@ impl SetOps for Bili { Any: Send, { let id = set.id.to_string(); + let is_archives_list = set.is_archives_list; + let mid = set.upper.mid.to_string(); info!("Fetching set<{}>", id); let page_count = set.media_count.saturating_sub(1) / 20 + 1; let mut stream = tokio_stream::iter(1..=page_count) .map(|pn| { let pn = pn.to_string(); - let params = vec![id.clone(), pn, "20".to_string()]; - self.request_proto::(ApiKind::FetchSet, params, "/data") + let mut params = vec![id.clone(), pn, "20".to_string()]; + match is_archives_list { + true => { + params.insert(0, mid.clone()); + self.request_proto::(ApiKind::FetchAchivesSet, params, "/data") + } + false => self.request_proto::(ApiKind::FetchSet, params, "/data"), + } }) .buffer_unordered(8); tokio::select! { res = async { while let Some(res) = stream.next().await { match res { - Ok(res) => *set |= res.with_res_status_on(StatusFlags::FAV), + Ok(s) => *set |= s.with_res_status_on(StatusFlags::FAV), Err(e) => return Err(e), } } @@ -93,7 +120,7 @@ impl SetOps for Bili { } => { res } - _ = cancelled => Err(FavCoreError::Cancel) + _ = cancelled => Err(FavCoreError::Cancel) } } } diff --git a/fav_utils/src/bili/res.rs b/fav_utils/src/bili/res.rs index c8e86de..825c6cd 100644 --- a/fav_utils/src/bili/res.rs +++ b/fav_utils/src/bili/res.rs @@ -9,8 +9,17 @@ impl BitOrAssign for BiliSets { .into_iter() .for_each(|s| match self.iter_mut().find(|s1| s1.id == s.id) { Some(s1) => { - s1.media_count = s.media_count; - *s1 |= s + if s.media_count == 0 + && !s1.check_status(StatusFlags::EXPIRED) + && s.title == "该合集已失效" + { + s1.title += "（已失效）"; + s1.on_status(StatusFlags::EXPIRED); + } else { + s1.title = s.title; + s1.media_count = s.media_count; + s1.off_status(StatusFlags::EXPIRED); + } } None => cache.push(s), }); @@ -21,14 +30,17 @@ impl BitOrAssign for BiliSets { impl BitOrAssign for BiliSet { /// Merge two sets. If the left set is track, the resources merged into will be track fn bitor_assign(&mut self, rhs: Self) { - rhs.medias.into_iter().for_each(|mut r| { - if self.iter().all(|r1| r1.bvid != r.bvid) { - if self.check_status(StatusFlags::TRACK) { - r.on_status(StatusFlags::TRACK); + rhs.medias + .into_iter() + .chain(rhs.archives) + .for_each(|mut r| { + if self.iter().all(|r1| r1.bvid != r.bvid) { + if self.check_status(StatusFlags::TRACK) { + r.on_status(StatusFlags::TRACK); + } + self.medias.push(r); } - self.medias.push(r); - } - }); + }); } } @@ -57,11 +69,11 @@ impl Set for BiliSet { type Res = BiliRes; fn iter(&self) -> impl Iterator { - self.medias.iter() + self.medias.iter().chain(self.archives.iter()) } fn iter_mut(&mut self) -> impl Iterator { - self.medias.iter_mut() + self.medias.iter_mut().chain(self.archives.iter_mut()) } } From 9230469302781077666743eb47ee7f858ec0ec40 Mon Sep 17 00:00:00 2001 From: Louis <836250617@qq.com> Date: Tue, 18 Jun 2024 20:39:44 +0800 Subject: [PATCH 2/2] feature/from_archives: fix status bug --- fav_utils/src/bili/res.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fav_utils/src/bili/res.rs b/fav_utils/src/bili/res.rs index 825c6cd..bafce30 100644 --- a/fav_utils/src/bili/res.rs +++ b/fav_utils/src/bili/res.rs @@ -7,21 +7,23 @@ impl BitOrAssign for BiliSets { let mut cache = vec![]; rhs.list .into_iter() - .for_each(|s| match self.iter_mut().find(|s1| s1.id == s.id) { + .for_each(|mut s| match self.iter_mut().find(|s1| s1.id == s.id) { Some(s1) => { - if s.media_count == 0 - && !s1.check_status(StatusFlags::EXPIRED) - && s.title == "该合集已失效" - { - s1.title += "（已失效）"; - s1.on_status(StatusFlags::EXPIRED); - } else { + if s.media_count != 0 && s.title != "该合集已失效" { s1.title = s.title; s1.media_count = s.media_count; s1.off_status(StatusFlags::EXPIRED); + } else if !s1.check_status(StatusFlags::EXPIRED) { + s1.title += "（已失效）"; + s1.on_status(StatusFlags::EXPIRED); + } + } + None => { + if s.media_count == 0 && s.title == "该合集已失效" { + s.on_status(StatusFlags::EXPIRED); } + cache.push(s) } - None => cache.push(s), }); self.list.extend(cache); }