From a1fe300ec7906144065f51f30df12cbb39f02cb4 Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Thu, 22 Sep 2022 09:04:57 +0800 Subject: [PATCH] feat: split by size or time clippy clippy --- Cargo.lock | 11 ++++-- crates/biliup/Cargo.toml | 1 + crates/biliup/src/downloader.rs | 12 +++---- crates/biliup/src/downloader/extractor.rs | 23 +++++++------ .../src/downloader/extractor/bilibili.rs | 9 ++--- .../biliup/src/downloader/extractor/huya.rs | 3 +- crates/biliup/src/downloader/flv_writer.rs | 24 ++++++++++--- crates/biliup/src/downloader/hls.rs | 2 +- crates/biliup/src/downloader/httpflv.rs | 5 ++- crates/biliup/src/downloader/util.rs | 21 +++++++----- crates/bin/cli.rs | 34 ++++++++++++++++++- crates/bin/downloader.rs | 12 +++++-- crates/bin/main.rs | 9 +++-- crates/stream-gears/src/lib.rs | 6 ++-- 14 files changed, 120 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 661e750..47cfeab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,7 @@ dependencies = [ "dialoguer", "futures", "glob", + "humantime", "image", "indicatif", "m3u8-rs", @@ -752,6 +753,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.20" @@ -1769,9 +1776,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" dependencies = [ "bitflags", "core-foundation", diff --git a/crates/biliup/Cargo.toml b/crates/biliup/Cargo.toml index c6a820b..4558008 100644 --- a/crates/biliup/Cargo.toml +++ b/crates/biliup/Cargo.toml @@ -60,6 +60,7 @@ chrono = "0.4" byteorder = "1.4.3" regex = "1.6.0" async-trait = "0.1.57" +humantime = "2.1.0" #[features] #default = ["default-tls"] diff --git a/crates/biliup/src/downloader.rs b/crates/biliup/src/downloader.rs index 329b17b..712a881 100644 --- a/crates/biliup/src/downloader.rs +++ b/crates/biliup/src/downloader.rs @@ -9,12 +9,8 @@ use std::collections::HashMap; use crate::downloader::util::Segmentable; use crate::uploader::retryable::retry; use reqwest::Response; -use std::io::Read; + use std::str::FromStr; -use std::thread::sleep; -use std::time::Duration; -use tokio::io::AsyncReadExt; -use util::Segment; pub mod error; pub mod extractor; @@ -31,7 +27,7 @@ pub async fn download( file_name: &str, segment: Segmentable, ) -> anyhow::Result<()> { - let mut response = get_response(url, &headers).await?; + let response = get_response(url, &headers).await?; let mut connection = Connection::new(response); // let buf = &mut [0u8; 9]; let bytes = connection.read_frame(9).await?; @@ -106,7 +102,7 @@ pub async fn get_response(url: &str, headers: &HeaderMap) -> reqwest::Result) -> std::fmt::Result { - write!(f, "Name: {}\n", self.name)?; - write!(f, "Title: {}\n", self.title)?; + writeln!(f, "Name: {}", self.name)?; + writeln!(f, "Title: {}", self.title)?; write!(f, "Direct url: {}", self.direct_url) } } @@ -49,6 +48,10 @@ impl Site { file_name: &str, segment: Segmentable, ) -> downloader::error::Result<()> { + let file_name = file_name.replace("{title}", &self.title); + // file_name.canonicalize() + + println!("Save to {}", Path::new(&file_name).display()); println!("{}", self); match self.extension { Extension::Flv => { @@ -56,10 +59,10 @@ impl Site { // response.bytes_stream() let mut connection = Connection::new(response); connection.read_frame(9).await?; - httpflv::parse_flv(connection, file_name, segment).await? + httpflv::parse_flv(connection, &file_name, segment).await? } Extension::Ts => { - hls::download(&self.direct_url, &self.header_map, file_name, segment).await? + hls::download(&self.direct_url, &self.header_map, &file_name, segment).await? } } Ok(()) @@ -72,5 +75,5 @@ pub fn find_extractor(url: &str) -> Option<&dyn SiteDefinition> { return Some(extractor); } } - return None; + None } diff --git a/crates/biliup/src/downloader/extractor/bilibili.rs b/crates/biliup/src/downloader/extractor/bilibili.rs index f83dd06..d94cbd2 100644 --- a/crates/biliup/src/downloader/extractor/bilibili.rs +++ b/crates/biliup/src/downloader/extractor/bilibili.rs @@ -1,11 +1,8 @@ use crate::downloader::error::{Error, Result}; use crate::downloader::extractor::{Extension, Site, SiteDefinition}; use async_trait::async_trait; -use futures::{FutureExt, StreamExt}; -use nom::Parser; use reqwest::header::{HeaderMap, HeaderValue, REFERER}; use serde_json::Value; -use std::ops::Deref; pub struct BiliLive {} @@ -53,7 +50,7 @@ impl SiteDefinition for BiliLive { ("ptype", "8"), ("dolby", "5"), ]; - let mut room_play_info: Value = client + let room_play_info: Value = client .get("https://api.live.bilibili.com/xlive/web-room/v2/index/getRoomPlayInfo") .query(¶ms) .send() @@ -67,7 +64,7 @@ impl SiteDefinition for BiliLive { let direct_url = room_play_info["data"]["playurl_info"]["playurl"]["stream"] .as_array() .and_then(|v| { - v.into_iter() + v.iter() .filter_map(|v| v["format"].as_array()) .flatten() .find(|v| v["format_name"] == "flv") @@ -76,7 +73,7 @@ impl SiteDefinition for BiliLive { let url_info = v["codec"][0]["url_info"] .as_array() .and_then(|info| { - info.into_iter() + info.iter() .find(|i| !i["host"].to_string().contains(".mcdn.")) }) .unwrap_or(&v["codec"][0]["url_info"][0]); diff --git a/crates/biliup/src/downloader/extractor/huya.rs b/crates/biliup/src/downloader/extractor/huya.rs index b157e49..abfb678 100644 --- a/crates/biliup/src/downloader/extractor/huya.rs +++ b/crates/biliup/src/downloader/extractor/huya.rs @@ -2,7 +2,6 @@ use crate::downloader::error::Result; use crate::downloader::extractor::{Extension, Site, SiteDefinition}; use async_trait::async_trait; use serde_json::Value; -use std::fmt::format; pub struct HuyaLive {} @@ -35,7 +34,7 @@ impl SiteDefinition for HuyaLive { let mut v_multi_stream_info = stream["vMultiStreamInfo"].take(); // vec![1,2].iter().max() // println!("{}", v_multi_stream_info); - let stream_info = v_multi_stream_info + let _stream_info = v_multi_stream_info .as_array() .and_then(|v| v.iter().max_by_key(|info| info["iBitRate"].as_i64())); // println!("{:?}", stream_info); diff --git a/crates/biliup/src/downloader/flv_writer.rs b/crates/biliup/src/downloader/flv_writer.rs index 86affee..ccdb511 100644 --- a/crates/biliup/src/downloader/flv_writer.rs +++ b/crates/biliup/src/downloader/flv_writer.rs @@ -5,9 +5,11 @@ use crate::downloader::flv_parser::{ use crate::downloader::util; use byteorder::{BigEndian, WriteBytesExt}; use serde::Serialize; +use std::fs; use std::fs::File; use std::io::{BufWriter, Write}; -use tracing::error; +use std::path::{PathBuf}; +use tracing::{error, info}; const FLV_HEADER: [u8; 9] = [ 0x46, // 'F' @@ -26,8 +28,22 @@ pub struct FlvFile { impl FlvFile { pub fn new(file_name: &str) -> std::io::Result { let file_name = util::format_filename(file_name); - let out = - File::create(format!("{file_name}.flv.part")).expect("Unable to create flv file."); + let mut path = PathBuf::from(&file_name); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)? + } + path.set_extension("flv.part"); + let result = File::create(&path); + let out = match result { + Ok(o) => o, + Err(e) => { + return Err(std::io::Error::new( + e.kind(), + format!("Unable to create flv file {file_name}"), + )) + } + }; + info!("create flv file {}", path.display()); let mut buf_writer = BufWriter::new(out); buf_writer.write_all(&FLV_HEADER)?; Self::write_previous_tag_size(&mut buf_writer, 0)?; @@ -73,7 +89,7 @@ impl Drop for FlvFile { format!("{}.flv.part", self.name), format!("{}.flv", self.name), ) - .unwrap_or_else(|e| error!("{e}")) + .unwrap_or_else(|e| error!("drop {} {e}", self.name)) } } diff --git a/crates/biliup/src/downloader/hls.rs b/crates/biliup/src/downloader/hls.rs index 765b313..7199c90 100644 --- a/crates/biliup/src/downloader/hls.rs +++ b/crates/biliup/src/downloader/hls.rs @@ -1,5 +1,5 @@ use crate::downloader::error::Result; -use crate::downloader::util::{format_filename, Segment, Segmentable}; +use crate::downloader::util::{format_filename, Segmentable}; use m3u8_rs::Playlist; use reqwest::header::HeaderMap; use std::fs::File; diff --git a/crates/biliup/src/downloader/httpflv.rs b/crates/biliup/src/downloader/httpflv.rs index af85ad2..381926f 100644 --- a/crates/biliup/src/downloader/httpflv.rs +++ b/crates/biliup/src/downloader/httpflv.rs @@ -3,12 +3,11 @@ use crate::downloader::flv_parser::{ AACPacketType, AVCPacketType, CodecId, FrameType, SoundFormat, TagData, TagHeader, }; use crate::downloader::flv_writer::{FlvFile, FlvTag, TagDataHeader}; -use crate::downloader::util::{Segment, Segmentable}; +use crate::downloader::util::Segmentable; use bytes::{Buf, BufMut, Bytes, BytesMut}; -use futures::AsyncReadExt; use nom::{Err, IResult}; use reqwest::Response; -use std::io::{ErrorKind, Read}; + use std::time::Duration; use tracing::{info, warn}; diff --git a/crates/biliup/src/downloader/util.rs b/crates/biliup/src/downloader/util.rs index 196c274..16a47c7 100644 --- a/crates/biliup/src/downloader/util.rs +++ b/crates/biliup/src/downloader/util.rs @@ -1,5 +1,5 @@ use chrono::{DateTime, Local}; -use std::ops::AddAssign; + use std::time::Duration; #[derive(Debug)] @@ -15,18 +15,18 @@ pub struct Segmentable { } #[derive(Debug)] struct Time { - expected: Duration, + expected: Option, start: Duration, current: Duration, } #[derive(Debug)] struct Size { - expected: u64, + expected: Option, current: u64, } impl Segmentable { - pub fn new(expected_time: Duration, expected_size: u64) -> Self { + pub fn new(expected_time: Option, expected_size: Option) -> Self { Self { time: Time { expected: expected_time, @@ -41,8 +41,13 @@ impl Segmentable { } pub fn needed(&self) -> bool { - (self.time.current - self.time.start) >= self.time.expected - || self.size.current > self.size.expected + if let Some(expected_time) = self.time.expected { + return (self.time.current - self.time.start) >= expected_time; + } + if let Some(expected_size) = self.size.expected { + return self.size.current > expected_size; + } + false } pub fn increase_time(&mut self, number: Duration) { @@ -75,12 +80,12 @@ impl Default for Segmentable { fn default() -> Self { Segmentable { time: Time { - expected: Duration::MAX, + expected: None, start: Duration::ZERO, current: Duration::ZERO, }, size: Size { - expected: u64::MAX, + expected: None, current: 0, }, } diff --git a/crates/bin/cli.rs b/crates/bin/cli.rs index 14ca514..4f9baf7 100644 --- a/crates/bin/cli.rs +++ b/crates/bin/cli.rs @@ -2,6 +2,7 @@ use biliup::video::{Studio, Vid}; use clap::{ArgEnum, Parser, Subcommand}; use std::path::PathBuf; + #[derive(Parser)] #[clap(author, version, about)] pub struct Cli { @@ -80,7 +81,21 @@ pub enum Commands { file_name: PathBuf, }, /// 下载视频 - Download { url: String }, + Download { + url: String, + + /// Output filename template. e.p. "./video/%Y-%m-%dT%H_%M_%S{title}" + #[clap(short, long, default_value = "{title}")] + output: String, + + /// 按照大小分割视频 + #[clap(long, value_parser = human_size)] + split_size: Option, + + /// 按照时间分割视频 + #[clap(long)] + split_time: Option, + }, } #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, ArgEnum)] @@ -93,6 +108,23 @@ pub enum UploadLine { CosInternal, } +fn human_size(s: &str) -> Result { + let ret = match s.as_bytes() { + [init @ .., b'K'] => parse_u8(init)? * 1000.0, + [init @ .., b'M'] => parse_u8(init)? * 1000.0 * 1000.0, + [init @ .., b'G'] => parse_u8(init)? * 1000.0 * 1000.0 * 1000.0, + init => parse_u8(init)?, + }; + Ok(ret as u64) +} + +fn parse_u8(string: &[u8]) -> Result { + let string = String::from_utf8_lossy(string); + string + .parse() + .map_err(|e| format!("{string} is not ascii digit. {:?}", e)) +} + // pub async fn parse() -> Result<()> { // let cli = Cli::parse(); // diff --git a/crates/bin/downloader.rs b/crates/bin/downloader.rs index 310b564..7d9bee3 100644 --- a/crates/bin/downloader.rs +++ b/crates/bin/downloader.rs @@ -7,15 +7,23 @@ use biliup::downloader::flv_parser::{ use biliup::downloader::flv_writer; use biliup::downloader::flv_writer::{FlvTag, TagDataHeader}; use biliup::downloader::httpflv::map_parse_err; +use biliup::downloader::util::Segmentable; use bytes::{Buf, BufMut, Bytes, BytesMut}; use std::io::{BufReader, BufWriter, ErrorKind, Read}; use std::path::PathBuf; + use tracing::warn; -pub async fn download(url: &str) -> Result<()> { +pub async fn download( + url: &str, + output: String, + split_size: Option, + split_time: Option, +) -> Result<()> { + let segmentable = Segmentable::new(split_time.map(|t| t.into()), split_size); if let Some(extractor) = find_extractor(url) { let site = extractor.get_site(url).await?; - site.download(&site.title, Default::default()).await?; + site.download(&output, segmentable).await?; } else { warn!("not find extractor for {url}") } diff --git a/crates/bin/main.rs b/crates/bin/main.rs index 6005c0d..17cba7a 100644 --- a/crates/bin/main.rs +++ b/crates/bin/main.rs @@ -7,7 +7,7 @@ use anyhow::Result; use crate::cli::{Cli, Commands}; use crate::downloader::{download, generate_json}; use crate::uploader::{append, login, renew, show, upload_by_command, upload_by_config}; -use biliup::downloader::extractor::find_extractor; + use clap::Parser; use tracing::Level; use tracing_subscriber::FmtSubscriber; @@ -71,7 +71,12 @@ async fn parse() -> Result<()> { } => append(cli.user_cookie, vid, video_path, line, limit).await?, Commands::Show { vid } => show(cli.user_cookie, vid).await?, Commands::DumpFlv { file_name } => generate_json(file_name)?, - Commands::Download { url } => download(&url).await?, + Commands::Download { + url, + output, + split_size, + split_time, + } => download(&url, output, split_size, split_time).await?, }; Ok(()) } diff --git a/crates/stream-gears/src/lib.rs b/crates/stream-gears/src/lib.rs index 107b731..cce3862 100644 --- a/crates/stream-gears/src/lib.rs +++ b/crates/stream-gears/src/lib.rs @@ -9,7 +9,7 @@ use std::time::Duration; use crate::uploader::UploadLine; use biliup::downloader::construct_headers; -use biliup::downloader::util::{Segment, Segmentable}; +use biliup::downloader::util::Segmentable; use tracing_subscriber::layer::SubscriberExt; #[derive(FromPyObject)] @@ -47,8 +47,8 @@ fn download( let collector = formatting_layer.with(file_layer); let segment = match segment { - PySegment::Time { time } => Segmentable::new(Duration::from_secs(time), u64::MAX), - PySegment::Size { size } => Segmentable::new(Duration::ZERO, size), + PySegment::Time { time } => Segmentable::new(Some(Duration::from_secs(time)), None), + PySegment::Size { size } => Segmentable::new(None, Some(size)), }; tracing::subscriber::with_default(collector, || -> PyResult<()> { match biliup::downloader::download(url, map, file_name, segment) {