feat: split by size or time

clippy

clippy

crates/biliup/Cargo.toml

@@ -60,6 +60,7 @@ chrono = "0.4"
 byteorder = "1.4.3"
 regex = "1.6.0"
 async-trait = "0.1.57"
+humantime = "2.1.0"
 
 #[features]
 #default = ["default-tls"]

crates/biliup/src/downloader.rs

@@ -9,12 +9,8 @@ use std::collections::HashMap;
 use crate::downloader::util::Segmentable;
 use crate::uploader::retryable::retry;
 use reqwest::Response;
-use std::io::Read;
+
 use std::str::FromStr;
-use std::thread::sleep;
-use std::time::Duration;
-use tokio::io::AsyncReadExt;
-use util::Segment;
 
 pub mod error;
 pub mod extractor;
@@ -31,7 +27,7 @@ pub async fn download(
     file_name: &str,
     segment: Segmentable,
 ) -> anyhow::Result<()> {
-    let mut response = get_response(url, &headers).await?;
+    let response = get_response(url, &headers).await?;
     let mut connection = Connection::new(response);
     // let buf = &mut [0u8; 9];
     let bytes = connection.read_frame(9).await?;
@@ -106,7 +102,7 @@ pub async fn get_response(url: &str, headers: &HeaderMap) -> reqwest::Result<Res
 #[cfg(test)]
 mod tests {
     use crate::downloader::download;
-    use crate::downloader::util::{Segment, Segmentable};
+    use crate::downloader::util::Segmentable;
     use anyhow::Result;
     use reqwest::header::{HeaderMap, HeaderValue, REFERER};
 
@@ -125,7 +121,7 @@ mod tests {
             headers,
             "testdouyu%Y-%m-%dT%H_%M_%S",
             // Segment::Size(20 * 1024 * 1024, 0),
-            Segmentable::new(std::time::Duration::from_secs(6000), u64::MAX),
+            Segmentable::new(Some(std::time::Duration::from_secs(6000)), None),
         )?;
         Ok(())
     }

crates/biliup/src/downloader/extractor.rs

@@ -1,13 +1,12 @@
 use crate::downloader;
 use crate::downloader::httpflv::Connection;
-use crate::downloader::util::{Segment, Segmentable};
+use crate::downloader::util::Segmentable;
 use crate::downloader::{get_response, hls, httpflv};
 use async_trait::async_trait;
-use reqwest::header::{HeaderMap, HeaderValue};
+use reqwest::header::HeaderMap;
 use std::fmt::{Display, Formatter};
-use std::future::Future;
-use std::pin::Pin;
-use std::task::{Context, Poll};
+use std::path::Path;
+
 
 mod bilibili;
 mod huya;
@@ -32,8 +31,8 @@ pub struct Site {
 
 impl Display for Site {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Name: {}\n", self.name)?;
-        write!(f, "Title: {}\n", self.title)?;
+        writeln!(f, "Name: {}", self.name)?;
+        writeln!(f, "Title: {}", self.title)?;
         write!(f, "Direct url: {}", self.direct_url)
     }
 }
@@ -49,17 +48,21 @@ impl Site {
         file_name: &str,
         segment: Segmentable,
     ) -> downloader::error::Result<()> {
+        let file_name = file_name.replace("{title}", &self.title);
+        // file_name.canonicalize()
+
+        println!("Save to {}", Path::new(&file_name).display());
         println!("{}", self);
         match self.extension {
             Extension::Flv => {
                 let response = get_response(&self.direct_url, &self.header_map).await?;
                 // response.bytes_stream()
                 let mut connection = Connection::new(response);
                 connection.read_frame(9).await?;
-                httpflv::parse_flv(connection, file_name, segment).await?
+                httpflv::parse_flv(connection, &file_name, segment).await?
             }
             Extension::Ts => {
-                hls::download(&self.direct_url, &self.header_map, file_name, segment).await?
+                hls::download(&self.direct_url, &self.header_map, &file_name, segment).await?
             }
         }
         Ok(())
@@ -72,5 +75,5 @@ pub fn find_extractor(url: &str) -> Option<&dyn SiteDefinition> {
             return Some(extractor);
         }
     }
-    return None;
+    None
 }

crates/biliup/src/downloader/extractor/bilibili.rs

@@ -1,11 +1,8 @@
 use crate::downloader::error::{Error, Result};
 use crate::downloader::extractor::{Extension, Site, SiteDefinition};
 use async_trait::async_trait;
-use futures::{FutureExt, StreamExt};
-use nom::Parser;
 use reqwest::header::{HeaderMap, HeaderValue, REFERER};
 use serde_json::Value;
-use std::ops::Deref;
 
 pub struct BiliLive {}
 
@@ -53,7 +50,7 @@ impl SiteDefinition for BiliLive {
             ("ptype", "8"),
             ("dolby", "5"),
         ];
-        let mut room_play_info: Value = client
+        let room_play_info: Value = client
             .get("https://api.live.bilibili.com/xlive/web-room/v2/index/getRoomPlayInfo")
             .query(&params)
             .send()
@@ -67,7 +64,7 @@ impl SiteDefinition for BiliLive {
         let direct_url = room_play_info["data"]["playurl_info"]["playurl"]["stream"]
             .as_array()
             .and_then(|v| {
-                v.into_iter()
+                v.iter()
                     .filter_map(|v| v["format"].as_array())
                     .flatten()
                     .find(|v| v["format_name"] == "flv")
@@ -76,7 +73,7 @@ impl SiteDefinition for BiliLive {
                 let url_info = v["codec"][0]["url_info"]
                     .as_array()
                     .and_then(|info| {
-                        info.into_iter()
+                        info.iter()
                             .find(|i| !i["host"].to_string().contains(".mcdn."))
                     })
                     .unwrap_or(&v["codec"][0]["url_info"][0]);

crates/biliup/src/downloader/extractor/huya.rs

@@ -2,7 +2,6 @@ use crate::downloader::error::Result;
 use crate::downloader::extractor::{Extension, Site, SiteDefinition};
 use async_trait::async_trait;
 use serde_json::Value;
-use std::fmt::format;
 
 pub struct HuyaLive {}
 
@@ -35,7 +34,7 @@ impl SiteDefinition for HuyaLive {
         let mut v_multi_stream_info = stream["vMultiStreamInfo"].take();
         // vec![1,2].iter().max()
         // println!("{}", v_multi_stream_info);
-        let stream_info = v_multi_stream_info
+        let _stream_info = v_multi_stream_info
             .as_array()
             .and_then(|v| v.iter().max_by_key(|info| info["iBitRate"].as_i64()));
         // println!("{:?}", stream_info);

crates/biliup/src/downloader/flv_writer.rs

@@ -5,9 +5,11 @@ use crate::downloader::flv_parser::{
 use crate::downloader::util;
 use byteorder::{BigEndian, WriteBytesExt};
 use serde::Serialize;
+use std::fs;
 use std::fs::File;
 use std::io::{BufWriter, Write};
-use tracing::error;
+use std::path::{PathBuf};
+use tracing::{error, info};
 
 const FLV_HEADER: [u8; 9] = [
     0x46, // 'F'
@@ -26,8 +28,22 @@ pub struct FlvFile {
 impl FlvFile {
     pub fn new(file_name: &str) -> std::io::Result<Self> {
         let file_name = util::format_filename(file_name);
-        let out =
-            File::create(format!("{file_name}.flv.part")).expect("Unable to create flv file.");
+        let mut path = PathBuf::from(&file_name);
+        if let Some(parent) = path.parent() {
+            fs::create_dir_all(parent)?
+        }
+        path.set_extension("flv.part");
+        let result = File::create(&path);
+        let out = match result {
+            Ok(o) => o,
+            Err(e) => {
+                return Err(std::io::Error::new(
+                    e.kind(),
+                    format!("Unable to create flv file {file_name}"),
+                ))
+            }
+        };
+        info!("create flv file {}", path.display());
         let mut buf_writer = BufWriter::new(out);
         buf_writer.write_all(&FLV_HEADER)?;
         Self::write_previous_tag_size(&mut buf_writer, 0)?;
@@ -73,7 +89,7 @@ impl Drop for FlvFile {
             format!("{}.flv.part", self.name),
             format!("{}.flv", self.name),
         )
-        .unwrap_or_else(|e| error!("{e}"))
+        .unwrap_or_else(|e| error!("drop {} {e}", self.name))
     }
 }
 

crates/biliup/src/downloader/hls.rs

@@ -1,5 +1,5 @@
 use crate::downloader::error::Result;
-use crate::downloader::util::{format_filename, Segment, Segmentable};
+use crate::downloader::util::{format_filename, Segmentable};
 use m3u8_rs::Playlist;
 use reqwest::header::HeaderMap;
 use std::fs::File;

crates/biliup/src/downloader/httpflv.rs

@@ -3,12 +3,11 @@ use crate::downloader::flv_parser::{
     AACPacketType, AVCPacketType, CodecId, FrameType, SoundFormat, TagData, TagHeader,
 };
 use crate::downloader::flv_writer::{FlvFile, FlvTag, TagDataHeader};
-use crate::downloader::util::{Segment, Segmentable};
+use crate::downloader::util::Segmentable;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
-use futures::AsyncReadExt;
 use nom::{Err, IResult};
 use reqwest::Response;
-use std::io::{ErrorKind, Read};
+
 use std::time::Duration;
 use tracing::{info, warn};
 

crates/biliup/src/downloader/util.rs

@@ -1,5 +1,5 @@
 use chrono::{DateTime, Local};
-use std::ops::AddAssign;
+
 use std::time::Duration;
 
 #[derive(Debug)]
@@ -15,18 +15,18 @@ pub struct Segmentable {
 }
 #[derive(Debug)]
 struct Time {
-    expected: Duration,
+    expected: Option<Duration>,
     start: Duration,
     current: Duration,
 }
 #[derive(Debug)]
 struct Size {
-    expected: u64,
+    expected: Option<u64>,
     current: u64,
 }
 
 impl Segmentable {
-    pub fn new(expected_time: Duration, expected_size: u64) -> Self {
+    pub fn new(expected_time: Option<Duration>, expected_size: Option<u64>) -> Self {
         Self {
             time: Time {
                 expected: expected_time,
@@ -41,8 +41,13 @@ impl Segmentable {
     }
 
     pub fn needed(&self) -> bool {
-        (self.time.current - self.time.start) >= self.time.expected
-            || self.size.current > self.size.expected
+        if let Some(expected_time) = self.time.expected {
+            return (self.time.current - self.time.start) >= expected_time;
+        }
+        if let Some(expected_size) = self.size.expected {
+            return self.size.current > expected_size;
+        }
+        false
     }
 
     pub fn increase_time(&mut self, number: Duration) {
@@ -75,12 +80,12 @@ impl Default for Segmentable {
     fn default() -> Self {
         Segmentable {
             time: Time {
-                expected: Duration::MAX,
+                expected: None,
                 start: Duration::ZERO,
                 current: Duration::ZERO,
             },
             size: Size {
-                expected: u64::MAX,
+                expected: None,
                 current: 0,
             },
         }

crates/bin/cli.rs

@@ -2,6 +2,7 @@ use biliup::video::{Studio, Vid};
 use clap::{ArgEnum, Parser, Subcommand};
 use std::path::PathBuf;
 
+
 #[derive(Parser)]
 #[clap(author, version, about)]
 pub struct Cli {
@@ -80,7 +81,21 @@ pub enum Commands {
         file_name: PathBuf,
     },
     /// 下载视频
-    Download { url: String },
+    Download {
+        url: String,
+
+        /// Output filename template. e.p. "./video/%Y-%m-%dT%H_%M_%S{title}"
+        #[clap(short, long, default_value = "{title}")]
+        output: String,
+
+        /// 按照大小分割视频
+        #[clap(long, value_parser = human_size)]
+        split_size: Option<u64>,
+
+        /// 按照时间分割视频
+        #[clap(long)]
+        split_time: Option<humantime::Duration>,
+    },
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, ArgEnum)]
@@ -93,6 +108,23 @@ pub enum UploadLine {
     CosInternal,
 }
 
+fn human_size(s: &str) -> Result<u64, String> {
+    let ret = match s.as_bytes() {
+        [init @ .., b'K'] => parse_u8(init)? * 1000.0,
+        [init @ .., b'M'] => parse_u8(init)? * 1000.0 * 1000.0,
+        [init @ .., b'G'] => parse_u8(init)? * 1000.0 * 1000.0 * 1000.0,
+        init => parse_u8(init)?,
+    };
+    Ok(ret as u64)
+}
+
+fn parse_u8(string: &[u8]) -> Result<f64, String> {
+    let string = String::from_utf8_lossy(string);
+    string
+        .parse()
+        .map_err(|e| format!("{string} is not ascii digit. {:?}", e))
+}
+
 // pub async fn parse() -> Result<()> {
 //     let cli = Cli::parse();
 //