diff --git a/module/move/unitore/Cargo.toml b/module/move/unitore/Cargo.toml index 812ea26e02..8f98fba818 100644 --- a/module/move/unitore/Cargo.toml +++ b/module/move/unitore/Cargo.toml @@ -32,6 +32,7 @@ enabled = [] [dependencies] error_tools = { workspace = true, features = [ "default" ] } +proper_path_tools = { workspace = true, features = [ "default" ] } tokio = { version = "1.36.0", features = [ "rt", "rt-multi-thread", "io-std", "macros" ] } hyper = { version = "1.1.0", features = [ "client" ] } hyper-tls = "0.6.0" diff --git a/module/move/unitore/Readme.md b/module/move/unitore/Readme.md index 3513ec3729..e631beb82b 100644 --- a/module/move/unitore/Readme.md +++ b/module/move/unitore/Readme.md @@ -47,7 +47,7 @@ cargo run .feeds.list ``` To get custom information about feeds or frames run SQL query to storage database using command `.query.execute` with query string: ```bash -cargo run .query.execute \'SELECT title, links, MIN\(published\) FROM frame\' +cargo run .query.execute 'SELECT title, links, MIN(published) FROM frame' ``` To remove config file from storage use command `.config.delete` with path to config file: ```bash diff --git a/module/move/unitore/src/Readme.md b/module/move/unitore/src/Readme.md new file mode 100644 index 0000000000..be6a15dfc1 --- /dev/null +++ b/module/move/unitore/src/Readme.md @@ -0,0 +1,6 @@ +## File structure + +- `command` - Contains commands for unitore cli. +- `action` - Contains functions that are executed when command are performed. +- `entity` - Contains entities that are used in commands execution. +- `tool` - Additional functions for convenient use of application. \ No newline at end of file diff --git a/module/move/unitore/src/executor/actions/config.rs b/module/move/unitore/src/action/config.rs similarity index 65% rename from module/move/unitore/src/executor/actions/config.rs rename to module/move/unitore/src/action/config.rs index 6a24392965..49b63a4773 100644 --- a/module/move/unitore/src/executor/actions/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -1,25 +1,22 @@ -//! Actions and report for commands for config files. +//! Actions and report for config files. + +use std::path::PathBuf; use crate::*; -use super::*; -use error_tools::{ err, for_app::Context, BasicError, Result }; -use executor::FeedManager; -use storage:: +use error_tools::{ for_app::Context, Result }; +use sled_adapter::FeedStorage; +use entity:: { - FeedStorage, feed::{ FeedStore, Feed }, config::{ ConfigStore, Config }, }; +use action::Report; use gluesql::{ prelude::Payload, sled_storage::SledStorage }; /// Add configuration file with subscriptions to storage. -pub async fn add_config( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_add( mut storage : FeedStorage< SledStorage >, path : &PathBuf ) -> Result< impl Report > { - let path : std::path::PathBuf = args - .get_owned::< wca::Value >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get path argument for command .config.add" ) )? - .into() - ; + let path = proper_path_tools::path::normalize( path ); let mut err_str = format!( "Invalid path for config file {:?}", path ); @@ -37,54 +34,52 @@ pub async fn add_config( storage : FeedStorage< SledStorage >, args : &wca::Args err_str = format!( "Invalid path for config file {:?}", abs_path ); } } - let path = path.canonicalize().context( err_str )?; - let config = Config::new( path.to_string_lossy().to_string() ); - let mut manager = FeedManager::new( storage ); + if !path.exists() + { + return Err( error_tools::for_app::Error::msg( err_str ) ); + } + + //let abs_path = proper_path_tools::path::canonicalize( path )?; + let abs_path = path.canonicalize()?; + let config = Config::new( abs_path.to_string_lossy().to_string() ); - let config_report = manager.storage - .add_config( &config ) + let config_report = storage + .config_add( &config ) .await .context( "Added 0 config files.\n Failed to add config file to storage." )? ; let feeds = feed_config::read( config.path() )? .into_iter() - .map( | feed | Feed::new( feed.link, feed.update_period ) ) + .map( | feed | Feed::new( feed.link, feed.update_period, config.path() ) ) .collect::< Vec< _ > >() ; - let new_feeds = manager.storage.save_feeds( feeds ).await?; + let new_feeds = storage.feeds_save( feeds ).await?; Ok( ConfigReport{ payload : config_report, new_feeds : Some( new_feeds ) } ) } /// Remove configuration file from storage. -pub async fn delete_config( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_delete( mut storage : FeedStorage< SledStorage >, path : &PathBuf ) -> Result< impl Report > { - let path : std::path::PathBuf = args - .get_owned::< wca::Value >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get path argument for command .config.delete" ) )? - .into() - ; - + let path = proper_path_tools::path::normalize( path ); let path = path.canonicalize().context( format!( "Invalid path for config file {:?}", path ) )?; let config = Config::new( path.to_string_lossy().to_string() ); - let mut manager = FeedManager::new( storage ); Ok( ConfigReport::new( - manager.storage - .delete_config( &config ) + storage + .config_delete( &config ) .await .context( "Failed to remove config from storage." )? ) ) } /// List all files with subscriptions that are currently in storage. -pub async fn list_configs( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > +pub async fn config_list( mut storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - Ok( ConfigReport::new( manager.storage.list_configs().await? ) ) + Ok( ConfigReport::new( storage.config_list().await? ) ) } /// Information about result of command for subscription config. @@ -140,7 +135,7 @@ impl std::fmt::Display for ConfigReport rows.push( vec![ EMPTY_CELL.to_owned(), String::from( row[ 0 ].clone() ) ] ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { write!( f, "{}", table )?; diff --git a/module/move/unitore/src/executor/actions/feed.rs b/module/move/unitore/src/action/feed.rs similarity index 70% rename from module/move/unitore/src/executor/actions/feed.rs rename to module/move/unitore/src/action/feed.rs index 850384c846..f7840a5f55 100644 --- a/module/move/unitore/src/executor/actions/feed.rs +++ b/module/move/unitore/src/action/feed.rs @@ -1,22 +1,15 @@ -//! Endpoints and report for feed commands. +//! Feed actions and reports. use crate::*; -use executor:: -{ - FeedManager, - actions::{ Report, frame::SelectedEntries }, -}; -use storage::{ FeedStorage, feed::FeedStore }; +use action::{ Report, frame::SelectedEntries }; +use sled_adapter::FeedStorage; +use entity::feed::FeedStore; use error_tools::Result; -/// List all feeds. -pub async fn list_feeds( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > +/// List all feeds from storage. +pub async fn feeds_list( mut storage : FeedStorage< gluesql::sled_storage::SledStorage > ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - manager.storage.get_all_feeds().await + storage.feeds_list().await } const EMPTY_CELL : &'static str = ""; @@ -51,7 +44,7 @@ impl std::fmt::Display for FeedsReport let mut headers = vec![ EMPTY_CELL.to_owned() ]; headers.extend( self.0.selected_columns.iter().map( | str | str.to_owned() ) ); - let table = table_display::table_with_headers( headers, rows ); + let table = tool::table_display::table_with_headers( headers, rows ); if let Some( table ) = table { write!( f, "{}", table )?; diff --git a/module/move/unitore/src/executor/actions/frame.rs b/module/move/unitore/src/action/frame.rs similarity index 81% rename from module/move/unitore/src/executor/actions/frame.rs rename to module/move/unitore/src/action/frame.rs index b6e3e40d8b..f26d538e20 100644 --- a/module/move/unitore/src/executor/actions/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -1,42 +1,33 @@ -//! Frames commands actions. +//! Frames actions and reports. use crate::*; -use super::*; -use executor::FeedManager; -use storage:: +use sled_adapter::FeedStorage; +use entity:: { - FeedStorage, feed::FeedStore, config::ConfigStore, - frame::{ FrameStore, RowValue } + frame::{ FrameStore, CellValue } }; use gluesql::prelude::{ Payload, Value, SledStorage }; use feed_config; use error_tools::{ err, Result }; +use action::Report; // qqq : review the whole project and make sure all names are consitant: actions, commands, its tests /// List all frames. -pub async fn list_frames -( - storage : FeedStorage< SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > +pub async fn frames_list( mut storage : FeedStorage< SledStorage > ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - manager.storage.list_frames().await + storage.frames_list().await } /// Update all frames from config files saved in storage. -pub async fn download_frames +pub async fn frames_download ( - storage : FeedStorage< SledStorage >, - _args : &wca::Args, + mut storage : FeedStorage< SledStorage > ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - let payload = manager.storage.list_configs().await?; - + let payload = storage.config_list().await?; let configs = match &payload { Payload::Select { labels: _, rows: rows_vec } => @@ -71,12 +62,12 @@ pub async fn download_frames let mut feeds = Vec::new(); let client = retriever::FeedClient; - for i in 0..subscriptions.len() + for subscription in subscriptions { - let feed = retriever::FeedFetch::fetch(&client, subscriptions[ i ].link.clone()).await?; - feeds.push( ( feed, subscriptions[ i ].update_period.clone(), subscriptions[ i ].link.clone() ) ); + let feed = client.fetch( subscription.link.clone() ).await?; + feeds.push( ( feed, subscription.update_period.clone(), subscription.link ) ); } - manager.storage.process_feeds( feeds ).await + storage.feeds_process( feeds ).await } @@ -123,7 +114,7 @@ impl std::fmt::Display for FramesReport fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result { let initial = vec![ vec![ format!( "Feed title: {}", self.feed_link ) ] ]; - let table = table_display::table_with_headers( initial[ 0 ].clone(), Vec::new() ); + let table = tool::table_display::table_with_headers( initial[ 0 ].clone(), Vec::new() ); if let Some( table ) = table { write!( f, "{}", table )?; @@ -133,7 +124,7 @@ impl std::fmt::Display for FramesReport [ vec![ EMPTY_CELL.to_owned(), format!( "Updated frames: {}", self.updated_frames ) ], vec![ EMPTY_CELL.to_owned(), format!( "Inserted frames: {}", self.new_frames ) ], - vec![ EMPTY_CELL.to_owned(), format!( "Number of frames in storage: {}", self.existing_frames ) ], + vec![ EMPTY_CELL.to_owned(), format!( "Number of frames in storage: {}", self.existing_frames + self.new_frames ) ], ]; if !self.selected_frames.selected_columns.is_empty() @@ -141,7 +132,7 @@ impl std::fmt::Display for FramesReport rows.push( vec![ EMPTY_CELL.to_owned(), format!( "Selected frames:" ) ] ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { write!( f, "{}", table )?; @@ -149,8 +140,14 @@ impl std::fmt::Display for FramesReport for frame in &self.selected_frames.selected_rows { + let first_row = vec! + [ + INDENT_CELL.to_owned(), + self.selected_frames.selected_columns[ 0 ].clone(), + textwrap::fill( &String::from( frame[ 0 ].clone() ), 120 ), + ]; let mut rows = Vec::new(); - for i in 0..self.selected_frames.selected_columns.len() + for i in 1..self.selected_frames.selected_columns.len() { let inner_row = vec! [ @@ -161,7 +158,7 @@ impl std::fmt::Display for FramesReport rows.push( inner_row ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::table_with_headers( first_row, rows ); if let Some( table ) = table { writeln!( f, "{}", table )?; @@ -174,7 +171,7 @@ impl std::fmt::Display for FramesReport impl Report for FramesReport {} -/// Items get from select query from storage. +/// Items retrieved by select queries from storage. #[ derive( Debug ) ] pub struct SelectedEntries { @@ -203,7 +200,7 @@ impl std::fmt::Display for SelectedEntries { for i in 0..self.selected_columns.len() { - write!( f, "{} : {}, ", self.selected_columns[ i ], RowValue( &row[ i ] ) )?; + write!( f, "{} : {}, ", self.selected_columns[ i ], CellValue( &row[ i ] ) )?; } writeln!( f, "" )?; } diff --git a/module/move/unitore/src/executor/actions/mod.rs b/module/move/unitore/src/action/mod.rs similarity index 85% rename from module/move/unitore/src/executor/actions/mod.rs rename to module/move/unitore/src/action/mod.rs index 80471e3650..5f958526f9 100644 --- a/module/move/unitore/src/executor/actions/mod.rs +++ b/module/move/unitore/src/action/mod.rs @@ -8,6 +8,8 @@ // entity - with all entities // tool - with something not directly related to the problem, but convenient to have as a separate function/structure +// aaa: added folders + pub mod frame; pub mod feed; pub mod config; @@ -15,7 +17,8 @@ pub mod query; pub mod table; // qqq : what is it for? purpose? -/// General report. +// aaa : added explanation +/// General report trait for commands return type. pub trait Report : std::fmt::Display + std::fmt::Debug { /// Print report of executed command. diff --git a/module/move/unitore/src/executor/actions/query.rs b/module/move/unitore/src/action/query.rs similarity index 78% rename from module/move/unitore/src/executor/actions/query.rs rename to module/move/unitore/src/action/query.rs index 84064075a7..0f5edd06a7 100644 --- a/module/move/unitore/src/executor/actions/query.rs +++ b/module/move/unitore/src/action/query.rs @@ -1,28 +1,21 @@ -//! Query command endpoint and report. +//! Query actions and report. // qqq : don't use both +// aaa : fixed use crate::*; -use super::*; use gluesql::core::executor::Payload; -use storage::{ FeedStorage, Store }; -use executor::FeedManager; -use error_tools::{ err, BasicError, Result }; +use sled_adapter::{ FeedStorage, Store }; +use action::Report; +use error_tools::Result; /// Execute query specified in query string. -pub async fn execute_query +pub async fn query_execute ( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, - args : &wca::Args, + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, + query_str : String, ) -> Result< impl Report > { - let query = args - .get_owned::< Vec::< String > >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get Query argument for command .query.execute" ) )? - .join( " " ) - ; - - let mut manager = FeedManager::new( storage ); - manager.storage.execute_query( query ).await + storage.execute_query( query_str ).await } const EMPTY_CELL : &'static str = ""; @@ -68,7 +61,7 @@ impl std::fmt::Display for QueryReport ]; rows.push( new_row ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { writeln!( f, "{}", table )?; @@ -91,3 +84,4 @@ impl Report for QueryReport {} // qqq : good tests for query action // all tables should be touched by these tests +// aaa : added in https://github.com/Wandalen/wTools/pull/1284 diff --git a/module/move/unitore/src/action/table.rs b/module/move/unitore/src/action/table.rs new file mode 100644 index 0000000000..5e0c92663b --- /dev/null +++ b/module/move/unitore/src/action/table.rs @@ -0,0 +1,418 @@ +//! Tables metadata actions and reports. + +use crate::*; +use gluesql::prelude::Payload; +use std::collections::HashMap; +use action::Report; +use sled_adapter::FeedStorage; +use entity::table::TableStore; +use error_tools::Result; + +/// Get labels of column for specified table. +pub async fn table_list +( + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, + table_name : Option< String >, +) -> Result< impl Report > +{ + let mut table_names = Vec::new(); + if let Some( name ) = table_name + { + table_names.push( name ); + } + else + { + let tables = storage.tables_list().await?; + + let names = tables.0.keys().map( | k | k.clone() ).collect::< Vec< _ > >(); + table_names.extend( names.into_iter() ); + } + + let mut reports = Vec::new(); + for table_name in table_names + { + let result = storage.table_list( table_name.clone() ).await?; + + let mut table_description = String::new(); + let mut columns = HashMap::new(); + if let Payload::Select { labels: _label_vec, rows: rows_vec } = &result[ 0 ] + { + for row in rows_vec + { + let table = String::from( row[ 0 ].clone() ); + columns.entry( table ) + .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) + .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) + ; + } + } + let mut columns_desc = HashMap::new(); + match table_name.as_str() + { + "feed" => + { + table_description = String::from( "Contains feed items." ); + + for label in columns.get( "feed" ).unwrap() + { + match label.as_str() + { + "link" => { columns_desc.insert + ( + label.clone(), + String::from( "Link to feed source, unique identifier for the feed" ), + ); } + "title" => { columns_desc.insert( label.clone(), String::from( "The title of the feed" ) ); } + "updated" => + { + columns_desc.insert( label.clone(), String::from + ( + "The time at which the feed was last modified. If not provided in the source, or invalid, is Null." + ) ); + }, + "type" => { columns_desc.insert( label.clone(), String::from( "Type of this feed (e.g. RSS2, Atom etc)" ) ); } + "authors" => { columns_desc.insert + ( + label.clone(), + String::from( "Collection of authors defined at the feed level" ) + ); } + "description" => { columns_desc.insert( label.clone(), String::from( "Description of the feed" ) ); } + "published" => { columns_desc.insert + ( + label.clone(), + String::from( "The publication date for the content in the channel" ), + ); } + "update_period" => { columns_desc.insert( label.clone(), String::from( "How often this feed must be updated" ) ); } + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } + } + }, + "frame" => + { + table_description = String::from( "Contains frame items." ); + for label in columns.get( "frame" ).unwrap() + { + match label.as_str() + { + "id" => + { + columns_desc.insert + ( + label.clone(), + String::from( "A unique identifier for this frame in the feed. " ), + ); + }, + "title" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Title of the frame" ), + ); + }, + "updated" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was fetched from source." ), + ); + }, + "authors" => + { + columns_desc.insert + ( + label.clone(), + String::from( "List of authors of the frame, optional." ) + ); + }, + "content" => + { + columns_desc.insert + ( + label.clone(), + String::from( "The content of the frame in html or plain text, optional." ), + ); + }, + "links" => + { + columns_desc.insert + ( + label.clone(), + String::from( "List of links associated with this item of related Web page and attachments." ), + ); + }, + "summary" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Short summary, abstract, or excerpt of the frame item, optional." ), + ); + }, + "categories" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies a list of categories that the item belongs to." ), + ); + }, + "published" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was first published or updated." ), + ); + }, + "source" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ), + ); + }, + "rights" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Conveys information about copyrights over the feed, optional." ), + ); + }, + "media" => + { + columns_desc.insert + ( + label.clone(), + String::from( "List of media oblects, encountered in the frame, optional." ), + ); + }, + "language" => + { + columns_desc.insert + ( + label.clone(), + String::from( "The language specified on the item, optional." ), + ); + }, + "feed_link" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Link of feed that contains this frame." ), + ); + }, + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } + } + } + "config" => + { + table_description = String::from( "Contains paths to feed config files." ); + for label in columns.get( "config" ).unwrap() + { + match label.as_str() + { + "path" => { columns_desc.insert( label.clone(), String::from( "Path to configuration file" ) ); } + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } + } + }, + _ => {}, + } + + reports.push( ColumnsReport::new( table_name, table_description, columns_desc ) ); + } + + Ok( TablesColumnsReport( reports ) ) +} + +/// Get information about tables in storage. +pub async fn tables_list( mut storage : FeedStorage< gluesql::sled_storage::SledStorage > ) -> Result< impl Report > +{ + storage.tables_list().await +} + +const EMPTY_CELL : &'static str = ""; + +/// Information about execution of table columns commands. +#[ derive( Debug ) ] +pub struct TablesColumnsReport( pub Vec< ColumnsReport > ); + +impl std::fmt::Display for TablesColumnsReport +{ + fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + for report in &self.0 + { + writeln!( f, "{}", report )?; + } + + Ok( () ) + } +} + +impl Report for TablesColumnsReport {} + +/// Information about execution of columns listing action. +#[ derive( Debug ) ] +pub struct ColumnsReport +{ + table_name : String, + table_description : String, + columns : std::collections::HashMap< String, String > +} + +impl ColumnsReport +{ + /// Create new table columns report. + pub fn new( table_name : String, table_description : String, columns : HashMap< String, String > ) -> Self + { + Self + { + table_name, + table_description, + columns, + } + } +} + +impl std::fmt::Display for ColumnsReport +{ + fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + writeln!( f, "Table name: {}", self.table_name )?; + writeln!( f, "Description: {}", self.table_description )?; + + if !self.columns.is_empty() + { + writeln!( f, "Columns:" )?; + let mut rows = Vec::new(); + for ( label, desc ) in &self.columns + { + rows.push + ( + vec! + [ + EMPTY_CELL.to_owned(), + label.clone(), + desc.clone(), + ] + ); + } + let table = tool::table_display::table_with_headers + ( + vec! + [ + EMPTY_CELL.to_owned(), + "label".to_owned(), + "description".to_owned(), + ], + rows, + ); + + if let Some( table ) = table + { + writeln!( f, "{}", table )?; + } + } + else + { + writeln!( f, "No columns" )?; + } + + Ok( () ) + } +} + +impl Report for ColumnsReport {} + +/// Information about execution of tables commands. +/// Contains tables name, description and list of columns. +#[ derive( Debug ) ] +pub struct TablesReport( pub HashMap< String, ( String, Vec< String > ) > ); + +impl TablesReport +{ + /// Create new report from payload. + pub fn new( payload : Vec< Payload > ) -> Self + { + let mut result = std::collections::HashMap::new(); + if let Payload::Select { labels: _label_vec, rows: rows_vec } = &payload[ 0 ] + { + { + for row in rows_vec + { + let table = String::from( row[ 0 ].clone() ); + let table_description = match table.as_str() + { + "feed" => String::from( "Contains feed items." ), + "frame" => String::from( "Contains frame items." ), + "config" => String::from( "Contains paths to feed config files." ), + _ => String::new(), + }; + result.entry( table ) + .and_modify( | ( _, vec ) : &mut ( String, Vec< String > ) | vec.push( String::from( row[ 1 ].clone() ) ) ) + .or_insert( ( table_description, vec![ String::from( row[ 1 ].clone() ) ] ) ) + ; + } + } + } + TablesReport( result ) + } +} + +impl std::fmt::Display for TablesReport +{ + fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + writeln!( f, "Storage tables:" )?; + let mut rows = Vec::new(); + for ( table_name, ( desc, columns ) ) in &self.0 + { + let columns_str = if !columns.is_empty() + { + format!( "{};", columns.join( ", " ) ) + } + else + { + String::from( "No columns" ) + }; + + rows.push + ( + vec! + [ + EMPTY_CELL.to_owned(), + table_name.to_owned(), + textwrap::fill( desc, 80 ), + textwrap::fill( &columns_str, 80 ), + ] + ); + } + + let table = tool::table_display::table_with_headers + ( + vec! + [ + EMPTY_CELL.to_owned(), + "name".to_owned(), + "description".to_owned(), + "columns".to_owned(), + ], + rows, + ); + if let Some( table ) = table + { + writeln!( f, "{}", table )?; + } + + Ok( () ) + } +} + +impl Report for TablesReport {} diff --git a/module/move/unitore/src/command/config.rs b/module/move/unitore/src/command/config.rs new file mode 100644 index 0000000000..72eb063007 --- /dev/null +++ b/module/move/unitore/src/command/config.rs @@ -0,0 +1,165 @@ +//! Config files commands. + +use std::path::PathBuf; + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use sled_adapter::FeedStorage; +use action::{ Report, config::{ config_add, config_delete, config_list } }; +use error_tools::Result; + +/// Struct that provides commands for config files. +#[ derive( Debug ) ] +pub struct ConfigCommand; + +impl ConfigCommand +{ + /// Create command for adding config. + pub fn add() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "config.add" ) + .long_hint( concat! + ( + "Add file with feeds configurations. Subject: path to config file.\n", + " Example: .config.add ./config/feeds.toml\n", + " The file should contain config entities with fields:\n", + " - `update_period` : update frequency for feed. Example values: `12h`, `1h 20min`, `2days 5h`;\n", + " - `link` : URL for feed source;\n\n", + " Example:\n", + " [[config]]\n", + " update_period = \"1min\"\n", + " link = \"https://feeds.bbci.co.uk/news/world/rss.xml\"\n", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let path_arg = o.args + .get_owned::< wca::Value >( 0 ); + + if let Some( path ) = path_arg + { + let path : PathBuf = path.into(); + + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_add( feed_storage, &path ).await + } + ); + + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + } + }) + .end() + ) + } + + /// Create command for deleting config. + pub fn delete() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok( + Command::former() + .phrase( "config.delete" ) + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let path_arg = o.args + .get_owned::< wca::Value >( 0 ); + + if let Some( path ) = path_arg + { + let path : PathBuf = path.into(); + + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_delete( feed_storage, &path ).await + } + ); + + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + } + }) + .end() + ) + } + + /// Create command for listing all config files in storage. + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "config.list" ) + .long_hint( concat! + ( + "List all config files saved in storage.\n", + " Example: .config.list", + )) + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_list( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } +} diff --git a/module/move/unitore/src/command/feed.rs b/module/move/unitore/src/command/feed.rs new file mode 100644 index 0000000000..148d404952 --- /dev/null +++ b/module/move/unitore/src/command/feed.rs @@ -0,0 +1,55 @@ +//! Feed command. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, VerifiedCommand }; +use sled_adapter::FeedStorage; +use action::{ Report, feed::feeds_list }; +use error_tools::Result; + +/// Struct that provides commands for feed. +#[ derive( Debug ) ] +pub struct FeedCommand; + +impl FeedCommand +{ + /// Create command that lists all feeds in storage. + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "feeds.list" ) + .long_hint( concat! + ( + "List all feeds from storage.\n", + " Example: .feeds.list", + )) + .routine( move | _o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + feeds_list( feed_storage ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/frame.rs b/module/move/unitore/src/command/frame.rs new file mode 100644 index 0000000000..8a4f18a756 --- /dev/null +++ b/module/move/unitore/src/command/frame.rs @@ -0,0 +1,93 @@ +//! Frame commands. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, VerifiedCommand }; +use sled_adapter::FeedStorage; +use action::{ Report, frame::{ frames_list, frames_download } }; +use error_tools::Result; + +/// Struct that provides commands for frames. +#[ derive( Debug ) ] +pub struct FrameCommand; + +impl FrameCommand +{ + /// Create command that lists all frames in storage. + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "frames.list" ) + .long_hint( concat! + ( + "List all frames saved in storage.\n", + " Example: .frames.list", + )) + .routine( move | _o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + frames_list( feed_storage ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } + + /// Creates command that downloads frames from feeds specified in config files. + pub fn download() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok( + Command::former() + .phrase( "frames.download" ) + .hint( "Download frames from feed sources provided in config files." ) + .long_hint(concat! + ( + "Download frames from feed sources provided in config files.\n", + " Example: .frames.download", + )) + .routine( move | _o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + frames_download( feed_storage ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/mod.rs b/module/move/unitore/src/command/mod.rs new file mode 100644 index 0000000000..5548eb2f17 --- /dev/null +++ b/module/move/unitore/src/command/mod.rs @@ -0,0 +1,7 @@ +//! Commands for unitore executor. + +pub mod table; +pub mod frame; +pub mod feed; +pub mod query; +pub mod config; \ No newline at end of file diff --git a/module/move/unitore/src/command/query.rs b/module/move/unitore/src/command/query.rs new file mode 100644 index 0000000000..24519e1a86 --- /dev/null +++ b/module/move/unitore/src/command/query.rs @@ -0,0 +1,71 @@ +//! Query command. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use sled_adapter::FeedStorage; +use action::{ Report, query::query_execute }; +use error_tools::Result; + +/// Struct that provides commands for queries. +#[ derive( Debug ) ] +pub struct QueryCommand; + +impl QueryCommand +{ + /// Creates command for custom query execution. + pub fn execute() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "query.execute" ) + .long_hint( concat! + ( + "Execute custom query. Subject: query string.\n", + " Example query:\n", + " - select all frames:\n", + r#" .query.execute 'SELECT * FROM frame'"#, + "\n", + " - select title and link to the most recent frame:\n", + r#" .query.execute 'SELECT title, links, MIN( published ) FROM frame'"#, + "\n\n", + )) + .subject().hint( "Query" ).kind( Type::String ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let query_arg = o.args + .get_owned::< String >( 0 ) + ; + + if let Some( query_str ) = query_arg + { + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + query_execute( feed_storage, query_str ).await + } + ); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + } + + }) + .end() + ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs new file mode 100644 index 0000000000..67c82f23a0 --- /dev/null +++ b/module/move/unitore/src/command/table.rs @@ -0,0 +1,105 @@ +//! Table and columns commands. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use sled_adapter::FeedStorage; +use action::{ Report, table::{ table_list, tables_list } }; +use error_tools::Result; + +/// Struct that provides commands for table information. +#[ derive( Debug ) ] +pub struct TableCommand; + +impl TableCommand +{ + /// Creates command to list info about tables in storage. + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "table.list" ) + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let table_name_arg = o.args.get_owned::< String >( 0 ); + + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + table_list( feed_storage, table_name_arg ).await + } ); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() + ) + } +} + +/// Struct that provides commands for table columns information. +#[ derive( Debug ) ] +pub struct TablesCommand; + +impl TablesCommand +{ + /// Creates command to list info about table columns in storage. + pub fn list() -> Result< Command > + { + + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "tables.list" ) + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | _o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + tables_list( feed_storage ).await + } ); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() + ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/entity/config.rs b/module/move/unitore/src/entity/config.rs new file mode 100644 index 0000000000..b28a90a1dc --- /dev/null +++ b/module/move/unitore/src/entity/config.rs @@ -0,0 +1,56 @@ +//! Functionality for storing and retrieving config files. + +use error_tools::Result; +use gluesql::core::executor::Payload; + +/// Config file path. +#[ derive( Debug ) ] +pub struct Config( pub String ); + +impl Config +{ + /// Create new config with provided path. + pub fn new( path : String ) -> Self + { + Self( path ) + } + + /// Get path of config file. + pub fn path( &self ) -> String + { + self.0.clone() + } +} + +/// Functionality of config storing. +#[ async_trait::async_trait( ?Send ) ] +pub trait ConfigStore +{ + /// Add subscription. + async fn config_add( &mut self, config : &Config ) -> Result< Payload >; + + /// Remove subscription. + async fn config_delete( &mut self, config : &Config ) -> Result< Payload >; + + /// List subscriptions. + async fn config_list( &mut self ) -> Result< Payload >; +} + +// qqq : port and adapters should not be in the same file +// Ideally, they should be in different crates, but you should at least put them in different folders +// there should be a `sled_adapter`` folder +// aaa : moved to separate folder + + +// qqq : use AbsolutePath newtype from `path_tools` +// qqq : normalize all paths with `path_tools::path::normalize` +// https://docs.rs/proper_path_tools/latest/proper_path_tools/path/fn.normalize.html +// added path normalization + +// unitore .query.execute \'SELECT \* FROM feed\' +// qqq : something is broken in this table. also lack of association with config files +// aaa : added association with config + +// unitore .query.execute \'SELECT \* FROM x\' +// qqq : it is not obvious where one record ends and another begins +// aaa : added id highlight diff --git a/module/move/unitore/src/entity/feed.rs b/module/move/unitore/src/entity/feed.rs new file mode 100644 index 0000000000..7084e841dd --- /dev/null +++ b/module/move/unitore/src/entity/feed.rs @@ -0,0 +1,106 @@ +//! Feed storage entity and storage functions. + +use crate::*; +use std::time::Duration; +use error_tools::Result; +use gluesql::core:: +{ + ast_builder::{ null, text, timestamp, ExprNode }, + executor::Payload, + chrono::{ Utc, DateTime, SecondsFormat }, +}; + +use action:: +{ + feed::FeedsReport, + frame::UpdateReport, +}; + +/// Feed item. +#[ derive( Debug ) ] +pub struct Feed +{ + /// Link to feed source. + pub link : url::Url, + /// Title of feed. + pub title : Option< String >, + /// Last time the feed was fetched. + pub updated : Option< DateTime< Utc > >, + /// Authors of feed. + pub authors : Option< String >, + /// Short description of feed content. + pub description : Option< String >, + /// Date and time when feed was published. + pub published : Option< DateTime< Utc > >, + /// How often the feed frames must be fetched. + pub update_period : Duration, + /// Path to config file, from which this feed was saved. + pub config_file : String, +} + +impl Feed +{ + /// Create new feed item from source url and update period. + pub fn new( link : url::Url, update_period : Duration, config: String ) -> Self + { + Self + { + link, + title : None, + updated : None, + authors : None, + description : None, + published : None, + update_period, + config_file : config, + } + } +} + +/// Functionality of feed storage. +#[ mockall::automock ] +#[ async_trait::async_trait( ?Send ) ] +pub trait FeedStore +{ + /// Save new feeds to storage. + /// New feeds from config files that doesn't exist in storage will be inserted into `feed` table. + async fn feeds_save( &mut self, feeds : Vec< Feed > ) -> Result< Payload >; + + /// Update existing feeds in storage with new information. + /// Feed is updated one time during first fetch. + async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () >; + + /// Process new fetched feeds and frames. + /// Frames from recent fetch will be sorted into three categories: + /// - new items that will be inserted into `frame` table; + /// - modified items that will be updated; + /// - unchanged frames saved from previous fetches will be ignored. + async fn feeds_process( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) > ) -> Result< UpdateReport >; + + /// Get existing feeds from storage. + /// Retrieves all feeds from `feed` table in storage. + async fn feeds_list( &mut self ) -> Result< FeedsReport >; +} +// qqq : poor description and probably naming. improve, please +// aaa : updated description + + +/// Get convenient format of frame item for using with GlueSQL expression builder. +/// Converts from Feed struct into vec of GlueSQL expression nodes. +impl From< Feed > for Vec< ExprNode< 'static > > +{ + fn from( value : Feed ) -> Self + { + vec! + [ + text( value.link.to_string() ), + value.title.map( text ).unwrap_or( null() ), + value.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + value.authors.map( text ).unwrap_or( null() ), + value.description.map( text ).unwrap_or( null() ), + value.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + text( value.update_period.as_secs().to_string() ), + text( value.config_file ), + ] + } +} diff --git a/module/move/unitore/src/entity/frame.rs b/module/move/unitore/src/entity/frame.rs new file mode 100644 index 0000000000..8fb522ad58 --- /dev/null +++ b/module/move/unitore/src/entity/frame.rs @@ -0,0 +1,280 @@ +//! Frame storing and retrieving functionality. + +use crate::*; +use error_tools::Result; +use gluesql::core:: +{ + ast_builder::{ null, text, timestamp, ExprNode }, chrono::{ DateTime, SecondsFormat, Utc }, executor::Payload +}; +use action::frame::ListReport; + +/// Frame entity. +#[ derive( Debug ) ] +pub struct Frame +{ + /// Frame id. + pub id : String, + /// Frame title. + pub title : Option< String >, + /// Time at which this item was fetched from source. + pub stored_time : Option< DateTime< Utc > >, + /// List of authors of the frame. + pub authors : Option< Vec< String > >, + /// The content of the frame in html or plain text. + pub content : Option< String >, + /// List of links associated with this item of related Web page and attachments. + pub links : Option< Vec< String > >, + /// Short summary, abstract, or excerpt of the frame item. + pub summary : Option< String >, + /// A list of categories that the item belongs to. + pub categories : Option< Vec< String > >, + /// Time at which this item was first published or updated. + pub published : Option< DateTime< Utc > >, + /// Specifies the source feed if the frame was copied from one feed into another feed. + pub source : Option< String >, + /// Information about copyrights over the feed. + pub rights : Option< String >, + /// List of media oblects, encountered in the frame. + pub media : Option< Vec< String > >, + /// The language of the frame. + pub language : Option< String >, + /// Link to feed that contains this frame. + pub feed_link : String, +} + +// qqq : not obvious +// aaa : added explanation +/// Convert from feed_rs feed entry and feed link to Frame struct for convenient use and storage. +impl From< ( feed_rs::model::Entry, String ) > for Frame +{ + fn from( ( entry, feed_link ) : ( feed_rs::model::Entry, String ) ) -> Self + { + let authors = entry.authors + .iter() + .map( | p | p.name.clone() ) + .collect::< Vec< _ > >() + ; + + let content = entry.content + .map( | c | c.body.unwrap_or( c.src.map( | link | link.href ).unwrap_or_default() ) ) + .filter( | s | !s.is_empty() ) + .clone() + ; + + let links = entry.links + .iter() + .map( | link | link.href.clone() ) + .collect::< Vec< _ > >() + .clone() + ; + + let categories = entry.categories + .iter() + .map( | cat | cat.term.clone() ) + .collect::< Vec< _ > >() + ; + + let media = entry.media + .iter() + .flat_map( | m | m.content.clone() ) + .filter_map( | m | m.url.map( | url | url.to_string() ) ) + .collect::< Vec< _ > >() + ; + + Frame + { + id : entry.id, + title : entry.title.map( | title | title.content ).clone(), + stored_time : entry.updated, + authors: ( !authors.is_empty() ).then( || authors ), + // qqq : why join? + // aaa : fixed, saved as list + content, + links: ( !links.is_empty() ).then( || links ), + // qqq : why join? + // aaa : fixed, saved as list + summary : entry.summary.map( | c | c.content ).clone(), + categories: ( !categories.is_empty() ).then( || categories ), + // qqq : why join? + // aaa : fixed, saved as list + published : entry.published.clone(), + source : entry.source.clone(), + rights : entry.rights.map( | r | r.content ).clone(), + media: ( !media.is_empty() ).then( || media ), + // qqq : why join? + // aaa : fixed, saved as list + language : entry.language.clone(), + feed_link, + } + } +} + +/// Frames storing and retrieving. +#[ async_trait::async_trait( ?Send ) ] +pub trait FrameStore +{ + /// Save new frames to storage. + /// New frames will be inserted into `frame` table. + async fn frames_save( &mut self, feed : Vec< Frame > ) -> Result< Payload >; + + /// Update existing frames in storage with new changes. + /// If frames in storage were modified in feed source, they will be changed to match new version. + async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () >; + + /// Get all feed frames from storage. + async fn frames_list( &mut self ) -> Result< ListReport >; +} +// qqq : what is update? what update? don't use word update without noun and explanation what deos it mean +// aaa : fixed comments + + +// qqq : what is it for and why? +// aaa : added explanation + +/// Get convenient frame format for using with GlueSQL expression builder. +/// Converts from Frame struct into vec of GlueSQL expression nodes. +impl From< Frame > for Vec< ExprNode< 'static > > +{ + fn from( entry : Frame ) -> Self + { + let title = entry.title + .map( | title | text( title ) ) + .unwrap_or( null() ) + ; + + let stored_time = entry.stored_time + .map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ) + .unwrap_or( null() ) + ; + + let authors = entry.authors + .map( | authors | + text + ( + format!( "[{}]", authors.into_iter().map( | a | format!( "\"{}\"", a ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) + .unwrap_or( null() ) + ; + + let content = entry.content + .map( | content | text( content ) ) + .unwrap_or( null() ) + ; + + let links = entry.links + .map( | links | + text + ( + format!( "[{}]", links.into_iter().map( | link | format!( "\"{}\"", link ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) + .unwrap_or( null() ) + ; + + let summary = entry.summary + .map( | summary | text( summary ) ) + .unwrap_or( null() ) + ; + + let categories = entry.categories + .map( | categories | + text + ( + format! + ( + "[{}]", + categories.into_iter().map( | category | format!( "\"{}\"", category ) ).collect::< Vec< _ > >().join( ", " ), + ) + ) + ) + .unwrap_or( null() ) + ; + + let published = entry.published + .map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ) + .unwrap_or( null() ) + ; + + let source = entry.source.map( | s | text( s ) ).unwrap_or( null() ); + let rights = entry.rights.map( | r | text( r ) ).unwrap_or( null() ); + let media = entry.media + .map( | media | + text + ( + format!( "[{}]", media.into_iter().map( | media | format!( "\"{}\"", media ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) + .unwrap_or( null() ) + ; + + let language = entry.language.clone().map( text ).unwrap_or( null() ); + + vec! + [ + text( entry.id ), + title, + stored_time, + authors, + content, + links, + summary, + categories, + published, + source, + rights, + media, + language, + text( entry.feed_link ) + ] + } +} + +// qqq : RowValue or CellValue? +// aaa : fixed name +/// GlueSQL Value wrapper for display. +#[ derive( Debug ) ] +pub struct CellValue< 'a >( pub &'a gluesql::prelude::Value ); + +impl std::fmt::Display for CellValue< '_ > +{ + fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + use gluesql::prelude::Value::*; + match &self.0 + { + Bool( val ) => write!( f, "{}", val )?, + I8( val ) => write!( f, "{}", val )?, + I16( val ) => write!( f, "{}", val )?, + I32( val ) => write!( f, "{}", val )?, + I64( val ) => write!( f, "{}", val )?, + I128( val ) => write!( f, "{}", val )?, + U8( val ) => write!( f, "{}", val )?, + U16( val ) => write!( f, "{}", val )?, + U32( val ) => write!( f, "{}", val )?, + U64( val ) => write!( f, "{}", val )?, + U128( val ) => write!( f, "{}", val )?, + F32( val ) => write!( f, "{}", val )?, + F64( val ) => write!( f, "{}", val )?, + Str( val ) => write!( f, "{}", val )?, + Null => write!( f, "Null" )?, + Timestamp( val ) => write!( f, "{}", val )?, + _ => write!( f, "" )?, + } + + Ok( () ) + } +} + +impl From< CellValue< '_ > > for String +{ + fn from( value : CellValue< '_ > ) -> Self + { + use gluesql::core::data::Value::*; + match &value.0 + { + Str( val ) => val.clone(), + _ => String::new(), + } + } +} diff --git a/module/move/unitore/src/entity/mod.rs b/module/move/unitore/src/entity/mod.rs new file mode 100644 index 0000000000..94a95a552b --- /dev/null +++ b/module/move/unitore/src/entity/mod.rs @@ -0,0 +1,7 @@ +//! Entities of application. + +pub mod config; +pub mod frame; +pub mod table; +pub mod feed; + diff --git a/module/move/unitore/src/entity/table.rs b/module/move/unitore/src/entity/table.rs new file mode 100644 index 0000000000..b177c3c934 --- /dev/null +++ b/module/move/unitore/src/entity/table.rs @@ -0,0 +1,18 @@ +//! Functionality for storage tables information. + +use crate::*; +use error_tools::Result; +use gluesql::prelude::Payload; + +use action::table::TablesReport; + +/// Functions for tables informantion. +#[ async_trait::async_trait( ?Send ) ] +pub trait TableStore +{ + /// List tables in storage. + async fn tables_list( &mut self ) -> Result< TablesReport >; + + /// List columns of table. + async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > >; +} diff --git a/module/move/unitore/src/executor.rs b/module/move/unitore/src/executor.rs new file mode 100644 index 0000000000..8010dbd9cc --- /dev/null +++ b/module/move/unitore/src/executor.rs @@ -0,0 +1,64 @@ +//! Execute plan. + +use crate::*; +use wca::{ Dictionary, Executor, Parser, Verifier }; +use error_tools::Result; + +/// Run feed updates. +pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > +{ + // init parser + let parser = Parser; + + // init converter + let dictionary = &Dictionary::former() + .command + ( + command::config::ConfigCommand::add()? + ) + .command + ( + command::config::ConfigCommand::delete()? + ) + .command + ( + command::config::ConfigCommand::list()? + ) + .command + ( + command::frame::FrameCommand::list()? + ) + .command + ( + command::frame::FrameCommand::download()? + ) + .command + ( + command::feed::FeedCommand::list()? + ) + .command + ( + command::table::TablesCommand::list()? + ) + .command + ( + command::table::TableCommand::list()? + ) + .command + ( + command::query::QueryCommand::execute()? + ) + .form(); + let verifier = Verifier; + + // init executor + let executor = Executor::former().form(); + + let args = std::env::args().skip( 1 ).collect::< Vec< String > >(); + let raw_program = parser.parse( args ).unwrap(); + let grammar_program = verifier.to_program( dictionary, raw_program ).unwrap(); + + executor.program( dictionary, grammar_program )?; + + Ok( () ) +} diff --git a/module/move/unitore/src/executor/actions/table.rs b/module/move/unitore/src/executor/actions/table.rs deleted file mode 100644 index 0c0b6e726d..0000000000 --- a/module/move/unitore/src/executor/actions/table.rs +++ /dev/null @@ -1,301 +0,0 @@ -//! Tables metadata commands actions and reports. - -use crate::*; -use gluesql::prelude::Payload; -use std::collections::HashMap; -use executor::{ FeedManager, Report }; -use storage::{ FeedStorage, table::TableStore }; -use error_tools::{ err, BasicError, Result }; - -/// Get labels of column for specified table. -pub async fn list_columns -( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, - args : &wca::Args, -) -> Result< impl Report > -{ - let table_name : String = args - .get_owned::< String >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get 'Name' argument for command .table.list" ) )? - .into() - ; - - let mut manager = FeedManager::new( storage ); - let result = manager.storage.list_columns( table_name.clone() ).await?; - - let mut table_description = String::new(); - let mut columns = std::collections::HashMap::new(); - match &result[ 0 ] - { - Payload::Select { labels: _label_vec, rows: rows_vec } => - { - for row in rows_vec - { - let table = String::from( row[ 0 ].clone() ); - columns.entry( table ) - .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) - .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) - ; - } - }, - _ => {}, - } - let mut columns_desc = HashMap::new(); - match table_name.as_str() - { - "feed" => - { - table_description = String::from( "Contains feed items." ); - - for label in columns.get( "feed" ).unwrap() - { - match label.as_str() - { - "link" => { columns_desc.insert - ( - label.clone(), - String::from( "Link to feed source, unique identifier for the feed" ), - ); } - "title" => { columns_desc.insert( label.clone(), String::from( "The title of the feed" ) ); } - "updated" => - { - columns_desc.insert( label.clone(), String::from - ( - "The time at which the feed was last modified. If not provided in the source, or invalid, is Null." - ) ); - }, - "type" => { columns_desc.insert( label.clone(), String::from( "Type of this feed (e.g. RSS2, Atom etc)" ) ); } - "authors" => { columns_desc.insert - ( - label.clone(), - String::from( "Collection of authors defined at the feed level" ) - ); } - "description" => { columns_desc.insert( label.clone(), String::from( "Description of the feed" ) ); } - "published" => { columns_desc.insert - ( - label.clone(), - String::from( "The publication date for the content in the channel" ), - ); } - "update_period" => { columns_desc.insert( label.clone(), String::from( "How often this feed must be updated" ) ); } - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } - } - } - }, - "frame" => - { - table_description = String::from( "Contains frame items." ); - for label in columns.get( "frame" ).unwrap() - { - match label.as_str() - { - "id" => { columns_desc.insert( label.clone(), String::from( "A unique identifier for this frame in the feed. " ) ); }, - "title" => { columns_desc.insert( label.clone(), String::from( "Title of the frame" ) ); }, - "updated" => { columns_desc.insert( label.clone(), String::from( "Time at which this item was fetched from source." ) ); }, - "authors" => { columns_desc.insert( label.clone(), String::from( "List of authors of the frame, optional." ) ); }, - "content" => { columns_desc.insert( label.clone(), String::from( "The content of the frame in html or plain text, optional." ) ); }, - "links" => { columns_desc.insert( label.clone(), String::from( "List of links associated with this item of related Web page and attachments." ) ); }, - "summary" => { columns_desc.insert( label.clone(), String::from( "Short summary, abstract, or excerpt of the frame item, optional." ) ); }, - "categories" => { columns_desc.insert( label.clone(), String::from( "Specifies a list of categories that the item belongs to." ) ); }, - "published" => { columns_desc.insert( label.clone(), String::from( "Time at which this item was first published or updated." ) ); }, - "source" => { columns_desc.insert( label.clone(), String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ) ); }, - "rights" => { columns_desc.insert( label.clone(), String::from( "Conveys information about copyrights over the feed, optional." ) ); }, - "media" => { columns_desc.insert( label.clone(), String::from( "List of media oblects, encountered in the frame, optional." ) ); }, - "language" => { columns_desc.insert( label.clone(), String::from( "The language specified on the item, optional." ) ); }, - "feed_link" => { columns_desc.insert( label.clone(), String::from( "Link of feed that contains this frame." ) ); }, - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } - } - } - } - "config" => - { - table_description = String::from( "Contains paths to feed config files." ); - for label in columns.get( "config" ).unwrap() - { - match label.as_str() - { - "path" => { columns_desc.insert( label.clone(), String::from( "Path to configuration file" ) ); } - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } - } - } - }, - _ => {}, - } - - Ok( ColumnsReport::new( table_name, table_description, columns_desc ) ) -} - -/// Get names of tables in storage. -pub async fn list_tables -( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > -{ - let mut manager = FeedManager::new( storage ); - manager.storage.list_tables().await -} - -const EMPTY_CELL : &'static str = ""; - -/// Information about execution of tables commands. -#[ derive( Debug ) ] -pub struct ColumnsReport -{ - table_name : String, - table_description : String, - columns : std::collections::HashMap< String, String > -} - -impl ColumnsReport -{ - /// Create new table columns report. - pub fn new( table_name : String, table_description : String, columns : HashMap< String, String > ) -> Self - { - Self - { - table_name, - table_description, - columns, - } - } -} - -impl std::fmt::Display for ColumnsReport -{ - fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - writeln!( f, "Table name: {}", self.table_name )?; - writeln!( f, "Description: {}", self.table_description )?; - - if !self.columns.is_empty() - { - writeln!( f, "Columns:" )?; - let mut rows = Vec::new(); - for ( label, desc ) in &self.columns - { - rows.push - ( - vec! - [ - EMPTY_CELL.to_owned(), - label.clone(), - desc.clone(), - ] - ); - } - let table = table_display::table_with_headers - ( - vec! - [ - EMPTY_CELL.to_owned(), - "label".to_owned(), - "description".to_owned(), - ], - rows, - ); - - if let Some( table ) = table - { - writeln!( f, "{}", table )?; - } - } - else - { - writeln!( f, "No columns" )?; - } - - Ok( () ) - } -} - -impl Report for ColumnsReport {} - -/// Information about execution of tables commands. -#[ derive( Debug ) ] -pub struct TablesReport -{ - tables : std::collections::HashMap< String, ( String, Vec< String > ) > -} - -impl TablesReport -{ - /// Create new report from payload. - pub fn new( payload : Vec< Payload > ) -> Self - { - let mut result = std::collections::HashMap::new(); - match &payload[ 0 ] - { - Payload::Select { labels: _label_vec, rows: rows_vec } => - { - for row in rows_vec - { - let table = String::from( row[ 0 ].clone() ); - let table_description = match table.as_str() - { - "feed" => String::from( "Contains feed items." ), - "frame" => String::from( "Contains frame items." ), - "config" => String::from( "Contains paths to feed config files." ), - _ => String::new(), - }; - result.entry( table ) - .and_modify( | ( _, vec ) : &mut ( String, Vec< String > ) | vec.push( String::from( row[ 1 ].clone() ) ) ) - .or_insert( ( table_description, vec![ String::from( row[ 1 ].clone() ) ] ) ) - ; - } - }, - _ => {}, - } - TablesReport{ tables : result } - } -} - -impl std::fmt::Display for TablesReport -{ - fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - writeln!( f, "Storage tables:" )?; - let mut rows = Vec::new(); - for ( table_name, ( desc, columns ) ) in &self.tables - { - let columns_str = if !columns.is_empty() - { - format!( "{};", columns.join( ", " ) ) - } - else - { - String::from( "No columns" ) - }; - - rows.push - ( - vec! - [ - EMPTY_CELL.to_owned(), - table_name.to_owned(), - textwrap::fill( desc, 80 ), - textwrap::fill( &columns_str, 80 ), - ] - ); - } - - let table = table_display::table_with_headers - ( - vec! - [ - EMPTY_CELL.to_owned(), - "name".to_owned(), - "description".to_owned(), - "columns".to_owned(), - ], - rows, - ); - if let Some( table ) = table - { - writeln!( f, "{}", table )?; - } - - Ok( () ) - } -} - -impl Report for TablesReport {} diff --git a/module/move/unitore/src/executor/mod.rs b/module/move/unitore/src/executor/mod.rs deleted file mode 100644 index 97c5601448..0000000000 --- a/module/move/unitore/src/executor/mod.rs +++ /dev/null @@ -1,272 +0,0 @@ -//! Execute plan. - -use super::*; -use feed_config::SubscriptionConfig; -use gluesql::sled_storage::{ sled::Config, SledStorage }; -use retriever::{ FeedClient, FeedFetch }; -use storage::{ Store, FeedStorage, feed::FeedStore, config::ConfigStore, table::TableStore, frame::FrameStore }; -use wca::{ Args, Type, VerifiedCommand }; -use executor::actions::Report; -use error_tools::Result; - -pub mod actions; -use actions:: -{ - frame::{ list_frames, download_frames }, - feed::list_feeds, - config::{ add_config, delete_config, list_configs }, - query::execute_query, - table::{ list_columns, list_tables }, -}; - -fn action< 'a, F, Fut, R >( async_endpoint : F, args : &'a Args ) -> Result< R > -where - F : FnOnce( FeedStorage< SledStorage >, &'a Args ) -> Fut, - Fut : std::future::Future< Output = Result< R > >, - R : actions::Report, -{ - let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) - .unwrap_or( String::from( "./_data" ) ) - ; - - let config = Config::default() - .path( path_to_storage ) - ; - let rt = tokio::runtime::Runtime::new()?; - - rt.block_on( async move - { - let feed_storage = FeedStorage::init_storage( config ).await?; - async_endpoint( feed_storage, args ).await - } ) -} - -/// Run feed updates. -pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > -{ - let ca = wca::CommandsAggregator::former() - .command( "frames.download" ) - .hint( "Download frames from feed sources provided in config files." ) - .long_hint(concat! - ( - "Download frames from feed sources provided in config files.\n", - " Example: .frames.download", - )) - .routine( | o : VerifiedCommand | - { - match action( download_frames, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "feeds.list" ) - .long_hint( concat! - ( - "List all feeds from storage.\n", - " Example: .feeds.list", - )) - .routine( | o : VerifiedCommand | - { - match action( list_feeds, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "frames.list" ) - .long_hint( concat! - ( - "List all frames saved in storage.\n", - " Example: .frames.list", - )) - .routine( | o : VerifiedCommand | - { - match action( list_frames, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.add" ) - .long_hint( concat! - ( - "Add file with feeds configurations. Subject: path to config file.\n", - " Example: .config.add ./config/feeds.toml\n", - " The file should contain config entities with fields:\n", - " - `update_period` : update frequency for feed. Example values: `12h`, `1h 20min`, `2days 5h`;\n", - " - `link` : URL for feed source;\n\n", - " Example:\n", - " [[config]]\n", - " update_period = \"1min\"\n", - " link = \"https://feeds.bbci.co.uk/news/world/rss.xml\"\n", - )) - .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( add_config, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.delete" ) - .long_hint( concat! - ( - "Delete file with feeds configuraiton. Subject: path to config file.\n", - " Example: .config.delete ./config/feeds.toml", - )) - .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( delete_config, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.list" ) - .long_hint( concat! - ( - "List all config files saved in storage.\n", - " Example: .config.list", - )) - .routine( | o : VerifiedCommand | - { - match action( list_configs, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "tables.list" ) - .long_hint( concat! - ( - "List all tables saved in storage.\n", - " Example: .tables.list", - )) - .routine( | o : VerifiedCommand | - { - match action( list_tables, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "table.list" ) - .long_hint( concat! - ( - "List fields of specified table.\n", - "Subject: table name.\n", - " Example: .table.list feed", - )) - .subject().hint( "Name" ).kind( wca::Type::String ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( list_columns, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "query.execute" ) - .long_hint( concat! - ( - "Execute custom query. Subject: query string, with special characters escaped.\n", - " Example query:\n", - " - select all frames:\n", - r#" .query.execute \'SELECT \* FROM frame\'"#, - "\n", - " - select title and link to the most recent frame:\n", - r#" .query.execute \'SELECT title, links, MIN\(published\) FROM frame\'"#, - "\n\n", - )) - .subject().hint( "Query" ).kind( Type::String ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( execute_query, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - .help_variants( [ wca::HelpVariants::General, wca::HelpVariants::SubjectCommand ] ) - .perform(); - - let args = std::env::args().skip( 1 ).collect::< Vec< String > >(); - ca.perform( args )?; - - Ok( () ) -} - -/// Manages feed subsriptions and updates. -pub struct FeedManager< C, S : FeedStore + ConfigStore + FrameStore + Store + Send > -{ - /// Subscription configuration with link and update period. - pub config : Vec< SubscriptionConfig >, - /// Storage for saving feed. - pub storage : S, - /// Client for fetching feed from links in FeedConfig. - pub client : C, -} - -impl< C, S : FeedStore + ConfigStore + FrameStore + Store + Send > std::fmt::Debug for FeedManager< C, S > -{ - fn fmt( &self, f: &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - writeln!(f, "Feed manager with storage and client" ) - } -} - -impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< FeedClient, S > -{ - /// Create new instance of FeedManager. - pub fn new( storage : S ) -> FeedManager< FeedClient, S > - { - Self - { - storage, - config : Vec::new(), - client : FeedClient, - } - } -} - -impl< C : FeedFetch, S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< C, S > -{ - /// Set configurations for subscriptions. - pub fn set_config( &mut self, configs : Vec< SubscriptionConfig > ) - { - self.config = configs; - } - - /// Set client for fetching feed. - pub fn set_client( &mut self, client : C ) - { - self.client = client; - } - - /// Execute custom query, print result. - pub async fn execute_custom_query( &mut self, query : String ) -> Result< impl actions::Report > - { - self.storage.execute_query( query ).await - } - -} diff --git a/module/move/unitore/src/feed_config.rs b/module/move/unitore/src/feed_config.rs index 71d852731f..88b5f8a791 100644 --- a/module/move/unitore/src/feed_config.rs +++ b/module/move/unitore/src/feed_config.rs @@ -23,7 +23,15 @@ pub struct Subscriptions pub config : Vec< SubscriptionConfig > } -/// Reads provided configuration file with list of subscriptions. +/// Get list of feed subscriptions from provided configuration file. +/// +/// # Arguments +/// +/// * `file_path` - Path to the configuration file. +/// +/// # Returns +/// +/// Result with list of feed subscriptions serialized as SubscriptionConfig. pub fn read( file_path : String ) -> Result< Vec< SubscriptionConfig > > { let read_file = OpenOptions::new() diff --git a/module/move/unitore/src/lib.rs b/module/move/unitore/src/lib.rs index d871675d0c..f6e0df9632 100644 --- a/module/move/unitore/src/lib.rs +++ b/module/move/unitore/src/lib.rs @@ -2,7 +2,11 @@ pub mod retriever; pub mod feed_config; pub mod executor; -pub mod storage; -pub mod table_display; +pub mod tool; +pub mod command; +pub mod action; +pub mod entity; +pub mod sled_adapter; // qqq : src/Readmу.md with file structure please +// aaa : added Readme.md diff --git a/module/move/unitore/src/main.rs b/module/move/unitore/src/main.rs index b85f4af722..49e83d2564 100644 --- a/module/move/unitore/src/main.rs +++ b/module/move/unitore/src/main.rs @@ -1,4 +1,6 @@ -//! qqq : ? +//! Runs unitore command executor. +//! qqq : ? aaa: added documantation. + pub use unitore::executor; fn main() -> Result< (), Box< dyn std::error::Error + Send + Sync > > diff --git a/module/move/unitore/src/retriever.rs b/module/move/unitore/src/retriever.rs index 32a3f4e47a..ac4e94be11 100644 --- a/module/move/unitore/src/retriever.rs +++ b/module/move/unitore/src/retriever.rs @@ -12,22 +12,24 @@ use feed_rs::parser as feed_parser; use error_tools::{ Result, for_app::Context }; // qqq : purpose of trait if any? -/// Fetch feed from provided source link. -#[ async_trait::async_trait ] -pub trait FeedFetch -{ - /// Get feed from source specified by its link. - async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed >; -} +// aaa : removed unnecessary trait /// Feed client for fetching feed. #[ derive( Debug ) ] pub struct FeedClient; -#[ async_trait::async_trait ] -impl FeedFetch for FeedClient +impl FeedClient { - async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed > + /// Fetch feed frames from provided url source. + /// + /// # Arguments + /// + /// * `source` - The link to feed source. + /// + /// # Returns + /// + /// Result with fetched feed as feed_rs Feed struct. + pub async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed > { let https = HttpsConnector::new(); let client = Client::builder( TokioExecutor::new() ).build::< _, Empty< Bytes > >( https ); diff --git a/module/move/unitore/src/sled_adapter/config.rs b/module/move/unitore/src/sled_adapter/config.rs new file mode 100644 index 0000000000..35ad1ae3cd --- /dev/null +++ b/module/move/unitore/src/sled_adapter/config.rs @@ -0,0 +1,56 @@ +//! Config file operation with Sled storage. + +use crate::*; +use error_tools::{ err, Result }; +use gluesql:: +{ + core:: + { + ast_builder::{ col, table, text, Execute, }, + executor::Payload, + }, + sled_storage::SledStorage, +}; +use entity::config::{ Config, ConfigStore }; +use sled_adapter::FeedStorage; + +#[ async_trait::async_trait( ?Send ) ] +impl ConfigStore for FeedStorage< SledStorage > +{ + async fn config_add( &mut self, config : &Config ) -> Result< Payload > + { + let res = table( "config" ) + .insert() + .columns + ( + "path", + ) + .values( vec![ vec![ text( config.path() ) ] ] ) + .execute( &mut *self.storage.lock().await ) + .await; + + Ok( res? ) + } + + async fn config_delete( &mut self, config : &Config ) -> Result< Payload > + { + let res = table( "config" ) + .delete() + .filter( col( "path" ).eq( format!( "'{}'", config.path() ) ) ) + .execute( &mut *self.storage.lock().await ) + .await?; + + if res == Payload::Delete( 0 ) + { + return Err( err!( format!( "Config file with path {} not found in storage", config.path() ) ) ) + } + + Ok( res ) + } + + async fn config_list( &mut self ) -> Result< Payload > + { + let res = table( "config" ).select().execute( &mut *self.storage.lock().await ).await?; + Ok( res ) + } +} diff --git a/module/move/unitore/src/storage/feed.rs b/module/move/unitore/src/sled_adapter/feed.rs similarity index 51% rename from module/move/unitore/src/storage/feed.rs rename to module/move/unitore/src/sled_adapter/feed.rs index d28a1182ab..2659657103 100644 --- a/module/move/unitore/src/storage/feed.rs +++ b/module/move/unitore/src/sled_adapter/feed.rs @@ -1,92 +1,44 @@ -//! Feed storage entity and storage functions. +//! Feed operation with Sled storage. use crate::*; use std::time::Duration; -use error_tools::{ for_app::Context, Result }; +use error_tools::{ Result, for_app::Context }; use gluesql:: { core:: { - ast_builder::{ null, col, table, text, Execute, timestamp, ExprNode }, - data::Value, + ast_builder::{ col, null, table, text, Execute, timestamp, ExprNode }, executor::Payload, - chrono::{ Utc, DateTime, SecondsFormat }, + data::Value, + chrono::SecondsFormat, }, sled_storage::SledStorage, }; - -use executor::actions:: +use entity:: +{ + feed::{ Feed, FeedStore }, + frame::FrameStore, +}; +use action:: { feed::FeedsReport, frame::{ UpdateReport, SelectedEntries, FramesReport }, }; -use storage::{ FeedStorage, frame::FrameStore }; +use sled_adapter::FeedStorage; use wca::wtools::Itertools; -/// Feed item. -#[ derive( Debug ) ] -pub struct Feed -{ - /// Link to feed source. - pub link : url::Url, - /// Title of feed. - pub title : Option< String >, - /// Last time the feed was fetched. - pub updated : Option< DateTime< Utc > >, - /// Authors of feed. - pub authors : Option< String >, - /// Short description of feed content. - pub description : Option< String >, - /// Date and time when feed was published. - pub published : Option< DateTime< Utc > >, - /// How often the feed frames must be fetched. - pub update_period : Duration, -} - -impl Feed -{ - /// Create new feed item from source url and update period. - pub fn new( link : url::Url, update_period : Duration ) -> Self - { - Self - { - link, - title : None, - updated : None, - authors : None, - description : None, - published : None, - update_period, - } - } -} - -/// Functionality of feed storage. -#[ mockall::automock ] -#[ async_trait::async_trait( ?Send ) ] -pub trait FeedStore -{ - - /// Insert items from list into feed table. - async fn update_feed( &mut self, feed : Vec< Feed > ) -> Result< () >; - - /// Process fetched feed, new items will be saved, modified items will be updated. - async fn process_feeds( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) > ) -> Result< UpdateReport >; - - /// Get all feeds from storage. - async fn get_all_feeds( &mut self ) -> Result< FeedsReport >; - - /// Add feeds entries. - async fn save_feeds( &mut self, feeds : Vec< Feed > ) -> Result< Payload >; -} -// qqq : poor description and probably naming. improve, please - #[ async_trait::async_trait( ?Send ) ] impl FeedStore for FeedStorage< SledStorage > { - async fn get_all_feeds( &mut self ) -> Result< FeedsReport > + async fn feeds_list( &mut self ) -> Result< FeedsReport > { - let res = table( "feed" ).select().project( "title, link, update_period" ).execute( &mut *self.storage.lock().await ).await?; + let res = table( "feed" ) + .select() + .project( "title, link, update_period, config_file" ) + .execute( &mut *self.storage.lock().await ) + .await? + ; + let mut report = FeedsReport::new(); match res { @@ -104,17 +56,23 @@ impl FeedStore for FeedStorage< SledStorage > Ok( report ) } - async fn update_feed( &mut self, feed : Vec< Feed > ) -> Result< () > + async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () > { for feed in feed { let _update = table( "feed" ) .update() .set( "title", feed.title.map( text ).unwrap_or( null() ) ) - .set( "updated", feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ) ) + .set( + "updated", + feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) .set( "authors", feed.authors.map( text ).unwrap_or( null() ) ) .set( "description", feed.description.map( text ).unwrap_or( null() ) ) - .set( "published", feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ) ) + .set( + "published", + feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) .filter( col( "link" ).eq( feed.link.to_string() ) ) .execute( &mut *self.storage.lock().await ) .await @@ -125,7 +83,7 @@ impl FeedStore for FeedStorage< SledStorage > Ok( () ) } - async fn process_feeds + async fn feeds_process ( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) >, @@ -199,19 +157,19 @@ impl FeedStore for FeedStorage< SledStorage > reports.push( frames_report ); } - if new_entries.len() > 0 + if !new_entries.is_empty() { - let _saved_report = self.save_frames( new_entries ).await?; + let _saved_report = self.frames_save( new_entries ).await?; } - if modified_entries.len() > 0 + if !modified_entries.is_empty() { - let _updated_report = self.update_frames( modified_entries ).await?; + let _updated_report = self.frames_update( modified_entries ).await?; } Ok( UpdateReport( reports ) ) } - async fn save_feeds( &mut self, feed : Vec< Feed > ) -> Result< Payload > + async fn feeds_save( &mut self, feed : Vec< Feed > ) -> Result< Payload > { let feeds_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | feed | feed.into() ).collect_vec(); @@ -225,7 +183,8 @@ impl FeedStore for FeedStorage< SledStorage > authors, description, published, - update_period", + update_period, + config_file", ) .values( feeds_rows ) .execute( &mut *self.storage.lock().await ) @@ -236,44 +195,3 @@ impl FeedStore for FeedStorage< SledStorage > Ok( insert ) } } - -impl From< ( feed_rs::model::Feed, Duration, url::Url ) > for Feed -{ - fn from( val : ( feed_rs::model::Feed, Duration, url::Url ) ) -> Self - { - let duration = val.1; - let link = val.2; - let value = val.0; - - let authors = value.authors.into_iter().map( | p | p.name ).collect::< Vec< _ > >(); - let description = value.description.map( | desc | desc.content ); - - Self - { - link, - title : value.title.map( | title | title.content ), - updated : value.updated, - published : value.published, - description, - authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), - update_period : duration, - } - } -} - -impl From< Feed > for Vec< ExprNode< 'static > > -{ - fn from( value : Feed ) -> Self - { - vec! - [ - text( value.link.to_string() ), - value.title.map( text ).unwrap_or( null() ), - value.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), - value.authors.map( text ).unwrap_or( null() ), - value.description.map( text ).unwrap_or( null() ), - value.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), - text( value.update_period.as_secs().to_string() ), - ] - } -} diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs new file mode 100644 index 0000000000..e2224f4958 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -0,0 +1,124 @@ +//! Frames operation with Sled storage. + +use crate::*; +use std::collections::HashMap; +use error_tools::{ Result, for_app::Context }; +use gluesql:: +{ + core:: + { + ast_builder::{ col, table, Execute, ExprNode }, + executor::Payload, + data::Value, + }, + sled_storage::SledStorage, +}; +use entity::frame::{ FrameStore, Frame }; +use action::frame::{ SelectedEntries, FramesReport, ListReport }; +use sled_adapter::FeedStorage; +use wca::wtools::Itertools; + +#[ async_trait::async_trait( ?Send ) ] +impl FrameStore for FeedStorage< SledStorage > +{ + async fn frames_list( &mut self ) -> Result< ListReport > + { + let res = table( "frame" ).select().execute( &mut *self.storage.lock().await ).await?; + + let mut reports = Vec::new(); + let all_frames = + if let Payload::Select { labels: label_vec, rows: rows_vec } = res + { + SelectedEntries + { + selected_rows : rows_vec, + selected_columns : label_vec, + } + } + else + { + SelectedEntries::new() + }; + + let mut feeds_map = HashMap::new(); + + for row in all_frames.selected_rows + { + let title_val = row.last().unwrap().clone(); + let title = String::from( title_val ); + feeds_map.entry( title ) + .and_modify( | vec : &mut Vec< Vec< Value > > | vec.push( row.clone() ) ) + .or_insert( vec![ row ] ) + ; + } + + for ( title, frames ) in feeds_map + { + let mut report = FramesReport::new( title ); + report.existing_frames = frames.len(); + report.selected_frames = SelectedEntries + { + selected_rows : frames, + selected_columns : all_frames.selected_columns.clone(), + }; + reports.push( report ); + } + + Ok( ListReport( reports ) ) + } + + async fn frames_save( &mut self, frames : Vec< Frame > ) -> Result< Payload > + { + let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); + + let insert = table( "frame" ) + .insert() + .columns + ( + "id, + title, + stored_time, + authors, + content, + links, + summary, + categories, + published, + source, + rights, + media, + language, + feed_link" + ) + .values( entries_rows ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to insert frames" )? + ; + + Ok( insert ) + } + + async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () > + { + let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); + + for entry in entries_rows + { + let _update = table( "frame" ) + .update() + .set( "title", entry[ 1 ].to_owned() ) + .set( "content", entry[ 4 ].to_owned() ) + .set( "links", entry[ 5 ].to_owned() ) + .set( "summary", entry[ 6 ].to_owned() ) + .set( "published", entry[ 8 ].to_owned() ) + .set( "media", entry[ 9 ].to_owned() ) + .filter( col( "id" ).eq( entry[ 0 ].to_owned() ) ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to update frames" )? + ; + } + Ok( () ) + } +} diff --git a/module/move/unitore/src/storage/mod.rs b/module/move/unitore/src/sled_adapter/mod.rs similarity index 51% rename from module/move/unitore/src/storage/mod.rs rename to module/move/unitore/src/sled_adapter/mod.rs index 1eedc29afd..511d866e8e 100644 --- a/module/move/unitore/src/storage/mod.rs +++ b/module/move/unitore/src/sled_adapter/mod.rs @@ -14,13 +14,12 @@ use gluesql:: prelude::Glue, sled_storage::{ sled::Config, SledStorage }, }; +use action::query::QueryReport; -use executor::actions::query::QueryReport; - -pub mod config; -pub mod frame; -pub mod table; -pub mod feed; +mod frame; +mod table; +mod feed; +mod config; /// Storage for feed frames. #[ derive( Clone ) ] @@ -28,7 +27,6 @@ pub struct FeedStorage< S : GStore + GStoreMut + Send > { /// GlueSQL storage. pub storage : Arc< Mutex< Glue< S > > >, - frame_fields : Vec< [ &'static str; 3 ] >, } impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > @@ -42,7 +40,7 @@ impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > impl FeedStorage< SledStorage > { /// Initialize new storage from configuration, create feed table. - pub async fn init_storage( config : Config ) -> Result< Self > + pub async fn init_storage( config : &Config ) -> Result< Self > { let storage = SledStorage::try_from( config.clone() ) .context( format!( "Failed to initialize storage with config {:?}", config ) )? @@ -50,13 +48,13 @@ impl FeedStorage< SledStorage > let mut glue = Glue::new( storage ); - let sub_table = table( "config" ) + let config_table = table( "config" ) .create_table_if_not_exists() .add_column( "path TEXT PRIMARY KEY" ) .build()? ; - sub_table.execute( &mut glue ).await?; + config_table.execute( &mut glue ).await?; let feed_table = table( "feed" ) .create_table_if_not_exists() @@ -68,42 +66,34 @@ impl FeedStorage< SledStorage > .add_column( "description TEXT" ) .add_column( "published TIMESTAMP" ) .add_column( "update_period TEXT" ) + .add_column( "config_file TEXT FOREIGN KEY REFERENCES config(path)" ) .build()? ; feed_table.execute( &mut glue ).await?; - let frame_fields = vec! - [ - [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], - [ "title", "TEXT", "Title of the frame" ], - [ "updated", "TIMESTAMP", "Time at which this item was fetched from source." ], - [ "authors", "TEXT", "List of authors of the frame, optional." ], - [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], - [ "links", "TEXT", "List of links associated with this item of related Web page and attachments." ], - [ "summary", "TEXT", "Short summary, abstract, or excerpt of the frame item, optional." ], - [ "categories", "TEXT", "Specifies a list of categories that the item belongs to." ], - [ "published", "TIMESTAMP", "Time at which this item was first published or updated." ], - [ "source", "TEXT", "Specifies the source feed if the frame was copied from one feed into another feed, optional." ], - [ "rights", "TEXT", "Conveys information about copyrights over the feed, optional." ], - [ "media", "TEXT", "List of media oblects, encountered in the frame, optional." ], - [ "language", "TEXT", "The language specified on the item, optional." ], - [ "feed_link", "TEXT", "Link of feed that contains this frame." ], - ]; - let mut table = table( "frame" ).create_table_if_not_exists().add_column( "id TEXT PRIMARY KEY" ); - - for column in frame_fields.iter().skip( 1 ).take( frame_fields.len() - 2 ) - { - table = table.add_column( format!( "{} {}", column[ 0 ], column[ 1 ] ).as_str() ); - } - - let table = table.add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) + let frame_table = table( "frame" ) + .create_table_if_not_exists() + .add_column( "id TEXT PRIMARY KEY" ) + .add_column( "title TEXT" ) + .add_column( "stored_time TIMESTAMP" ) + .add_column( "authors LIST" ) + .add_column( "content TEXT" ) + .add_column( "links LIST" ) + .add_column( "summary TEXT" ) + .add_column( "categories LIST" ) + .add_column( "published TIMESTAMP" ) + .add_column( "source TEXT" ) + .add_column( "rights TEXT" ) + .add_column( "media LIST" ) + .add_column( "language TEXT" ) + .add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) .build()? ; - table.execute( &mut glue ).await?; + frame_table.execute( &mut glue ).await?; - Ok( Self{ storage : Arc::new( Mutex::new( glue ) ), frame_fields } ) + Ok( Self{ storage : Arc::new( Mutex::new( glue ) ) } ) } } diff --git a/module/move/unitore/src/storage/table.rs b/module/move/unitore/src/sled_adapter/table.rs similarity index 51% rename from module/move/unitore/src/storage/table.rs rename to module/move/unitore/src/sled_adapter/table.rs index 08038df8ee..ddf0664bfc 100644 --- a/module/move/unitore/src/storage/table.rs +++ b/module/move/unitore/src/sled_adapter/table.rs @@ -1,31 +1,20 @@ -//! Tables sroring functions. +//! Table and columns info operations from Sled storage. use crate::*; use error_tools::Result; use gluesql:: { + core::executor::Payload, sled_storage::SledStorage, - prelude::Payload, }; - -use executor::actions::table::TablesReport; -use storage::FeedStorage; - -/// Functions for tables informantion. -#[ async_trait::async_trait( ?Send ) ] -pub trait TableStore -{ - /// List tables in storage. - async fn list_tables( &mut self ) -> Result< TablesReport >; - - /// List columns of table. - async fn list_columns( &mut self, table_name : String ) -> Result< Vec< Payload > >; -} +use entity::table::TableStore; +use action::table::TablesReport; +use sled_adapter::FeedStorage; #[ async_trait::async_trait( ?Send ) ] impl TableStore for FeedStorage< SledStorage > { - async fn list_tables( &mut self ) -> Result< TablesReport > + async fn tables_list( &mut self ) -> Result< TablesReport > { let glue = &mut *self.storage.lock().await; let payloads = glue.execute( "SELECT * FROM GLUE_TABLE_COLUMNS" ).await?; @@ -35,7 +24,7 @@ impl TableStore for FeedStorage< SledStorage > Ok( report ) } - async fn list_columns( &mut self, table_name : String ) -> Result< Vec< Payload > > + async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > > { let glue = &mut *self.storage.lock().await; let query_str = format!( "SELECT * FROM GLUE_TABLE_COLUMNS WHERE TABLE_NAME='{}'", table_name ); @@ -43,5 +32,4 @@ impl TableStore for FeedStorage< SledStorage > Ok( payloads ) } - -} +} \ No newline at end of file diff --git a/module/move/unitore/src/storage/config.rs b/module/move/unitore/src/storage/config.rs deleted file mode 100644 index 5a3770ee8e..0000000000 --- a/module/move/unitore/src/storage/config.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! Functionality for storing and retrieving config files. - -use super::*; -use error_tools::{ err, Result }; -use gluesql:: -{ - core:: - { - ast_builder::{ col, table, text, Execute }, - executor::Payload, - }, - sled_storage::SledStorage, -}; - -/// Config file path. -#[ derive( Debug ) ] -pub struct Config( pub String ); - -impl Config -{ - /// Create new config with provided path. - pub fn new( path : String ) -> Self - { - Self( path ) - } - - /// Get path of config file. - pub fn path( &self ) -> String - { - self.0.clone() - } -} - -/// Functionality of config storing. -#[ async_trait::async_trait( ?Send ) ] -pub trait ConfigStore -{ - /// Add subscription. - async fn add_config( &mut self, config : &Config ) -> Result< Payload >; - - /// Remove subscription. - async fn delete_config( &mut self, config : &Config ) -> Result< Payload >; - - /// List subscriptions. - async fn list_configs( &mut self ) -> Result< Payload >; -} - -// qqq : port and adapters should not be in the same file -// Ideally, they should be in different crates, but you should at least put them in different folders -// there should be a `sled_adapter`` folder - -#[ async_trait::async_trait( ?Send ) ] -impl ConfigStore for FeedStorage< SledStorage > -{ - async fn add_config( &mut self, config : &Config ) -> Result< Payload > - { - let res = table( "config" ) - .insert() - .columns - ( - "path", - ) - .values( vec![ vec![ text( config.path() ) ] ] ) - .execute( &mut *self.storage.lock().await ) - .await; - - // let res = match &res - // { - // Err( err ) => - // { - // if let gluesql::core::error::Error::Validate( val_err ) = err - // { - // let res = match val_err - // { - // gluesql::core::error::ValidateError::DuplicateEntryOnPrimaryKeyField( _ ) => - // { - // res.context( "Config with same path already exists." ) - // }, - // _ => res.into() - // }; - - // res - // } - // res.into() - // }, - // Ok( _ ) => res.into(), - // }; - - Ok( res? ) - } - - async fn delete_config( &mut self, config : &Config ) -> Result< Payload > - { - let res = table( "config" ) - .delete() - .filter( col( "path" ).eq( format!( "'{}'", config.path() ) ) ) - .execute( &mut *self.storage.lock().await ) - .await?; - - if res == Payload::Delete( 0 ) - { - return Err( err!( format!( "Config file with path {} not found in storage", config.path() ) ) ) - } - - Ok( res ) - } - - async fn list_configs( &mut self ) -> Result< Payload > - { - let res = table( "config" ).select().execute( &mut *self.storage.lock().await ).await?; - Ok( res ) - } -} - -// qqq : use AbsolutePath newtype from `path_tools` -// qqq : normalize all paths with `path_tools::path::normalize` -// https://docs.rs/proper_path_tools/latest/proper_path_tools/path/fn.normalize.html - -// unitore .query.execute \'SELECT \* FROM feed\' -// qqq : something is broken in this table. also lack of association with config files - -// unitore .query.execute \'SELECT \* FROM x\' -// qqq : it is not obvious where one record ends and another begins diff --git a/module/move/unitore/src/storage/frame.rs b/module/move/unitore/src/storage/frame.rs deleted file mode 100644 index 49379e472f..0000000000 --- a/module/move/unitore/src/storage/frame.rs +++ /dev/null @@ -1,331 +0,0 @@ -//! Frame storing and retrieving functionality. - -use crate::*; -use std::collections::HashMap; -use error_tools::{ for_app::Context, Result }; -use gluesql:: -{ - core:: - { - ast_builder::{ null, col, table, text, Execute, timestamp, ExprNode }, - data::Value, - executor::Payload, - chrono::{ Utc, DateTime, SecondsFormat }, - }, - sled_storage::SledStorage, -}; - -use executor::actions::frame::{ FramesReport, ListReport, SelectedEntries }; -use storage::FeedStorage; -use wca::wtools::Itertools; - -/// Frame entity. -#[ derive( Debug ) ] -pub struct Frame -{ - /// Frame id. - pub id : String, - /// Frame title. - pub title : Option< String >, - updated : Option< DateTime< Utc > >, - authors : Option< String >, - content : Option< String >, - links : Option< String >, - summary : Option< String >, - categories : Option< String >, - published : Option< DateTime< Utc > >, - source : Option< String >, - rights : Option< String >, - media : Option< String >, - language : Option< String >, - feed_link : String, -} - -// qqq : not obvious -impl From< ( feed_rs::model::Entry, String ) > for Frame -{ - fn from( ( entry, feed_link ) : ( feed_rs::model::Entry, String ) ) -> Self - { - let authors = entry.authors - .iter() - .map( | p | p.name.clone() ) - .collect::< Vec< _ > >() - ; - - let content = entry.content - .map( | c | c.body.unwrap_or( c.src.map( | link | link.href ).unwrap_or_default() ) ) - .filter( | s | !s.is_empty() ) - .clone() - ; - - let mut links = entry.links - .iter() - .map( | link | link.href.clone() ) - .clone() - ; - - let categories = entry.categories - .iter() - .map( | cat | cat.term.clone() ) - .collect::< Vec< _ > >() - ; - - let media = entry.media - .iter() - .map( | m | m.content.clone() ) - .flatten() - .filter_map( | m | m.url.map( | url | url.to_string() ) ) - .collect::< Vec< _ > >() - ; - - Frame - { - id : entry.id, - title : entry.title.map( | title | title.content ).clone(), - updated : entry.updated.clone(), - authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), - // qqq : why join? - content, - links : ( !links.len() == 0 ).then( || links.join( ", " ) ), - // qqq : why join? - summary : entry.summary.map( | c | c.content ).clone(), - categories : ( !categories.is_empty() ).then( || categories.join( ", " ) ), - // qqq : why join? - published : entry.published.clone(), - source : entry.source.clone(), - rights : entry.rights.map( | r | r.content ).clone(), - media : ( !media.is_empty() ).then( || media.join( ", " ) ), - // qqq : why join? - language : entry.language.clone(), - feed_link, - } - } -} - -/// Frames storing and retrieving. -#[ async_trait::async_trait( ?Send ) ] -pub trait FrameStore -{ - /// Insert items from list into feed table. - async fn save_frames( &mut self, feed : Vec< Frame > ) -> Result< Payload >; - - /// Update items from list in feed table. - async fn update_frames( &mut self, feed : Vec< Frame > ) -> Result< () >; - - /// Get all feed frames from storage. - async fn list_frames( &mut self ) -> Result< ListReport >; -} -// qqq : what is update? what update? don't use word update without noun and explanation what deos it mean - -#[ async_trait::async_trait( ?Send ) ] -impl FrameStore for FeedStorage< SledStorage > -{ - async fn list_frames( &mut self ) -> Result< ListReport > - { - let res = table( "frame" ).select().execute( &mut *self.storage.lock().await ).await?; - - let mut reports = Vec::new(); - let all_frames = match res - { - Payload::Select { labels: label_vec, rows: rows_vec } => - { - SelectedEntries - { - selected_rows : rows_vec, - selected_columns : label_vec, - } - }, - _ => SelectedEntries::new(), - }; - - let mut feeds_map = HashMap::new(); - - for row in all_frames.selected_rows - { - let title_val = row.last().unwrap().clone(); - let title = String::from( title_val ); - feeds_map.entry( title ) - .and_modify( | vec : &mut Vec< Vec< Value > > | vec.push( row.clone() ) ) - .or_insert( vec![ row ] ) - ; - } - - for ( title, frames ) in feeds_map - { - let mut report = FramesReport::new( title ); - report.existing_frames = frames.len(); - report.selected_frames = SelectedEntries - { - selected_rows : frames, - selected_columns : all_frames.selected_columns.clone(), - }; - reports.push( report ); - } - - Ok( ListReport( reports ) ) - } - - async fn save_frames( &mut self, frames : Vec< Frame > ) -> Result< Payload > - { - let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); - - let insert = table( "frame" ) - .insert() - .columns - ( - self.frame_fields.iter().map( | field | field[ 0 ] ).join( "," ).as_str() - ) - .values( entries_rows ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to insert frames" )? - ; - - Ok( insert ) - } - - async fn update_frames( &mut self, feed : Vec< Frame > ) -> Result< () > - { - let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); - - for entry in entries_rows - { - let _update = table( "frame" ) - .update() - .set( "title", entry[ 1 ].to_owned() ) - .set( "content", entry[ 4 ].to_owned() ) - .set( "links", entry[ 5 ].to_owned() ) - .set( "summary", entry[ 6 ].to_owned() ) - .set( "published", entry[ 8 ].to_owned() ) - .set( "media", entry[ 9 ].to_owned() ) - .filter( col( "id" ).eq( entry[ 0 ].to_owned() ) ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to update frames" )? - ; - } - Ok( () ) - } -} - -// qqq : what is it for and why? -impl From< Frame > for Vec< ExprNode< 'static > > -{ - fn from( entry : Frame ) -> Self - { - let title = entry.title - .map( | title | text( title ) ) - .unwrap_or( null() ) - ; - - let updated = entry.updated - .map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ) - .unwrap_or( null() ) - ; - - let authors = entry.authors - .map( | authors | text( authors ) ) - .unwrap_or( null() ) - ; - - let content = entry.content - .map( | content | text ( content ) ) - .unwrap_or( null() ) - ; - - let links = entry.links - .map( | links | text ( links ) ) - .unwrap_or( null() ) - ; - - let summary = entry.summary - .map( | summary | text ( summary ) ) - .unwrap_or( null() ) - ; - - let categories = entry.categories - .map( | categories | text ( categories ) ) - .unwrap_or( null() ) - ; - - let published = entry.published - .map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ) - .unwrap_or( null() ) - ; - - let source = entry.source.map( | s | text( s ) ).unwrap_or( null() ); - let rights = entry.rights.map( | r | text( r ) ).unwrap_or( null() ); - let media = entry.media - .map( | media | text ( media ) ) - .unwrap_or( null() ) - ; - - let language = entry.language.clone().map( | l | text( l ) ).unwrap_or( null() ); - - vec! - [ - text( entry.id ), - title, - updated, - authors, - content, - links, - summary, - categories, - published, - source, - rights, - media, - language, - text( entry.feed_link ) - ] - } -} - -// qqq : RowValue or CellValue? -/// GlueSQL Value wrapper for display. -#[ derive( Debug ) ] -pub struct RowValue< 'a >( pub &'a gluesql::prelude::Value ); - -impl std::fmt::Display for RowValue< '_ > -{ - fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - use gluesql::prelude::Value::*; - match &self.0 - { - Bool( val ) => write!( f, "{}", val )?, - I8( val ) => write!( f, "{}", val )?, - I16( val ) => write!( f, "{}", val )?, - I32( val ) => write!( f, "{}", val )?, - I64( val ) => write!( f, "{}", val )?, - I128( val ) => write!( f, "{}", val )?, - U8( val ) => write!( f, "{}", val )?, - U16( val ) => write!( f, "{}", val )?, - U32( val ) => write!( f, "{}", val )?, - U64( val ) => write!( f, "{}", val )?, - U128( val ) => write!( f, "{}", val )?, - F32( val ) => write!( f, "{}", val )?, - F64( val ) => write!( f, "{}", val )?, - Str( val ) => write!( f, "{}", val )?, - Null => write!( f, "Null" )?, - Timestamp( val ) => write!( f, "{}", val )?, - _ => write!( f, "" )?, - } - - Ok( () ) - } -} - -impl From< RowValue< '_ > > for String -{ - fn from( value : RowValue< '_ > ) -> Self - { - use gluesql::core::data::Value::*; - match &value.0 - { - Str( val ) => val.clone(), - _ => String::new(), - } - } -} diff --git a/module/move/unitore/src/tool/mod.rs b/module/move/unitore/src/tool/mod.rs new file mode 100644 index 0000000000..749ba59a0e --- /dev/null +++ b/module/move/unitore/src/tool/mod.rs @@ -0,0 +1,3 @@ +//! Tools for additional functionality. + +pub mod table_display; \ No newline at end of file diff --git a/module/move/unitore/src/table_display.rs b/module/move/unitore/src/tool/table_display.rs similarity index 62% rename from module/move/unitore/src/table_display.rs rename to module/move/unitore/src/tool/table_display.rs index 9f334cc8ee..4b5f35475a 100644 --- a/module/move/unitore/src/table_display.rs +++ b/module/move/unitore/src/tool/table_display.rs @@ -1,12 +1,16 @@ -//! Helper for command report representation. +//! Wrapper for command report representation. +//! Separates usage of cli-table library behind facade for convenient changes in future. use cli_table:: { - format::{ Border, Separator }, Cell, Style, Table, TableDisplay + format::{ Border, HorizontalLine, Separator }, Cell, Style, Table, TableDisplay }; // qqq : purpose well defined should be always be in documentation -/// Wrapper struct for cli-table table with iplementation of Display. +// aaa : added explanation + +/// Wrapper struct for cli-table table with implementation of Display. +/// Separates usage of cli-table library behind facade for convenient changes in future. pub struct ReportTable( TableDisplay ); impl std::fmt::Display for ReportTable @@ -63,5 +67,22 @@ pub fn table_with_headers( headers : Vec< String >, rows : Vec< Vec< String > > .separator( Separator::builder().build() ) ; + table_struct.display().map( | table | ReportTable( table ) ).ok() +} + +/// Transform 2-dimensional vec of String data into displayable table with plain rows and bottom border. +pub fn plain_with_border( rows : Vec< Vec< String > > ) -> Option< ReportTable > +{ + let rows = rows + .into_iter() + .map( | row | row.into_iter().map( | cell_val | cell_val.cell() ).collect::< Vec< _ > >() ) + .collect::< Vec< _ > >() + ; + + let table_struct = rows.table() + .border( Border::builder().bottom(HorizontalLine::default()).build() ) + .separator( Separator::builder().build() ) + ; + table_struct.display().map( | table | ReportTable( table ) ).ok() } \ No newline at end of file diff --git a/module/move/unitore/tests/frame.rs b/module/move/unitore/tests/basic.rs similarity index 59% rename from module/move/unitore/tests/frame.rs rename to module/move/unitore/tests/basic.rs index 248f40330b..6e5df1ad4d 100644 --- a/module/move/unitore/tests/frame.rs +++ b/module/move/unitore/tests/basic.rs @@ -5,13 +5,9 @@ use error_tools::Result; async fn frame() -> Result< () > { let feed = feed_parser::parse( include_str!( "./fixtures/plain_feed.xml" ).as_bytes() )?; - - let frame = unitore::storage::frame::Frame::from( ( feed.entries[ 0 ].clone(), String::new() ) ); - + let frame = unitore::entity::frame::Frame::from( ( feed.entries[ 0 ].clone(), String::new() ) ); assert!( frame.id == feed.entries[ 0 ].id ); - println!( "{:#?}", feed.entries[ 0 ].media ); - println!( "{:#?}", frame ); Ok( () ) } diff --git a/module/move/unitore/tests/add_config.rs b/module/move/unitore/tests/config_add.rs similarity index 56% rename from module/move/unitore/tests/add_config.rs rename to module/move/unitore/tests/config_add.rs index 24e83d0d8a..a3de7479b7 100644 --- a/module/move/unitore/tests/add_config.rs +++ b/module/move/unitore/tests/config_add.rs @@ -1,29 +1,27 @@ use std::path::PathBuf; - use gluesql::sled_storage::sled::Config; use unitore:: { - executor::FeedManager, - storage::{ FeedStorage, feed::FeedStore }, + sled_adapter::FeedStorage, + entity::feed::FeedStore, + action::config, }; use error_tools::Result; #[ tokio::test ] -async fn add_config_file() -> Result< () > +async fn config_add() -> Result< () > { let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); - let path = path.canonicalize().expect( "Invalid path" ); let config = Config::default() - .path( "./test".to_owned() ) + .path( "./test_add".to_owned() ) .temporary( true ) ; - let feed_storage = FeedStorage::init_storage( config ).await?; - unitore::executor::actions::config::add_config( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &path ).await?; - let mut manager = FeedManager::new( feed_storage ); - let res = manager.storage.get_all_feeds().await?; + let res = feed_storage.feeds_list().await?; let feeds_links = res.0.selected_rows .iter() diff --git a/module/move/unitore/tests/config_delete.rs b/module/move/unitore/tests/config_delete.rs new file mode 100644 index 0000000000..95870d4700 --- /dev/null +++ b/module/move/unitore/tests/config_delete.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; +use gluesql:: +{ + sled_storage::sled::Config, + prelude::Payload::Select, +}; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::config::ConfigStore, + action::config, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn config_delete() -> Result< () > +{ + let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); + + let config = Config::default() + .path( "./test_del".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &path ).await?; + + config::config_delete( feed_storage.clone(), &path ).await?; + + let list = feed_storage.config_list().await?; + + if let Select{ labels : _, rows } = list + { + assert!( rows.len() == 0 ) + } + else + { + assert!( false ); + } + + Ok( () ) +} diff --git a/module/move/unitore/tests/fixtures/plain_feed.xml b/module/move/unitore/tests/fixtures/plain_feed.xml index 53c32e9fd1..7048caabd0 100644 --- a/module/move/unitore/tests/fixtures/plain_feed.xml +++ b/module/move/unitore/tests/fixtures/plain_feed.xml @@ -69,7 +69,7 @@ - + @@ -328,7 +328,7 @@ Thu, 14 Mar 2024 13:00:00 +0000 - + diff --git a/module/move/unitore/tests/frames_download.rs b/module/move/unitore/tests/frames_download.rs new file mode 100644 index 0000000000..ad5c3c2ff6 --- /dev/null +++ b/module/move/unitore/tests/frames_download.rs @@ -0,0 +1,117 @@ +use feed_rs::parser as feed_parser; +use gluesql:: +{ + core:: + { + chrono::{ DateTime, Utc }, + data::Value + }, + sled_storage::sled::Config, +}; +use wca::wtools::Itertools; +use unitore:: +{ + feed_config::SubscriptionConfig, + sled_adapter::FeedStorage, + entity::{ frame::FrameStore, feed::FeedStore }, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn test_save() -> Result< () > +{ + let config = gluesql::sled_storage::sled::Config::default() + .path( "./test_save".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + + let feed_config = SubscriptionConfig + { + update_period : std::time::Duration::from_secs( 1000 ), + link : url::Url::parse( "https://www.nasa.gov/feed/" )?, + }; + + let mut feeds = Vec::new(); + + let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; + feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); + feed_storage.feeds_process( feeds ).await?; + + let entries = feed_storage.frames_list().await?; + + let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); + assert_eq!( number_of_frames, 10 ); + + Ok( () ) +} + +#[ tokio::test ] +async fn test_update() -> Result< () > +{ + let config = Config::default() + .path( "./test_update".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + + let feed_config = SubscriptionConfig + { + update_period : std::time::Duration::from_secs( 1000 ), + link : url::Url::parse( "https://www.nasa.gov/feed/" )?, + }; + + // initial fetch + let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; + let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; + feed_storage.feeds_process( feeds ).await?; + + // updated fetch + let feed = feed_parser::parse( include_str!("./fixtures/updated_one_frame.xml").as_bytes() )?; + + let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; + feed_storage.feeds_process( feeds ).await?; + + // check + let payload = feed_storage.frames_list().await?; + + let entries = payload.0 + .iter() + .map( | val | val.selected_frames.selected_rows.clone() ) + .flatten() + .collect::< Vec< _ > >() + ; + + let entries = entries.iter().map( | entry | + { + let id = match &entry[ 0 ] + { + Value::Str( s ) => s.to_owned(), + _ => String::new(), + }; + + let published = match &entry[ 8 ] + { + Value::Timestamp( date_time ) => date_time.and_utc(), + _ => DateTime::< Utc >::default(), + }; + ( id, published ) + } + ) + .collect_vec() + ; + + // no duplicates + assert_eq!( entries.len(), 10 ); + + // check date + let updated = entries.iter().find + ( + | ( id, _published ) | id == "https://www.nasa.gov/?post_type=image-article&p=631537" + ); + assert!( updated.is_some() ); + let _updated = updated.unwrap(); + Ok( () ) +} diff --git a/module/move/unitore/tests/save_feed.rs b/module/move/unitore/tests/save_feed.rs deleted file mode 100644 index e6b20c18b6..0000000000 --- a/module/move/unitore/tests/save_feed.rs +++ /dev/null @@ -1,72 +0,0 @@ -use async_trait::async_trait; -use feed_rs::parser as feed_parser; -use unitore:: -{ - feed_config::SubscriptionConfig, - retriever::FeedFetch, - storage::{ FeedStorage, frame::FrameStore, feed::FeedStore }, -}; -use error_tools::Result; - -/// Feed client for testing. -#[derive(Debug)] -pub struct TestClient; - -#[ async_trait ] -impl FeedFetch for TestClient -{ - async fn fetch( &self, _ : url::Url ) -> Result< feed_rs::model::Feed > - { - let feed = feed_parser::parse( include_str!( "./fixtures/plain_feed.xml" ).as_bytes() )?; - - Ok( feed ) - } -} - -#[ tokio::test ] -async fn test_save_feed_plain() -> Result< () > -{ - // let mut f_store = MockFeedStore::new(); - // f_store - // .expect_process_feeds() - // .times( 1 ) - // .returning( | _ | Ok( UpdateReport( - // vec! [ FramesReport - // { - // new_frames : 2, - // updated_frames : 0, - // selected_frames : SelectedEntries::new(), - // existing_frames : 0, - // feed_link : String::new(), - // is_new_feed : false, - // } ] ) ) ) - // ; - - let config = gluesql::sled_storage::sled::Config::default() - .path( "./test".to_owned() ) - .temporary( true ) - ; - - let mut feed_storage = FeedStorage::init_storage( config ).await?; - - let feed_config = SubscriptionConfig - { - update_period : std::time::Duration::from_secs( 1000 ), - link : url::Url::parse( "https://www.nasa.gov/feed/" )?, - }; - - let mut feeds = Vec::new(); - let client = TestClient; - - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; - feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); - feed_storage.process_feeds( feeds ).await?; - - let entries = feed_storage.list_frames().await?; - - let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); - - assert_eq!( number_of_frames, 10 ); - - Ok( () ) -} diff --git a/module/move/unitore/tests/table_list.rs b/module/move/unitore/tests/table_list.rs new file mode 100644 index 0000000000..dc840b3633 --- /dev/null +++ b/module/move/unitore/tests/table_list.rs @@ -0,0 +1,45 @@ +use gluesql:: +{ + sled_storage::sled::Config, + prelude::{ Payload, Value::Str }, +}; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::table::TableStore, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn table_list() -> Result< () > +{ + let config = Config::default() + .path( "./test_list".to_owned() ) + .temporary( true ) + ; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + + let res = feed_storage.table_list( String::from( "feed" ) ).await?; + + if let Payload::Select { labels: _, rows } = &res[ 0 ] + { + let column_names = rows + .iter() + .map( | row | row[ 1 ].clone() ) + .collect::< Vec< _ > >() + ; + + assert_eq!( column_names.len(), 9 ); + assert!( column_names.contains( &Str( String::from( "published") ) ) ); + assert!( column_names.contains( &Str( String::from( "authors") ) ) ); + assert!( column_names.contains( &Str( String::from( "description") ) ) ); + assert!( column_names.contains( &Str( String::from( "type") ) ) ); + assert!( column_names.contains( &Str( String::from( "title") ) ) ); + assert!( column_names.contains( &Str( String::from( "updated") ) ) ); + assert!( column_names.contains( &Str( String::from( "link") ) ) ); + assert!( column_names.contains( &Str( String::from( "update_period" ) ) ) ); + assert!( column_names.contains( &Str( String::from( "config_file" ) ) ) ); + } + + Ok( () ) +} diff --git a/module/move/unitore/tests/tables_list.rs b/module/move/unitore/tests/tables_list.rs new file mode 100644 index 0000000000..6b306b1c19 --- /dev/null +++ b/module/move/unitore/tests/tables_list.rs @@ -0,0 +1,32 @@ +use gluesql::sled_storage::sled::Config; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::table::TableStore, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn tables_list() -> Result< () > +{ + let config = Config::default() + .path( "./test_list".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + let res = feed_storage.tables_list().await?; + + let table_names = res.0 + .iter() + .map( | ( table_name, _info ) | table_name ) + .collect::< Vec< _ > >() + ; + + assert_eq!( table_names.len(), 3 ); + assert!( table_names.contains( &&String::from( "config") ) ); + assert!( table_names.contains( &&String::from( "feed" ) ) ); + assert!( table_names.contains( &&String::from( "frame" ) ) ); + + Ok( () ) +} diff --git a/module/move/unitore/tests/update_newer_feed.rs b/module/move/unitore/tests/update_newer_feed.rs deleted file mode 100644 index 324ed68556..0000000000 --- a/module/move/unitore/tests/update_newer_feed.rs +++ /dev/null @@ -1,98 +0,0 @@ -use async_trait::async_trait; -use feed_rs::parser as feed_parser; -use gluesql:: -{ - core:: - { - chrono::{ DateTime, Utc }, - data::Value - }, - sled_storage::sled::Config, -}; -use unitore:: -{ - feed_config::SubscriptionConfig, - retriever::FeedFetch, - storage::{ feed::FeedStore, frame::FrameStore, FeedStorage }, -}; -use wca::wtools::Itertools; -use error_tools::Result; - -/// Feed client for testing. -#[derive(Debug)] -pub struct TestClient ( String ); - -#[ async_trait ] -impl FeedFetch for TestClient -{ - async fn fetch( &self, _ : url::Url ) -> Result< feed_rs::model::Feed > - { - let feed = feed_parser::parse( std::fs::read_to_string( &self.0 )?.as_bytes() )?; - Ok( feed ) - } -} - -#[ tokio::test ] -async fn test_update() -> Result< () > -{ - let config = Config::default() - .path( "./test".to_owned() ) - .temporary( true ) - ; - - let mut feed_storage = FeedStorage::init_storage( config ).await?; - - let feed_config = SubscriptionConfig - { - update_period : std::time::Duration::from_secs( 1000 ), - link : url::Url::parse( "https://www.nasa.gov/feed/" )?, - }; - - // initial fetch - let client = TestClient( "./tests/fixtures/plain_feed.xml".to_owned() ); - - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; - let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; - feed_storage.process_feeds( feeds ).await?; - - // updated fetch - let client = TestClient( "./tests/fixtures/updated_one_frame.xml".to_owned() ); - - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; - let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; - feed_storage.process_feeds( feeds ).await?; - - // check - let payload = feed_storage.list_frames().await?; - - let entries = payload.0.iter().map( | val | val.selected_frames.selected_rows.clone() ).flatten().collect::< Vec< _ > >(); - - let entries = entries.iter().map( | entry | - { - let id = match &entry[ 0 ] - { - Value::Str( s ) => s.to_owned(), - _ => String::new(), - }; - - let published = match &entry[ 8 ] - { - Value::Timestamp( date_time ) => date_time.and_utc(), - _ => DateTime::< Utc >::default(), - }; - ( id, published ) - } - ) - .collect_vec() - ; - - // no duplicates - assert_eq!( entries.len(), 10 ); - - // check date - println!( "{:?}", entries ); - let updated = entries.iter().find( | ( id, _published ) | id == "https://www.nasa.gov/?post_type=image-article&p=631537" ); - assert!( updated.is_some() ); - let _updated = updated.unwrap(); - Ok( () ) -} \ No newline at end of file