From 8ada86676593d18cd929b6479931a81ac19f550c Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Mon, 1 Apr 2024 15:48:57 +0300 Subject: [PATCH 01/11] fix naming --- .../unitore/src/executor/actions/config.rs | 15 +- .../move/unitore/src/executor/actions/feed.rs | 3 +- .../unitore/src/executor/actions/frame.rs | 25 ++- .../unitore/src/executor/actions/query.rs | 5 +- .../unitore/src/executor/actions/table.rs | 153 ++++++++++++++---- module/move/unitore/src/executor/mod.rs | 30 ++-- module/move/unitore/src/main.rs | 5 +- module/move/unitore/src/storage/config.rs | 38 +---- module/move/unitore/src/storage/feed.rs | 8 +- module/move/unitore/src/storage/frame.rs | 41 ++--- module/move/unitore/src/storage/table.rs | 10 +- .../tests/{add_config.rs => config_add.rs} | 4 +- module/move/unitore/tests/save_feed.rs | 2 +- .../move/unitore/tests/update_newer_feed.rs | 2 +- 14 files changed, 204 insertions(+), 137 deletions(-) rename module/move/unitore/tests/{add_config.rs => config_add.rs} (86%) diff --git a/module/move/unitore/src/executor/actions/config.rs b/module/move/unitore/src/executor/actions/config.rs index 9b01caf173..cea9b3e517 100644 --- a/module/move/unitore/src/executor/actions/config.rs +++ b/module/move/unitore/src/executor/actions/config.rs @@ -1,7 +1,6 @@ //! Endpoint and report for commands for config files. use crate::*; -use super::*; use error_tools::{ err, for_app::Context, BasicError, Result }; use executor::FeedManager; use storage:: @@ -13,7 +12,7 @@ use storage:: use gluesql::{ prelude::Payload, sled_storage::SledStorage }; /// Add configuration file with subscriptions to storage. -pub async fn add_config( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl executor::Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -43,7 +42,7 @@ pub async fn add_config( storage : FeedStorage< SledStorage >, args : &wca::Args let mut manager = FeedManager::new( storage ); let config_report = manager.storage - .add_config( &config ) + .config_add( &config ) .await .context( "Added 0 config files.\n Failed to add config file to storage." )? ; @@ -60,7 +59,7 @@ pub async fn add_config( storage : FeedStorage< SledStorage >, args : &wca::Args } /// Remove configuration file from storage. -pub async fn delete_config( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl executor::Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -74,17 +73,17 @@ pub async fn delete_config( storage : FeedStorage< SledStorage >, args : &wca::A let mut manager = FeedManager::new( storage ); Ok( ConfigReport::new( manager.storage - .delete_config( &config ) + .config_delete( &config ) .await .context( "Failed to remove config from storage." )? ) ) } /// List all files with subscriptions that are currently in storage. -pub async fn list_configs( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > +pub async fn config_list( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl executor::Report > { let mut manager = FeedManager::new( storage ); - Ok( ConfigReport::new( manager.storage.list_configs().await? ) ) + Ok( ConfigReport::new( manager.storage.config_list().await? ) ) } /// Information about result of command for subscription config. @@ -153,4 +152,4 @@ impl std::fmt::Display for ConfigReport } } -impl Report for ConfigReport {} +impl executor::Report for ConfigReport {} diff --git a/module/move/unitore/src/executor/actions/feed.rs b/module/move/unitore/src/executor/actions/feed.rs index 850384c846..92863c0317 100644 --- a/module/move/unitore/src/executor/actions/feed.rs +++ b/module/move/unitore/src/executor/actions/feed.rs @@ -10,7 +10,8 @@ use storage::{ FeedStorage, feed::FeedStore }; use error_tools::Result; /// List all feeds. -pub async fn list_feeds( +pub async fn feeds_list +( storage : FeedStorage< gluesql::sled_storage::SledStorage >, _args : &wca::Args, ) -> Result< impl Report > diff --git a/module/move/unitore/src/executor/actions/frame.rs b/module/move/unitore/src/executor/actions/frame.rs index 8ce982c1cc..14b8bd9098 100644 --- a/module/move/unitore/src/executor/actions/frame.rs +++ b/module/move/unitore/src/executor/actions/frame.rs @@ -1,7 +1,6 @@ //! Frames commands actions. use crate::*; -use super::*; use executor::FeedManager; use storage:: { @@ -15,25 +14,25 @@ use feed_config; use error_tools::{ err, Result }; /// List all frames. -pub async fn list_frames +pub async fn frames_list ( storage : FeedStorage< SledStorage >, _args : &wca::Args, -) -> Result< impl Report > +) -> Result< impl executor::Report > { let mut manager = FeedManager::new( storage ); - manager.storage.list_frames().await + manager.storage.frames_list().await } /// Update all frames from config files saved in storage. -pub async fn download_frames +pub async fn frames_download ( storage : FeedStorage< SledStorage >, _args : &wca::Args, -) -> Result< impl Report > +) -> Result< impl executor::Report > { let mut manager = FeedManager::new( storage ); - let payload = manager.storage.list_configs().await?; + let payload = manager.storage.config_list().await?; let configs = match &payload { @@ -69,10 +68,10 @@ pub async fn download_frames let mut feeds = Vec::new(); let client = retriever::FeedClient; - for i in 0..subscriptions.len() + for subscription in subscriptions { - let feed = retriever::FeedFetch::fetch(&client, subscriptions[ i ].link.clone()).await?; - feeds.push( ( feed, subscriptions[ i ].update_period.clone(), subscriptions[ i ].link.clone() ) ); + let feed = retriever::FeedFetch::fetch(&client, subscription.link.clone()).await?; + feeds.push( ( feed, subscription.update_period.clone(), subscription.link ) ); } manager.storage.process_feeds( feeds ).await @@ -170,7 +169,7 @@ impl std::fmt::Display for FramesReport } } -impl Report for FramesReport {} +impl executor::Report for FramesReport {} /// Items get from select query from storage. #[ derive( Debug ) ] @@ -237,7 +236,7 @@ impl std::fmt::Display for UpdateReport } } -impl Report for UpdateReport {} +impl executor::Report for UpdateReport {} /// Report for listing frames. #[ derive( Debug ) ] @@ -269,4 +268,4 @@ impl std::fmt::Display for ListReport } } -impl Report for ListReport {} +impl executor::Report for ListReport {} diff --git a/module/move/unitore/src/executor/actions/query.rs b/module/move/unitore/src/executor/actions/query.rs index 4b49bc9c37..d022076554 100644 --- a/module/move/unitore/src/executor/actions/query.rs +++ b/module/move/unitore/src/executor/actions/query.rs @@ -1,14 +1,13 @@ //! Query command endpoint and report. use crate::*; -use super::*; use gluesql::core::executor::Payload; use storage::{ FeedStorage, Store }; -use executor::FeedManager; +use executor::{ FeedManager, actions::Report }; use error_tools::{ err, BasicError, Result }; /// Execute query specified in query string. -pub async fn execute_query +pub async fn query_execute ( storage : FeedStorage< gluesql::sled_storage::SledStorage >, args : &wca::Args, diff --git a/module/move/unitore/src/executor/actions/table.rs b/module/move/unitore/src/executor/actions/table.rs index d22ad6eeff..7c88669833 100644 --- a/module/move/unitore/src/executor/actions/table.rs +++ b/module/move/unitore/src/executor/actions/table.rs @@ -8,7 +8,7 @@ use storage::{ FeedStorage, table::TableStore }; use error_tools::{ err, BasicError, Result }; /// Get labels of column for specified table. -pub async fn list_columns +pub async fn table_list ( storage : FeedStorage< gluesql::sled_storage::SledStorage >, args : &wca::Args, @@ -21,24 +21,20 @@ pub async fn list_columns ; let mut manager = FeedManager::new( storage ); - let result = manager.storage.list_columns( table_name.clone() ).await?; + let result = manager.storage.table_list( table_name.clone() ).await?; let mut table_description = String::new(); - let mut columns = std::collections::HashMap::new(); - match &result[ 0 ] + let mut columns = HashMap::new(); + if let Payload::Select { labels: _label_vec, rows: rows_vec } = &result[ 0 ] { - Payload::Select { labels: _label_vec, rows: rows_vec } => + for row in rows_vec { - for row in rows_vec - { - let table = String::from( row[ 0 ].clone() ); - columns.entry( table ) - .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) - .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) - ; - } - }, - _ => {}, + let table = String::from( row[ 0 ].clone() ); + columns.entry( table ) + .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) + .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) + ; + } } let mut columns_desc = HashMap::new(); match table_name.as_str() @@ -88,20 +84,115 @@ pub async fn list_columns { match label.as_str() { - "id" => { columns_desc.insert( label.clone(), String::from( "A unique identifier for this frame in the feed. " ) ); }, - "title" => { columns_desc.insert( label.clone(), String::from( "Title of the frame" ) ); }, - "updated" => { columns_desc.insert( label.clone(), String::from( "Time at which this item was fetched from source." ) ); }, - "authors" => { columns_desc.insert( label.clone(), String::from( "List of authors of the frame, optional." ) ); }, - "content" => { columns_desc.insert( label.clone(), String::from( "The content of the frame in html or plain text, optional." ) ); }, - "links" => { columns_desc.insert( label.clone(), String::from( "List of links associated with this item of related Web page and attachments." ) ); }, - "summary" => { columns_desc.insert( label.clone(), String::from( "Short summary, abstract, or excerpt of the frame item, optional." ) ); }, - "categories" => { columns_desc.insert( label.clone(), String::from( "Specifies a list of categories that the item belongs to." ) ); }, - "published" => { columns_desc.insert( label.clone(), String::from( "Time at which this item was first published or updated." ) ); }, - "source" => { columns_desc.insert( label.clone(), String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ) ); }, - "rights" => { columns_desc.insert( label.clone(), String::from( "Conveys information about copyrights over the feed, optional." ) ); }, - "media" => { columns_desc.insert( label.clone(), String::from( "List of media oblects, encountered in the frame, optional." ) ); }, - "language" => { columns_desc.insert( label.clone(), String::from( "The language specified on the item, optional." ) ); }, - "feed_link" => { columns_desc.insert( label.clone(), String::from( "Link of feed that contains this frame." ) ); }, + "id" => + { + columns_desc.insert + ( + label.clone(), + String::from( "A unique identifier for this frame in the feed. " ), + ); + }, + "title" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Title of the frame" ), + ); + }, + "updated" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was fetched from source." ), + ); + }, + "authors" => + { + columns_desc.insert( + label.clone(), + String::from( "List of authors of the frame, optional." ) + ); + }, + "content" => + { + columns_desc.insert( + label.clone(), + String::from( "The content of the frame in html or plain text, optional." ), + ); + }, + "links" => + { + columns_desc.insert( + label.clone(), + String::from( "List of links associated with this item of related Web page and attachments." ), + ); + }, + "summary" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Short summary, abstract, or excerpt of the frame item, optional." ), + ); + }, + "categories" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies a list of categories that the item belongs to." ), + ); + }, + "published" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was first published or updated." ), + ); + }, + "source" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ), + ); + }, + "rights" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Conveys information about copyrights over the feed, optional." ), + ); + }, + "media" => + { + columns_desc.insert + ( + label.clone(), + String::from( "List of media oblects, encountered in the frame, optional." ), + ); + }, + "language" => + { + columns_desc.insert + ( + label.clone(), + String::from( "The language specified on the item, optional." ), + ); + }, + "feed_link" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Link of feed that contains this frame." ), + ); + }, _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } } } @@ -125,14 +216,14 @@ pub async fn list_columns } /// Get names of tables in storage. -pub async fn list_tables +pub async fn tables_list ( storage : FeedStorage< gluesql::sled_storage::SledStorage >, _args : &wca::Args, ) -> Result< impl Report > { let mut manager = FeedManager::new( storage ); - manager.storage.list_tables().await + manager.storage.tables_list().await } const EMPTY_CELL : &'static str = ""; diff --git a/module/move/unitore/src/executor/mod.rs b/module/move/unitore/src/executor/mod.rs index e83d8c859e..0b344e337e 100644 --- a/module/move/unitore/src/executor/mod.rs +++ b/module/move/unitore/src/executor/mod.rs @@ -1,6 +1,6 @@ //! Execute plan. -use super::*; +use crate::*; use feed_config::SubscriptionConfig; use gluesql::sled_storage::{ sled::Config, SledStorage }; use retriever::{ FeedClient, FeedFetch }; @@ -12,11 +12,11 @@ use error_tools::Result; pub mod actions; use actions:: { - frame::{ list_frames, download_frames }, - feed::list_feeds, - config::{ add_config, delete_config, list_configs }, - query::execute_query, - table::{ list_columns, list_tables }, + frame::{ frames_list, frames_download }, + feed::feeds_list, + config::{ config_add, config_delete, config_list }, + query::query_execute, + table::{ table_list, tables_list }, }; fn action< 'a, F, Fut, R >( async_endpoint : F, args : &'a Args ) -> Result< R > @@ -54,7 +54,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > )) .routine( | args | { - match action( download_frames, &args ) + match action( frames_download, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -70,7 +70,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > )) .routine( | args | { - match action( list_feeds, &args ) + match action( feeds_list, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -86,7 +86,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > )) .routine( | args | { - match action( list_frames, &args ) + match action( frames_list, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -110,7 +110,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() .routine( | args : Args | { - match action( add_config, &args ) + match action( config_add, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -127,7 +127,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() .routine( | args : Args | { - match action( delete_config, &args ) + match action( config_delete, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -143,7 +143,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > )) .routine( | args | { - match action( list_configs, &args ) + match action( config_list, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -159,7 +159,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > )) .routine( | args | { - match action( list_tables, &args ) + match action( tables_list, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -177,7 +177,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > .subject().hint( "Name" ).kind( wca::Type::String ).optional( false ).end() .routine( | args : Args | { - match action( list_columns, &args ) + match action( table_list, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), @@ -200,7 +200,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > .subject().hint( "Query" ).kind( Type::List( Type::String.into(), ' ' ) ).optional( false ).end() .routine( | args : Args | { - match action( execute_query, &args ) + match action( query_execute, &args ) { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), diff --git a/module/move/unitore/src/main.rs b/module/move/unitore/src/main.rs index 12ce305f31..d9828ef31b 100644 --- a/module/move/unitore/src/main.rs +++ b/module/move/unitore/src/main.rs @@ -1,6 +1,5 @@ -//! -// use unitore::retriever::FeedClient; -// use unitore::feed_config::read_feed_config; +//! Runs unitore command executor. + pub use unitore::executor; fn main() -> Result< (), Box< dyn std::error::Error + Send + Sync > > diff --git a/module/move/unitore/src/storage/config.rs b/module/move/unitore/src/storage/config.rs index 39a4d3c1fd..b66bf4c910 100644 --- a/module/move/unitore/src/storage/config.rs +++ b/module/move/unitore/src/storage/config.rs @@ -1,6 +1,6 @@ //! Functionality for storing and retrieving config files. -use super::*; +use crate::*; use error_tools::{ err, Result }; use gluesql:: { @@ -36,19 +36,19 @@ impl Config pub trait ConfigStore { /// Add subscription. - async fn add_config( &mut self, config : &Config ) -> Result< Payload >; + async fn config_add( &mut self, config : &Config ) -> Result< Payload >; /// Remove subscription. - async fn delete_config( &mut self, config : &Config ) -> Result< Payload >; + async fn config_delete( &mut self, config : &Config ) -> Result< Payload >; /// List subscriptions. - async fn list_configs( &mut self ) -> Result< Payload >; + async fn config_list( &mut self ) -> Result< Payload >; } #[ async_trait::async_trait( ?Send ) ] -impl ConfigStore for FeedStorage< SledStorage > +impl ConfigStore for storage::FeedStorage< SledStorage > { - async fn add_config( &mut self, config : &Config ) -> Result< Payload > + async fn config_add( &mut self, config : &Config ) -> Result< Payload > { let res = table( "config" ) .insert() @@ -60,32 +60,10 @@ impl ConfigStore for FeedStorage< SledStorage > .execute( &mut *self.storage.lock().await ) .await; - // let res = match &res - // { - // Err( err ) => - // { - // if let gluesql::core::error::Error::Validate( val_err ) = err - // { - // let res = match val_err - // { - // gluesql::core::error::ValidateError::DuplicateEntryOnPrimaryKeyField( _ ) => - // { - // res.context( "Config with same path already exists." ) - // }, - // _ => res.into() - // }; - - // res - // } - // res.into() - // }, - // Ok( _ ) => res.into(), - // }; - Ok( res? ) } - async fn delete_config( &mut self, config : &Config ) -> Result< Payload > + async fn config_delete( &mut self, config : &Config ) -> Result< Payload > { let res = table( "config" ) .delete() @@ -101,7 +79,7 @@ impl ConfigStore for FeedStorage< SledStorage > Ok( res ) } - async fn list_configs( &mut self ) -> Result< Payload > + async fn config_list( &mut self ) -> Result< Payload > { let res = table( "config" ).select().execute( &mut *self.storage.lock().await ).await?; Ok( res ) diff --git a/module/move/unitore/src/storage/feed.rs b/module/move/unitore/src/storage/feed.rs index 59d612bb6d..36ddec17d7 100644 --- a/module/move/unitore/src/storage/feed.rs +++ b/module/move/unitore/src/storage/feed.rs @@ -198,13 +198,13 @@ impl FeedStore for FeedStorage< SledStorage > reports.push( frames_report ); } - if new_entries.len() > 0 + if !new_entries.is_empty() { - let _saved_report = self.save_frames( new_entries ).await?; + let _saved_report = self.frames_save( new_entries ).await?; } - if modified_entries.len() > 0 + if !modified_entries.is_empty() { - let _updated_report = self.update_frames( modified_entries ).await?; + let _updated_report = self.frames_update( modified_entries ).await?; } Ok( UpdateReport( reports ) ) diff --git a/module/move/unitore/src/storage/frame.rs b/module/move/unitore/src/storage/frame.rs index cb4d736b35..4ea353f970 100644 --- a/module/move/unitore/src/storage/frame.rs +++ b/module/move/unitore/src/storage/frame.rs @@ -71,8 +71,7 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame let media = entry.media .iter() - .map( | m | m.content.clone() ) - .flatten() + .flat_map( | m | m.content.clone() ) .filter_map( | m | m.url.map( | url | url.to_string() ) ) .collect::< Vec< _ > >() ; @@ -81,13 +80,13 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame { id : entry.id, title : entry.title.map( | title | title.content ).clone(), - updated : entry.updated.clone(), + updated : entry.updated, authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), content, links : ( !links.len() == 0 ).then( || links.join( ", " ) ), summary : entry.summary.map( | c | c.content ).clone(), categories : ( !categories.is_empty() ).then( || categories.join( ", " ) ), - published : entry.published.clone(), + published : entry.published, source : entry.source.clone(), rights : entry.rights.map( | r | r.content ).clone(), media : ( !media.is_empty() ).then( || media.join( ", " ) ), @@ -102,35 +101,37 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame pub trait FrameStore { /// Insert items from list into feed table. - async fn save_frames( &mut self, feed : Vec< Frame > ) -> Result< Payload >; + async fn frames_save( &mut self, feed : Vec< Frame > ) -> Result< Payload >; /// Update items from list in feed table. - async fn update_frames( &mut self, feed : Vec< Frame > ) -> Result< () >; + async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () >; /// Get all feed frames from storage. - async fn list_frames( &mut self ) -> Result< ListReport >; + async fn frames_list( &mut self ) -> Result< ListReport >; } #[ async_trait::async_trait( ?Send ) ] impl FrameStore for FeedStorage< SledStorage > { - async fn list_frames( &mut self ) -> Result< ListReport > + async fn frames_list( &mut self ) -> Result< ListReport > { let res = table( "frame" ).select().execute( &mut *self.storage.lock().await ).await?; let mut reports = Vec::new(); - let all_frames = match res + let all_frames = + if let Payload::Select { labels: label_vec, rows: rows_vec } = res { - Payload::Select { labels: label_vec, rows: rows_vec } => + SelectedEntries { - SelectedEntries - { - selected_rows : rows_vec, - selected_columns : label_vec, - } - }, - _ => SelectedEntries::new(), + selected_rows : rows_vec, + selected_columns : label_vec, + } + } + else + { + SelectedEntries::new() }; + let mut feeds_map = HashMap::new(); @@ -159,7 +160,7 @@ impl FrameStore for FeedStorage< SledStorage > Ok( ListReport( reports ) ) } - async fn save_frames( &mut self, frames : Vec< Frame > ) -> Result< Payload > + async fn frames_save( &mut self, frames : Vec< Frame > ) -> Result< Payload > { let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); @@ -178,7 +179,7 @@ impl FrameStore for FeedStorage< SledStorage > Ok( insert ) } - async fn update_frames( &mut self, feed : Vec< Frame > ) -> Result< () > + async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () > { let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); @@ -253,7 +254,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > .unwrap_or( null() ) ; - let language = entry.language.clone().map( | l | text( l ) ).unwrap_or( null() ); + let language = entry.language.clone().map( text ).unwrap_or( null() ); vec! [ diff --git a/module/move/unitore/src/storage/table.rs b/module/move/unitore/src/storage/table.rs index 08038df8ee..60428ce0ce 100644 --- a/module/move/unitore/src/storage/table.rs +++ b/module/move/unitore/src/storage/table.rs @@ -1,4 +1,4 @@ -//! Tables sroring functions. +//! Functionality for storage tables information. use crate::*; use error_tools::Result; @@ -16,16 +16,16 @@ use storage::FeedStorage; pub trait TableStore { /// List tables in storage. - async fn list_tables( &mut self ) -> Result< TablesReport >; + async fn tables_list( &mut self ) -> Result< TablesReport >; /// List columns of table. - async fn list_columns( &mut self, table_name : String ) -> Result< Vec< Payload > >; + async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > >; } #[ async_trait::async_trait( ?Send ) ] impl TableStore for FeedStorage< SledStorage > { - async fn list_tables( &mut self ) -> Result< TablesReport > + async fn tables_list( &mut self ) -> Result< TablesReport > { let glue = &mut *self.storage.lock().await; let payloads = glue.execute( "SELECT * FROM GLUE_TABLE_COLUMNS" ).await?; @@ -35,7 +35,7 @@ impl TableStore for FeedStorage< SledStorage > Ok( report ) } - async fn list_columns( &mut self, table_name : String ) -> Result< Vec< Payload > > + async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > > { let glue = &mut *self.storage.lock().await; let query_str = format!( "SELECT * FROM GLUE_TABLE_COLUMNS WHERE TABLE_NAME='{}'", table_name ); diff --git a/module/move/unitore/tests/add_config.rs b/module/move/unitore/tests/config_add.rs similarity index 86% rename from module/move/unitore/tests/add_config.rs rename to module/move/unitore/tests/config_add.rs index 24e83d0d8a..502ec144f0 100644 --- a/module/move/unitore/tests/add_config.rs +++ b/module/move/unitore/tests/config_add.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use gluesql::sled_storage::sled::Config; use unitore:: { - executor::FeedManager, + executor::{ FeedManager, actions }, storage::{ FeedStorage, feed::FeedStore }, }; use error_tools::Result; @@ -20,7 +20,7 @@ async fn add_config_file() -> Result< () > ; let feed_storage = FeedStorage::init_storage( config ).await?; - unitore::executor::actions::config::add_config( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; + actions::config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; let mut manager = FeedManager::new( feed_storage ); let res = manager.storage.get_all_feeds().await?; diff --git a/module/move/unitore/tests/save_feed.rs b/module/move/unitore/tests/save_feed.rs index e6b20c18b6..75dc54b5f7 100644 --- a/module/move/unitore/tests/save_feed.rs +++ b/module/move/unitore/tests/save_feed.rs @@ -62,7 +62,7 @@ async fn test_save_feed_plain() -> Result< () > feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); feed_storage.process_feeds( feeds ).await?; - let entries = feed_storage.list_frames().await?; + let entries = feed_storage.frames_list().await?; let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); diff --git a/module/move/unitore/tests/update_newer_feed.rs b/module/move/unitore/tests/update_newer_feed.rs index 324ed68556..c95d50619b 100644 --- a/module/move/unitore/tests/update_newer_feed.rs +++ b/module/move/unitore/tests/update_newer_feed.rs @@ -63,7 +63,7 @@ async fn test_update() -> Result< () > feed_storage.process_feeds( feeds ).await?; // check - let payload = feed_storage.list_frames().await?; + let payload = feed_storage.frames_list().await?; let entries = payload.0.iter().map( | val | val.selected_frames.selected_rows.clone() ).flatten().collect::< Vec< _ > >(); From 84805b825ac372cd6053b4d7e337f931dd35c6fd Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Thu, 4 Apr 2024 14:56:18 +0300 Subject: [PATCH 02/11] change file structure --- .../{executor/actions => action}/config.rs | 19 ++-- .../src/{executor/actions => action}/feed.rs | 16 ++-- .../src/{executor/actions => action}/frame.rs | 39 ++++---- .../src/{executor/actions => action}/mod.rs | 0 .../src/{executor/actions => action}/query.rs | 6 +- .../src/{executor/actions => action}/table.rs | 12 ++- module/move/unitore/src/command/mod.rs | 1 + module/move/unitore/src/command/table.rs | 56 ++++++++++++ .../unitore/src/{storage => entity}/config.rs | 0 .../unitore/src/{storage => entity}/feed.rs | 91 ++++++++++--------- .../unitore/src/{storage => entity}/frame.rs | 32 ++++--- module/move/unitore/src/entity/mod.rs | 7 ++ .../unitore/src/{storage => entity}/table.rs | 3 +- .../src/{executor/mod.rs => executor.rs} | 35 +++---- module/move/unitore/src/lib.rs | 5 +- module/move/unitore/src/retriever.rs | 13 +-- .../src/{storage/mod.rs => storage.rs} | 15 +-- module/move/unitore/src/tool/mod.rs | 1 + .../unitore/src/{ => tool}/table_display.rs | 27 +++++- module/move/unitore/tests/config_add.rs | 12 ++- module/move/unitore/tests/frame.rs | 2 +- module/move/unitore/tests/save_feed.rs | 27 +----- .../move/unitore/tests/update_newer_feed.rs | 32 ++----- 23 files changed, 253 insertions(+), 198 deletions(-) rename module/move/unitore/src/{executor/actions => action}/config.rs (90%) rename module/move/unitore/src/{executor/actions => action}/feed.rs (83%) rename module/move/unitore/src/{executor/actions => action}/frame.rs (86%) rename module/move/unitore/src/{executor/actions => action}/mod.rs (100%) rename module/move/unitore/src/{executor/actions => action}/query.rs (95%) rename module/move/unitore/src/{executor/actions => action}/table.rs (97%) create mode 100644 module/move/unitore/src/command/mod.rs create mode 100644 module/move/unitore/src/command/table.rs rename module/move/unitore/src/{storage => entity}/config.rs (100%) rename module/move/unitore/src/{storage => entity}/feed.rs (74%) rename module/move/unitore/src/{storage => entity}/frame.rs (89%) create mode 100644 module/move/unitore/src/entity/mod.rs rename module/move/unitore/src/{storage => entity}/table.rs (96%) rename module/move/unitore/src/{executor/mod.rs => executor.rs} (87%) rename module/move/unitore/src/{storage/mod.rs => storage.rs} (92%) create mode 100644 module/move/unitore/src/tool/mod.rs rename module/move/unitore/src/{ => tool}/table_display.rs (62%) diff --git a/module/move/unitore/src/executor/actions/config.rs b/module/move/unitore/src/action/config.rs similarity index 90% rename from module/move/unitore/src/executor/actions/config.rs rename to module/move/unitore/src/action/config.rs index 8d83c01ab8..952d997fbc 100644 --- a/module/move/unitore/src/executor/actions/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -3,16 +3,17 @@ use crate::*; use error_tools::{ err, for_app::Context, BasicError, Result }; use executor::FeedManager; -use storage:: +use storage::FeedStorage; +use entity:: { - FeedStorage, feed::{ FeedStore, Feed }, config::{ ConfigStore, Config }, }; +use action::Report; use gluesql::{ prelude::Payload, sled_storage::SledStorage }; /// Add configuration file with subscriptions to storage. -pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl executor::Report > +pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -49,17 +50,17 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args let feeds = feed_config::read( config.path() )? .into_iter() - .map( | feed | Feed::new( feed.link, feed.update_period ) ) + .map( | feed | Feed::new( feed.link, feed.update_period, config.path() ) ) .collect::< Vec< _ > >() ; - let new_feeds = manager.storage.save_feeds( feeds ).await?; + let new_feeds = manager.storage.feeds_save( feeds ).await?; Ok( ConfigReport{ payload : config_report, new_feeds : Some( new_feeds ) } ) } /// Remove configuration file from storage. -pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl executor::Report > +pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -80,7 +81,7 @@ pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::A } /// List all files with subscriptions that are currently in storage. -pub async fn config_list( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl executor::Report > +pub async fn config_list( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > { let mut manager = FeedManager::new( storage ); Ok( ConfigReport::new( manager.storage.config_list().await? ) ) @@ -139,7 +140,7 @@ impl std::fmt::Display for ConfigReport rows.push( vec![ EMPTY_CELL.to_owned(), String::from( row[ 0 ].clone() ) ] ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { write!( f, "{}", table )?; @@ -152,4 +153,4 @@ impl std::fmt::Display for ConfigReport } } -impl executor::Report for ConfigReport {} +impl Report for ConfigReport {} diff --git a/module/move/unitore/src/executor/actions/feed.rs b/module/move/unitore/src/action/feed.rs similarity index 83% rename from module/move/unitore/src/executor/actions/feed.rs rename to module/move/unitore/src/action/feed.rs index 92863c0317..6b253ef0a2 100644 --- a/module/move/unitore/src/executor/actions/feed.rs +++ b/module/move/unitore/src/action/feed.rs @@ -1,15 +1,13 @@ //! Endpoints and report for feed commands. use crate::*; -use executor:: -{ - FeedManager, - actions::{ Report, frame::SelectedEntries }, -}; -use storage::{ FeedStorage, feed::FeedStore }; +use executor::FeedManager; +use action::{ Report, frame::SelectedEntries }; +use storage::FeedStorage; +use entity::feed::FeedStore; use error_tools::Result; -/// List all feeds. +/// List all feeds from storage. pub async fn feeds_list ( storage : FeedStorage< gluesql::sled_storage::SledStorage >, @@ -17,7 +15,7 @@ pub async fn feeds_list ) -> Result< impl Report > { let mut manager = FeedManager::new( storage ); - manager.storage.get_all_feeds().await + manager.storage.feeds_list().await } const EMPTY_CELL : &'static str = ""; @@ -52,7 +50,7 @@ impl std::fmt::Display for FeedsReport let mut headers = vec![ EMPTY_CELL.to_owned() ]; headers.extend( self.0.selected_columns.iter().map( | str | str.to_owned() ) ); - let table = table_display::table_with_headers( headers, rows ); + let table = tool::table_display::table_with_headers( headers, rows ); if let Some( table ) = table { write!( f, "{}", table )?; diff --git a/module/move/unitore/src/executor/actions/frame.rs b/module/move/unitore/src/action/frame.rs similarity index 86% rename from module/move/unitore/src/executor/actions/frame.rs rename to module/move/unitore/src/action/frame.rs index f2ddef9471..3356bb1c53 100644 --- a/module/move/unitore/src/executor/actions/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -2,16 +2,17 @@ use crate::*; use executor::FeedManager; -use storage:: +use storage::FeedStorage; +use entity:: { - FeedStorage, feed::FeedStore, config::ConfigStore, - frame::{ FrameStore, RowValue } + frame::{ FrameStore, CellValue } }; use gluesql::prelude::{ Payload, Value, SledStorage }; use feed_config; use error_tools::{ err, Result }; +use action::Report; // qqq : review the whole project and make sure all names are consitant: actions, commands, its tests @@ -20,7 +21,7 @@ pub async fn frames_list ( storage : FeedStorage< SledStorage >, _args : &wca::Args, -) -> Result< impl executor::Report > +) -> Result< impl Report > { let mut manager = FeedManager::new( storage ); manager.storage.frames_list().await @@ -31,7 +32,7 @@ pub async fn frames_download ( storage : FeedStorage< SledStorage >, _args : &wca::Args, -) -> Result< impl executor::Report > +) -> Result< impl Report > { let mut manager = FeedManager::new( storage ); let payload = manager.storage.config_list().await?; @@ -72,10 +73,10 @@ pub async fn frames_download let client = retriever::FeedClient; for subscription in subscriptions { - let feed = retriever::FeedFetch::fetch(&client, subscription.link.clone()).await?; + let feed = client.fetch( subscription.link.clone() ).await?; feeds.push( ( feed, subscription.update_period.clone(), subscription.link ) ); } - manager.storage.process_feeds( feeds ).await + manager.storage.feeds_process( feeds ).await } @@ -122,7 +123,7 @@ impl std::fmt::Display for FramesReport fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result { let initial = vec![ vec![ format!( "Feed title: {}", self.feed_link ) ] ]; - let table = table_display::table_with_headers( initial[ 0 ].clone(), Vec::new() ); + let table = tool::table_display::table_with_headers( initial[ 0 ].clone(), Vec::new() ); if let Some( table ) = table { write!( f, "{}", table )?; @@ -140,7 +141,7 @@ impl std::fmt::Display for FramesReport rows.push( vec![ EMPTY_CELL.to_owned(), format!( "Selected frames:" ) ] ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { write!( f, "{}", table )?; @@ -148,8 +149,14 @@ impl std::fmt::Display for FramesReport for frame in &self.selected_frames.selected_rows { + let first_row = vec! + [ + INDENT_CELL.to_owned(), + self.selected_frames.selected_columns[ 0 ].clone(), + textwrap::fill( &String::from( frame[ 0 ].clone() ), 120 ), + ]; let mut rows = Vec::new(); - for i in 0..self.selected_frames.selected_columns.len() + for i in 1..self.selected_frames.selected_columns.len() { let inner_row = vec! [ @@ -160,7 +167,7 @@ impl std::fmt::Display for FramesReport rows.push( inner_row ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::table_with_headers( first_row, rows ); if let Some( table ) = table { writeln!( f, "{}", table )?; @@ -171,9 +178,9 @@ impl std::fmt::Display for FramesReport } } -impl executor::Report for FramesReport {} +impl Report for FramesReport {} -/// Items get from select query from storage. +/// Items retrieved by select queries from storage. #[ derive( Debug ) ] pub struct SelectedEntries { @@ -202,7 +209,7 @@ impl std::fmt::Display for SelectedEntries { for i in 0..self.selected_columns.len() { - write!( f, "{} : {}, ", self.selected_columns[ i ], RowValue( &row[ i ] ) )?; + write!( f, "{} : {}, ", self.selected_columns[ i ], CellValue( &row[ i ] ) )?; } writeln!( f, "" )?; } @@ -238,7 +245,7 @@ impl std::fmt::Display for UpdateReport } } -impl executor::Report for UpdateReport {} +impl Report for UpdateReport {} /// Report for listing frames. #[ derive( Debug ) ] @@ -270,4 +277,4 @@ impl std::fmt::Display for ListReport } } -impl executor::Report for ListReport {} +impl Report for ListReport {} diff --git a/module/move/unitore/src/executor/actions/mod.rs b/module/move/unitore/src/action/mod.rs similarity index 100% rename from module/move/unitore/src/executor/actions/mod.rs rename to module/move/unitore/src/action/mod.rs diff --git a/module/move/unitore/src/executor/actions/query.rs b/module/move/unitore/src/action/query.rs similarity index 95% rename from module/move/unitore/src/executor/actions/query.rs rename to module/move/unitore/src/action/query.rs index 71ee7ba52c..fb004825cb 100644 --- a/module/move/unitore/src/executor/actions/query.rs +++ b/module/move/unitore/src/action/query.rs @@ -1,10 +1,12 @@ //! Query command endpoint and report. // qqq : don't use both +// aaa : fixed use crate::*; use gluesql::core::executor::Payload; use storage::{ FeedStorage, Store }; -use executor::{ FeedManager, actions::Report }; +use executor::FeedManager; +use action::Report; use error_tools::{ err, BasicError, Result }; /// Execute query specified in query string. @@ -67,7 +69,7 @@ impl std::fmt::Display for QueryReport ]; rows.push( new_row ); } - let table = table_display::plain_table( rows ); + let table = tool::table_display::plain_table( rows ); if let Some( table ) = table { writeln!( f, "{}", table )?; diff --git a/module/move/unitore/src/executor/actions/table.rs b/module/move/unitore/src/action/table.rs similarity index 97% rename from module/move/unitore/src/executor/actions/table.rs rename to module/move/unitore/src/action/table.rs index 232ee1c499..088e4adc33 100644 --- a/module/move/unitore/src/executor/actions/table.rs +++ b/module/move/unitore/src/action/table.rs @@ -3,8 +3,10 @@ use crate::*; use gluesql::prelude::Payload; use std::collections::HashMap; -use executor::{ FeedManager, Report }; -use storage::{ FeedStorage, table::TableStore }; +use action::Report; +use executor::FeedManager; +use storage::FeedStorage; +use entity::table::TableStore; use error_tools::{ err, BasicError, Result }; /// Get labels of column for specified table. @@ -215,7 +217,7 @@ pub async fn table_list Ok( ColumnsReport::new( table_name, table_description, columns_desc ) ) } -/// Get names of tables in storage. +/// Get information about tables in storage. pub async fn tables_list ( storage : FeedStorage< gluesql::sled_storage::SledStorage >, @@ -274,7 +276,7 @@ impl std::fmt::Display for ColumnsReport ] ); } - let table = table_display::table_with_headers + let table = tool::table_display::table_with_headers ( vec! [ @@ -369,7 +371,7 @@ impl std::fmt::Display for TablesReport ); } - let table = table_display::table_with_headers + let table = tool::table_display::table_with_headers ( vec! [ diff --git a/module/move/unitore/src/command/mod.rs b/module/move/unitore/src/command/mod.rs new file mode 100644 index 0000000000..790b2c4ac0 --- /dev/null +++ b/module/move/unitore/src/command/mod.rs @@ -0,0 +1 @@ +pub mod table; \ No newline at end of file diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs new file mode 100644 index 0000000000..c5d8f5ffab --- /dev/null +++ b/module/move/unitore/src/command/table.rs @@ -0,0 +1,56 @@ +//! + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use storage::FeedStorage; +use action::{ Report, table::table_list }; +use error_tools::Result; + +pub struct TableCommand( Command ); + +impl TableCommand +{ + pub fn new() -> Result< Self > + { + + let rt = tokio::runtime::Runtime::new()?; + + Ok( Self + ( + Command::former() + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + table_list( feed_storage, &o.args ).await + } ); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() + ) ) + } + pub fn command() + { + + } +} \ No newline at end of file diff --git a/module/move/unitore/src/storage/config.rs b/module/move/unitore/src/entity/config.rs similarity index 100% rename from module/move/unitore/src/storage/config.rs rename to module/move/unitore/src/entity/config.rs diff --git a/module/move/unitore/src/storage/feed.rs b/module/move/unitore/src/entity/feed.rs similarity index 74% rename from module/move/unitore/src/storage/feed.rs rename to module/move/unitore/src/entity/feed.rs index 1506e1268e..2d05d317ff 100644 --- a/module/move/unitore/src/storage/feed.rs +++ b/module/move/unitore/src/entity/feed.rs @@ -15,12 +15,13 @@ use gluesql:: sled_storage::SledStorage, }; -use executor::actions:: +use action:: { feed::FeedsReport, frame::{ UpdateReport, SelectedEntries, FramesReport }, }; -use storage::{ FeedStorage, frame::FrameStore }; +use storage::FeedStorage; +use entity::frame::FrameStore; use wca::wtools::Itertools; /// Feed item. @@ -41,12 +42,14 @@ pub struct Feed pub published : Option< DateTime< Utc > >, /// How often the feed frames must be fetched. pub update_period : Duration, + /// Path to config file, from which this feed was saved. + pub config_file : String, } impl Feed { /// Create new feed item from source url and update period. - pub fn new( link : url::Url, update_period : Duration ) -> Self + pub fn new( link : url::Url, update_period : Duration, config: String ) -> Self { Self { @@ -57,6 +60,7 @@ impl Feed description : None, published : None, update_period, + config_file : config, } } } @@ -66,27 +70,40 @@ impl Feed #[ async_trait::async_trait( ?Send ) ] pub trait FeedStore { + /// Save new feeds to storage. + /// New feeds from config files that doesn't exist in storage will be inserted into `feed` table. + async fn feeds_save( &mut self, feeds : Vec< Feed > ) -> Result< Payload >; - /// Insert items from list into feed table. - async fn update_feed( &mut self, feed : Vec< Feed > ) -> Result< () >; + /// Update existing feeds in storage with new information. + /// Feed is updated one time during first fetch. + async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () >; - /// Process fetched feed, new items will be saved, modified items will be updated. - async fn process_feeds( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) > ) -> Result< UpdateReport >; + /// Process new fetched feeds and frames. + /// Frames from recent fetch will be sorted into three categories: + /// - new items that will be inserted into `frame` table; + /// - modified items that will be updated; + /// - unchanged frames saved from previous fetches will be ignored. + async fn feeds_process( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) > ) -> Result< UpdateReport >; - /// Get all feeds from storage. - async fn get_all_feeds( &mut self ) -> Result< FeedsReport >; - - /// Add feeds entries. - async fn save_feeds( &mut self, feeds : Vec< Feed > ) -> Result< Payload >; + /// Get existing feeds from storage. + /// Retrieves all feeds from `feed` table in storage. + async fn feeds_list( &mut self ) -> Result< FeedsReport >; } // qqq : poor description and probably naming. improve, please +// aaa : updated description #[ async_trait::async_trait( ?Send ) ] impl FeedStore for FeedStorage< SledStorage > { - async fn get_all_feeds( &mut self ) -> Result< FeedsReport > + async fn feeds_list( &mut self ) -> Result< FeedsReport > { - let res = table( "feed" ).select().project( "title, link, update_period" ).execute( &mut *self.storage.lock().await ).await?; + let res = table( "feed" ) + .select() + .project( "title, link, update_period, config_file" ) + .execute( &mut *self.storage.lock().await ) + .await? + ; + let mut report = FeedsReport::new(); match res { @@ -104,17 +121,23 @@ impl FeedStore for FeedStorage< SledStorage > Ok( report ) } - async fn update_feed( &mut self, feed : Vec< Feed > ) -> Result< () > + async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () > { for feed in feed { let _update = table( "feed" ) .update() .set( "title", feed.title.map( text ).unwrap_or( null() ) ) - .set( "updated", feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ) ) + .set( + "updated", + feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) .set( "authors", feed.authors.map( text ).unwrap_or( null() ) ) .set( "description", feed.description.map( text ).unwrap_or( null() ) ) - .set( "published", feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ) ) + .set( + "published", + feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) .filter( col( "link" ).eq( feed.link.to_string() ) ) .execute( &mut *self.storage.lock().await ) .await @@ -125,7 +148,7 @@ impl FeedStore for FeedStorage< SledStorage > Ok( () ) } - async fn process_feeds + async fn feeds_process ( &mut self, feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) >, @@ -211,7 +234,7 @@ impl FeedStore for FeedStorage< SledStorage > Ok( UpdateReport( reports ) ) } - async fn save_feeds( &mut self, feed : Vec< Feed > ) -> Result< Payload > + async fn feeds_save( &mut self, feed : Vec< Feed > ) -> Result< Payload > { let feeds_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | feed | feed.into() ).collect_vec(); @@ -225,7 +248,8 @@ impl FeedStore for FeedStorage< SledStorage > authors, description, published, - update_period", + update_period, + config_file", ) .values( feeds_rows ) .execute( &mut *self.storage.lock().await ) @@ -237,30 +261,8 @@ impl FeedStore for FeedStorage< SledStorage > } } -impl From< ( feed_rs::model::Feed, Duration, url::Url ) > for Feed -{ - fn from( val : ( feed_rs::model::Feed, Duration, url::Url ) ) -> Self - { - let duration = val.1; - let link = val.2; - let value = val.0; - - let authors = value.authors.into_iter().map( | p | p.name ).collect::< Vec< _ > >(); - let description = value.description.map( | desc | desc.content ); - - Self - { - link, - title : value.title.map( | title | title.content ), - updated : value.updated, - published : value.published, - description, - authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), - update_period : duration, - } - } -} - +/// Get convenient format of frame item for using with GlueSQL expression builder. +/// Converts from Feed struct into vec of GlueSQL expression nodes. impl From< Feed > for Vec< ExprNode< 'static > > { fn from( value : Feed ) -> Self @@ -274,6 +276,7 @@ impl From< Feed > for Vec< ExprNode< 'static > > value.description.map( text ).unwrap_or( null() ), value.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), text( value.update_period.as_secs().to_string() ), + text( value.config_file ), ] } } diff --git a/module/move/unitore/src/storage/frame.rs b/module/move/unitore/src/entity/frame.rs similarity index 89% rename from module/move/unitore/src/storage/frame.rs rename to module/move/unitore/src/entity/frame.rs index 416790b977..3d01cc2064 100644 --- a/module/move/unitore/src/storage/frame.rs +++ b/module/move/unitore/src/entity/frame.rs @@ -15,7 +15,7 @@ use gluesql:: sled_storage::SledStorage, }; -use executor::actions::frame::{ FramesReport, ListReport, SelectedEntries }; +use action::frame::{ FramesReport, ListReport, SelectedEntries }; use storage::FeedStorage; use wca::wtools::Itertools; @@ -27,7 +27,7 @@ pub struct Frame pub id : String, /// Frame title. pub title : Option< String >, - updated : Option< DateTime< Utc > >, + stored_time : Option< DateTime< Utc > >, authors : Option< String >, content : Option< String >, links : Option< String >, @@ -42,6 +42,7 @@ pub struct Frame } // qqq : not obvious +/// Convert from feed_rs feed entry and feed link to Frame struct for convenient use and storage. impl From< ( feed_rs::model::Entry, String ) > for Frame { fn from( ( entry, feed_link ) : ( feed_rs::model::Entry, String ) ) -> Self @@ -81,7 +82,7 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame { id : entry.id, title : entry.title.map( | title | title.content ).clone(), - updated : entry.updated, + stored_time : entry.updated, authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), // qqq : why join? content, @@ -105,16 +106,19 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame #[ async_trait::async_trait( ?Send ) ] pub trait FrameStore { - /// Insert items from list into feed table. + /// Save new frames to storage. + /// New frames will be inserted into `frame` table. async fn frames_save( &mut self, feed : Vec< Frame > ) -> Result< Payload >; - /// Update items from list in feed table. + /// Update existing frames in storage with new changes. + /// If frames in storage were modified in feed source, they will be changed to match new version. async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () >; /// Get all feed frames from storage. async fn frames_list( &mut self ) -> Result< ListReport >; } // qqq : what is update? what update? don't use word update without noun and explanation what deos it mean +// aaa : fixed comments #[ async_trait::async_trait( ?Send ) ] impl FrameStore for FeedStorage< SledStorage > @@ -138,7 +142,6 @@ impl FrameStore for FeedStorage< SledStorage > SelectedEntries::new() }; - let mut feeds_map = HashMap::new(); for row in all_frames.selected_rows @@ -210,6 +213,10 @@ impl FrameStore for FeedStorage< SledStorage > } // qqq : what is it for and why? +// aaa : added explanation + +/// Get convenient frame format for using with GlueSQL expression builder. +/// Converts from Frame struct into vec of GlueSQL expression nodes. impl From< Frame > for Vec< ExprNode< 'static > > { fn from( entry : Frame ) -> Self @@ -219,7 +226,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > .unwrap_or( null() ) ; - let updated = entry.updated + let stored_time = entry.stored_time .map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ) .unwrap_or( null() ) ; @@ -267,7 +274,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > [ text( entry.id ), title, - updated, + stored_time, authors, content, links, @@ -284,11 +291,12 @@ impl From< Frame > for Vec< ExprNode< 'static > > } // qqq : RowValue or CellValue? +// aaa : fixed name /// GlueSQL Value wrapper for display. #[ derive( Debug ) ] -pub struct RowValue< 'a >( pub &'a gluesql::prelude::Value ); +pub struct CellValue< 'a >( pub &'a gluesql::prelude::Value ); -impl std::fmt::Display for RowValue< '_ > +impl std::fmt::Display for CellValue< '_ > { fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result { @@ -318,9 +326,9 @@ impl std::fmt::Display for RowValue< '_ > } } -impl From< RowValue< '_ > > for String +impl From< CellValue< '_ > > for String { - fn from( value : RowValue< '_ > ) -> Self + fn from( value : CellValue< '_ > ) -> Self { use gluesql::core::data::Value::*; match &value.0 diff --git a/module/move/unitore/src/entity/mod.rs b/module/move/unitore/src/entity/mod.rs new file mode 100644 index 0000000000..94a95a552b --- /dev/null +++ b/module/move/unitore/src/entity/mod.rs @@ -0,0 +1,7 @@ +//! Entities of application. + +pub mod config; +pub mod frame; +pub mod table; +pub mod feed; + diff --git a/module/move/unitore/src/storage/table.rs b/module/move/unitore/src/entity/table.rs similarity index 96% rename from module/move/unitore/src/storage/table.rs rename to module/move/unitore/src/entity/table.rs index 60428ce0ce..b35ccdcac5 100644 --- a/module/move/unitore/src/storage/table.rs +++ b/module/move/unitore/src/entity/table.rs @@ -8,7 +8,7 @@ use gluesql:: prelude::Payload, }; -use executor::actions::table::TablesReport; +use action::table::TablesReport; use storage::FeedStorage; /// Functions for tables informantion. @@ -43,5 +43,4 @@ impl TableStore for FeedStorage< SledStorage > Ok( payloads ) } - } diff --git a/module/move/unitore/src/executor/mod.rs b/module/move/unitore/src/executor.rs similarity index 87% rename from module/move/unitore/src/executor/mod.rs rename to module/move/unitore/src/executor.rs index 02f34d72dc..687eef4771 100644 --- a/module/move/unitore/src/executor/mod.rs +++ b/module/move/unitore/src/executor.rs @@ -3,15 +3,14 @@ use crate::*; use feed_config::SubscriptionConfig; use gluesql::sled_storage::{ sled::Config, SledStorage }; -use retriever::{ FeedClient, FeedFetch }; -use storage::{ Store, FeedStorage, feed::FeedStore, config::ConfigStore, table::TableStore, frame::FrameStore }; +use storage::{ Store, FeedStorage }; +use entity::{ feed::FeedStore, config::ConfigStore, table::TableStore, frame::FrameStore }; use wca::{ Args, Type, VerifiedCommand }; -use executor::actions::Report; use error_tools::Result; -pub mod actions; -use actions:: +use action:: { + Report, frame::{ frames_list, frames_download }, feed::feeds_list, config::{ config_add, config_delete, config_list }, @@ -23,7 +22,7 @@ fn action< 'a, F, Fut, R >( async_endpoint : F, args : &'a Args ) -> Result< R > where F : FnOnce( FeedStorage< SledStorage >, &'a Args ) -> Fut, Fut : std::future::Future< Output = Result< R > >, - R : actions::Report, + R : action::Report, { let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) .unwrap_or( String::from( "./_data" ) ) @@ -36,7 +35,7 @@ where rt.block_on( async move { - let feed_storage = FeedStorage::init_storage( config ).await?; + let feed_storage = FeedStorage::init_storage( &config ).await?; async_endpoint( feed_storage, args ).await } ) } @@ -44,6 +43,7 @@ where /// Run feed updates. pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > { + //let ca = wca::CommandsAggregator::new(); let ca = wca::CommandsAggregator::former() .command( "frames.download" ) .hint( "Download frames from feed sources provided in config files." ) @@ -217,17 +217,15 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > } /// Manages feed subsriptions and updates. -pub struct FeedManager< C, S : FeedStore + ConfigStore + FrameStore + Store + Send > +pub struct FeedManager< S : FeedStore + ConfigStore + FrameStore + Store + Send > { /// Subscription configuration with link and update period. pub config : Vec< SubscriptionConfig >, /// Storage for saving feed. pub storage : S, - /// Client for fetching feed from links in FeedConfig. - pub client : C, } -impl< C, S : FeedStore + ConfigStore + FrameStore + Store + Send > std::fmt::Debug for FeedManager< C, S > +impl< S : FeedStore + ConfigStore + FrameStore + Store + Send > std::fmt::Debug for FeedManager< S > { fn fmt( &self, f: &mut std::fmt::Formatter<'_> ) -> std::fmt::Result { @@ -235,21 +233,20 @@ impl< C, S : FeedStore + ConfigStore + FrameStore + Store + Send > std::fmt::Deb } } -impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< FeedClient, S > +impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< S > { /// Create new instance of FeedManager. - pub fn new( storage : S ) -> FeedManager< FeedClient, S > + pub fn new( storage : S ) -> FeedManager< S > { Self { storage, config : Vec::new(), - client : FeedClient, } } } -impl< C : FeedFetch, S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< C, S > +impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< S > { /// Set configurations for subscriptions. pub fn set_config( &mut self, configs : Vec< SubscriptionConfig > ) @@ -257,14 +254,8 @@ impl< C : FeedFetch, S : FeedStore + ConfigStore + FrameStore + TableStore + Sto self.config = configs; } - /// Set client for fetching feed. - pub fn set_client( &mut self, client : C ) - { - self.client = client; - } - /// Execute custom query, print result. - pub async fn execute_custom_query( &mut self, query : String ) -> Result< impl actions::Report > + pub async fn execute_custom_query( &mut self, query : String ) -> Result< impl Report > { self.storage.execute_query( query ).await } diff --git a/module/move/unitore/src/lib.rs b/module/move/unitore/src/lib.rs index d871675d0c..cc00707681 100644 --- a/module/move/unitore/src/lib.rs +++ b/module/move/unitore/src/lib.rs @@ -3,6 +3,9 @@ pub mod retriever; pub mod feed_config; pub mod executor; pub mod storage; -pub mod table_display; +pub mod tool; +pub mod command; +pub mod action; +pub mod entity; // qqq : src/Readmу.md with file structure please diff --git a/module/move/unitore/src/retriever.rs b/module/move/unitore/src/retriever.rs index 32a3f4e47a..79eea6b407 100644 --- a/module/move/unitore/src/retriever.rs +++ b/module/move/unitore/src/retriever.rs @@ -12,22 +12,15 @@ use feed_rs::parser as feed_parser; use error_tools::{ Result, for_app::Context }; // qqq : purpose of trait if any? -/// Fetch feed from provided source link. -#[ async_trait::async_trait ] -pub trait FeedFetch -{ - /// Get feed from source specified by its link. - async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed >; -} +// aaa : removed unnecessary trait /// Feed client for fetching feed. #[ derive( Debug ) ] pub struct FeedClient; -#[ async_trait::async_trait ] -impl FeedFetch for FeedClient +impl FeedClient { - async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed > + pub async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed > { let https = HttpsConnector::new(); let client = Client::builder( TokioExecutor::new() ).build::< _, Empty< Bytes > >( https ); diff --git a/module/move/unitore/src/storage/mod.rs b/module/move/unitore/src/storage.rs similarity index 92% rename from module/move/unitore/src/storage/mod.rs rename to module/move/unitore/src/storage.rs index 1eedc29afd..86295eb4f0 100644 --- a/module/move/unitore/src/storage/mod.rs +++ b/module/move/unitore/src/storage.rs @@ -14,13 +14,7 @@ use gluesql:: prelude::Glue, sled_storage::{ sled::Config, SledStorage }, }; - -use executor::actions::query::QueryReport; - -pub mod config; -pub mod frame; -pub mod table; -pub mod feed; +use action::query::QueryReport; /// Storage for feed frames. #[ derive( Clone ) ] @@ -28,7 +22,7 @@ pub struct FeedStorage< S : GStore + GStoreMut + Send > { /// GlueSQL storage. pub storage : Arc< Mutex< Glue< S > > >, - frame_fields : Vec< [ &'static str; 3 ] >, + pub frame_fields : Vec< [ &'static str; 3 ] >, } impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > @@ -42,7 +36,7 @@ impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > impl FeedStorage< SledStorage > { /// Initialize new storage from configuration, create feed table. - pub async fn init_storage( config : Config ) -> Result< Self > + pub async fn init_storage( config : &Config ) -> Result< Self > { let storage = SledStorage::try_from( config.clone() ) .context( format!( "Failed to initialize storage with config {:?}", config ) )? @@ -68,6 +62,7 @@ impl FeedStorage< SledStorage > .add_column( "description TEXT" ) .add_column( "published TIMESTAMP" ) .add_column( "update_period TEXT" ) + .add_column( "config_file TEXT FOREIGN KEY REFERENCES config(path)" ) .build()? ; @@ -77,7 +72,7 @@ impl FeedStorage< SledStorage > [ [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], [ "title", "TEXT", "Title of the frame" ], - [ "updated", "TIMESTAMP", "Time at which this item was fetched from source." ], + [ "stored_time", "TIMESTAMP", "Time at which this item was fetched from source." ], [ "authors", "TEXT", "List of authors of the frame, optional." ], [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], [ "links", "TEXT", "List of links associated with this item of related Web page and attachments." ], diff --git a/module/move/unitore/src/tool/mod.rs b/module/move/unitore/src/tool/mod.rs new file mode 100644 index 0000000000..200a9b43be --- /dev/null +++ b/module/move/unitore/src/tool/mod.rs @@ -0,0 +1 @@ +pub mod table_display; \ No newline at end of file diff --git a/module/move/unitore/src/table_display.rs b/module/move/unitore/src/tool/table_display.rs similarity index 62% rename from module/move/unitore/src/table_display.rs rename to module/move/unitore/src/tool/table_display.rs index 9f334cc8ee..4b5f35475a 100644 --- a/module/move/unitore/src/table_display.rs +++ b/module/move/unitore/src/tool/table_display.rs @@ -1,12 +1,16 @@ -//! Helper for command report representation. +//! Wrapper for command report representation. +//! Separates usage of cli-table library behind facade for convenient changes in future. use cli_table:: { - format::{ Border, Separator }, Cell, Style, Table, TableDisplay + format::{ Border, HorizontalLine, Separator }, Cell, Style, Table, TableDisplay }; // qqq : purpose well defined should be always be in documentation -/// Wrapper struct for cli-table table with iplementation of Display. +// aaa : added explanation + +/// Wrapper struct for cli-table table with implementation of Display. +/// Separates usage of cli-table library behind facade for convenient changes in future. pub struct ReportTable( TableDisplay ); impl std::fmt::Display for ReportTable @@ -63,5 +67,22 @@ pub fn table_with_headers( headers : Vec< String >, rows : Vec< Vec< String > > .separator( Separator::builder().build() ) ; + table_struct.display().map( | table | ReportTable( table ) ).ok() +} + +/// Transform 2-dimensional vec of String data into displayable table with plain rows and bottom border. +pub fn plain_with_border( rows : Vec< Vec< String > > ) -> Option< ReportTable > +{ + let rows = rows + .into_iter() + .map( | row | row.into_iter().map( | cell_val | cell_val.cell() ).collect::< Vec< _ > >() ) + .collect::< Vec< _ > >() + ; + + let table_struct = rows.table() + .border( Border::builder().bottom(HorizontalLine::default()).build() ) + .separator( Separator::builder().build() ) + ; + table_struct.display().map( | table | ReportTable( table ) ).ok() } \ No newline at end of file diff --git a/module/move/unitore/tests/config_add.rs b/module/move/unitore/tests/config_add.rs index 502ec144f0..c938eb5d99 100644 --- a/module/move/unitore/tests/config_add.rs +++ b/module/move/unitore/tests/config_add.rs @@ -3,8 +3,10 @@ use std::path::PathBuf; use gluesql::sled_storage::sled::Config; use unitore:: { - executor::{ FeedManager, actions }, - storage::{ FeedStorage, feed::FeedStore }, + executor::FeedManager, + storage::FeedStorage, + entity::feed::FeedStore, + action::config, }; use error_tools::Result; @@ -19,11 +21,11 @@ async fn add_config_file() -> Result< () > .temporary( true ) ; - let feed_storage = FeedStorage::init_storage( config ).await?; - actions::config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; + let feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; let mut manager = FeedManager::new( feed_storage ); - let res = manager.storage.get_all_feeds().await?; + let res = manager.storage.feeds_list().await?; let feeds_links = res.0.selected_rows .iter() diff --git a/module/move/unitore/tests/frame.rs b/module/move/unitore/tests/frame.rs index 248f40330b..a5d7b510e1 100644 --- a/module/move/unitore/tests/frame.rs +++ b/module/move/unitore/tests/frame.rs @@ -6,7 +6,7 @@ async fn frame() -> Result< () > { let feed = feed_parser::parse( include_str!( "./fixtures/plain_feed.xml" ).as_bytes() )?; - let frame = unitore::storage::frame::Frame::from( ( feed.entries[ 0 ].clone(), String::new() ) ); + let frame = unitore::entity::frame::Frame::from( ( feed.entries[ 0 ].clone(), String::new() ) ); assert!( frame.id == feed.entries[ 0 ].id ); diff --git a/module/move/unitore/tests/save_feed.rs b/module/move/unitore/tests/save_feed.rs index 75dc54b5f7..2ac5cbcff3 100644 --- a/module/move/unitore/tests/save_feed.rs +++ b/module/move/unitore/tests/save_feed.rs @@ -1,28 +1,12 @@ -use async_trait::async_trait; use feed_rs::parser as feed_parser; use unitore:: { feed_config::SubscriptionConfig, - retriever::FeedFetch, - storage::{ FeedStorage, frame::FrameStore, feed::FeedStore }, + storage::FeedStorage, + entity::{ frame::FrameStore, feed::FeedStore }, }; use error_tools::Result; -/// Feed client for testing. -#[derive(Debug)] -pub struct TestClient; - -#[ async_trait ] -impl FeedFetch for TestClient -{ - async fn fetch( &self, _ : url::Url ) -> Result< feed_rs::model::Feed > - { - let feed = feed_parser::parse( include_str!( "./fixtures/plain_feed.xml" ).as_bytes() )?; - - Ok( feed ) - } -} - #[ tokio::test ] async fn test_save_feed_plain() -> Result< () > { @@ -47,7 +31,7 @@ async fn test_save_feed_plain() -> Result< () > .temporary( true ) ; - let mut feed_storage = FeedStorage::init_storage( config ).await?; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; let feed_config = SubscriptionConfig { @@ -56,11 +40,10 @@ async fn test_save_feed_plain() -> Result< () > }; let mut feeds = Vec::new(); - let client = TestClient; - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; + let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); - feed_storage.process_feeds( feeds ).await?; + feed_storage.feeds_process( feeds ).await?; let entries = feed_storage.frames_list().await?; diff --git a/module/move/unitore/tests/update_newer_feed.rs b/module/move/unitore/tests/update_newer_feed.rs index c95d50619b..4702acfcf0 100644 --- a/module/move/unitore/tests/update_newer_feed.rs +++ b/module/move/unitore/tests/update_newer_feed.rs @@ -1,4 +1,3 @@ -use async_trait::async_trait; use feed_rs::parser as feed_parser; use gluesql:: { @@ -12,26 +11,12 @@ use gluesql:: use unitore:: { feed_config::SubscriptionConfig, - retriever::FeedFetch, - storage::{ feed::FeedStore, frame::FrameStore, FeedStorage }, + storage::FeedStorage, + entity::{ feed::FeedStore, frame::FrameStore }, }; use wca::wtools::Itertools; use error_tools::Result; -/// Feed client for testing. -#[derive(Debug)] -pub struct TestClient ( String ); - -#[ async_trait ] -impl FeedFetch for TestClient -{ - async fn fetch( &self, _ : url::Url ) -> Result< feed_rs::model::Feed > - { - let feed = feed_parser::parse( std::fs::read_to_string( &self.0 )?.as_bytes() )?; - Ok( feed ) - } -} - #[ tokio::test ] async fn test_update() -> Result< () > { @@ -40,7 +25,7 @@ async fn test_update() -> Result< () > .temporary( true ) ; - let mut feed_storage = FeedStorage::init_storage( config ).await?; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; let feed_config = SubscriptionConfig { @@ -49,18 +34,15 @@ async fn test_update() -> Result< () > }; // initial fetch - let client = TestClient( "./tests/fixtures/plain_feed.xml".to_owned() ); - - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; + let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; - feed_storage.process_feeds( feeds ).await?; + feed_storage.feeds_process( feeds ).await?; // updated fetch - let client = TestClient( "./tests/fixtures/updated_one_frame.xml".to_owned() ); + let feed = feed_parser::parse( include_str!("./fixtures/updated_one_frame.xml").as_bytes() )?; - let feed = FeedFetch::fetch( &client, feed_config.link.clone()).await?; let feeds = vec![ ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ]; - feed_storage.process_feeds( feeds ).await?; + feed_storage.feeds_process( feeds ).await?; // check let payload = feed_storage.frames_list().await?; From 2d186c7d3c34184529ffababa842bdfb6d03da47 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Mon, 8 Apr 2024 10:12:54 +0300 Subject: [PATCH 03/11] fix commands --- module/move/unitore/Cargo.toml | 1 + module/move/unitore/src/Readme.md | 0 module/move/unitore/src/action/frame.rs | 2 +- module/move/unitore/src/action/table.rs | 394 ++++++++++-------- module/move/unitore/src/entity/config.rs | 54 +-- module/move/unitore/src/entity/feed.rs | 188 +-------- module/move/unitore/src/entity/frame.rs | 154 ++----- module/move/unitore/src/entity/table.rs | 30 +- module/move/unitore/src/executor.rs | 3 +- module/move/unitore/src/lib.rs | 1 + .../move/unitore/src/sled_adapter/config.rs | 53 +++ module/move/unitore/src/sled_adapter/feed.rs | 195 +++++++++ module/move/unitore/src/sled_adapter/frame.rs | 168 ++++++++ module/move/unitore/src/sled_adapter/mod.rs | 4 + module/move/unitore/src/sled_adapter/table.rs | 33 ++ module/move/unitore/tests/save_feed.rs | 2 + 16 files changed, 710 insertions(+), 572 deletions(-) create mode 100644 module/move/unitore/src/Readme.md create mode 100644 module/move/unitore/src/sled_adapter/config.rs create mode 100644 module/move/unitore/src/sled_adapter/feed.rs create mode 100644 module/move/unitore/src/sled_adapter/frame.rs create mode 100644 module/move/unitore/src/sled_adapter/mod.rs create mode 100644 module/move/unitore/src/sled_adapter/table.rs diff --git a/module/move/unitore/Cargo.toml b/module/move/unitore/Cargo.toml index 812ea26e02..8f98fba818 100644 --- a/module/move/unitore/Cargo.toml +++ b/module/move/unitore/Cargo.toml @@ -32,6 +32,7 @@ enabled = [] [dependencies] error_tools = { workspace = true, features = [ "default" ] } +proper_path_tools = { workspace = true, features = [ "default" ] } tokio = { version = "1.36.0", features = [ "rt", "rt-multi-thread", "io-std", "macros" ] } hyper = { version = "1.1.0", features = [ "client" ] } hyper-tls = "0.6.0" diff --git a/module/move/unitore/src/Readme.md b/module/move/unitore/src/Readme.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/module/move/unitore/src/action/frame.rs b/module/move/unitore/src/action/frame.rs index 3356bb1c53..88a445fed7 100644 --- a/module/move/unitore/src/action/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -133,7 +133,7 @@ impl std::fmt::Display for FramesReport [ vec![ EMPTY_CELL.to_owned(), format!( "Updated frames: {}", self.updated_frames ) ], vec![ EMPTY_CELL.to_owned(), format!( "Inserted frames: {}", self.new_frames ) ], - vec![ EMPTY_CELL.to_owned(), format!( "Number of frames in storage: {}", self.existing_frames ) ], + vec![ EMPTY_CELL.to_owned(), format!( "Number of frames in storage: {}", self.existing_frames + self.new_frames ) ], ]; if !self.selected_frames.selected_columns.is_empty() diff --git a/module/move/unitore/src/action/table.rs b/module/move/unitore/src/action/table.rs index 088e4adc33..11a34c21f4 100644 --- a/module/move/unitore/src/action/table.rs +++ b/module/move/unitore/src/action/table.rs @@ -7,7 +7,7 @@ use action::Report; use executor::FeedManager; use storage::FeedStorage; use entity::table::TableStore; -use error_tools::{ err, BasicError, Result }; +use error_tools::Result; /// Get labels of column for specified table. pub async fn table_list @@ -16,205 +16,220 @@ pub async fn table_list args : &wca::Args, ) -> Result< impl Report > { - let table_name : String = args - .get_owned::< String >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get 'Name' argument for command .table.list" ) )? - .into() - ; + let table_name = args.get_owned::< String >( 0 ); let mut manager = FeedManager::new( storage ); - let result = manager.storage.table_list( table_name.clone() ).await?; + let mut table_names = Vec::new(); + if let Some( name ) = table_name + { + table_names.push( name ); + } + else + { + let tables = manager.storage.tables_list().await?; - let mut table_description = String::new(); - let mut columns = HashMap::new(); - if let Payload::Select { labels: _label_vec, rows: rows_vec } = &result[ 0 ] + let names = tables.tables.keys().map( | k | k.clone() ).collect::< Vec< _ > >(); + table_names.extend( names.into_iter() ); + } + + let mut reports = Vec::new(); + for table_name in table_names { - for row in rows_vec + let result = manager.storage.table_list( table_name.clone() ).await?; + + let mut table_description = String::new(); + let mut columns = HashMap::new(); + if let Payload::Select { labels: _label_vec, rows: rows_vec } = &result[ 0 ] { - let table = String::from( row[ 0 ].clone() ); - columns.entry( table ) - .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) - .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) - ; + for row in rows_vec + { + let table = String::from( row[ 0 ].clone() ); + columns.entry( table ) + .and_modify( | vec : &mut Vec< String > | vec.push( String::from( row[ 1 ].clone() ) ) ) + .or_insert( vec![ String::from( row[ 1 ].clone() ) ] ) + ; + } } - } - let mut columns_desc = HashMap::new(); - match table_name.as_str() - { - "feed" => + let mut columns_desc = HashMap::new(); + match table_name.as_str() { - table_description = String::from( "Contains feed items." ); - - for label in columns.get( "feed" ).unwrap() + "feed" => { - match label.as_str() + table_description = String::from( "Contains feed items." ); + + for label in columns.get( "feed" ).unwrap() { - "link" => { columns_desc.insert - ( - label.clone(), - String::from( "Link to feed source, unique identifier for the feed" ), - ); } - "title" => { columns_desc.insert( label.clone(), String::from( "The title of the feed" ) ); } - "updated" => + match label.as_str() { - columns_desc.insert( label.clone(), String::from - ( - "The time at which the feed was last modified. If not provided in the source, or invalid, is Null." - ) ); - }, - "type" => { columns_desc.insert( label.clone(), String::from( "Type of this feed (e.g. RSS2, Atom etc)" ) ); } - "authors" => { columns_desc.insert - ( - label.clone(), - String::from( "Collection of authors defined at the feed level" ) - ); } - "description" => { columns_desc.insert( label.clone(), String::from( "Description of the feed" ) ); } - "published" => { columns_desc.insert - ( - label.clone(), - String::from( "The publication date for the content in the channel" ), - ); } - "update_period" => { columns_desc.insert( label.clone(), String::from( "How often this feed must be updated" ) ); } - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + "link" => { columns_desc.insert + ( + label.clone(), + String::from( "Link to feed source, unique identifier for the feed" ), + ); } + "title" => { columns_desc.insert( label.clone(), String::from( "The title of the feed" ) ); } + "updated" => + { + columns_desc.insert( label.clone(), String::from + ( + "The time at which the feed was last modified. If not provided in the source, or invalid, is Null." + ) ); + }, + "type" => { columns_desc.insert( label.clone(), String::from( "Type of this feed (e.g. RSS2, Atom etc)" ) ); } + "authors" => { columns_desc.insert + ( + label.clone(), + String::from( "Collection of authors defined at the feed level" ) + ); } + "description" => { columns_desc.insert( label.clone(), String::from( "Description of the feed" ) ); } + "published" => { columns_desc.insert + ( + label.clone(), + String::from( "The publication date for the content in the channel" ), + ); } + "update_period" => { columns_desc.insert( label.clone(), String::from( "How often this feed must be updated" ) ); } + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } } - } - }, - "frame" => - { - table_description = String::from( "Contains frame items." ); - for label in columns.get( "frame" ).unwrap() + }, + "frame" => { - match label.as_str() + table_description = String::from( "Contains frame items." ); + for label in columns.get( "frame" ).unwrap() { - "id" => - { - columns_desc.insert - ( - label.clone(), - String::from( "A unique identifier for this frame in the feed. " ), - ); - }, - "title" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Title of the frame" ), - ); - }, - "updated" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Time at which this item was fetched from source." ), - ); - }, - "authors" => - { - columns_desc.insert( - label.clone(), - String::from( "List of authors of the frame, optional." ) - ); - }, - "content" => - { - columns_desc.insert( - label.clone(), - String::from( "The content of the frame in html or plain text, optional." ), - ); - }, - "links" => - { - columns_desc.insert( - label.clone(), - String::from( "List of links associated with this item of related Web page and attachments." ), - ); - }, - "summary" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Short summary, abstract, or excerpt of the frame item, optional." ), - ); - }, - "categories" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Specifies a list of categories that the item belongs to." ), - ); - }, - "published" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Time at which this item was first published or updated." ), - ); - }, - "source" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ), - ); - }, - "rights" => + match label.as_str() { - columns_desc.insert - ( - label.clone(), - String::from( "Conveys information about copyrights over the feed, optional." ), - ); - }, - "media" => - { - columns_desc.insert - ( - label.clone(), - String::from( "List of media oblects, encountered in the frame, optional." ), - ); - }, - "language" => - { - columns_desc.insert - ( - label.clone(), - String::from( "The language specified on the item, optional." ), - ); - }, - "feed_link" => - { - columns_desc.insert - ( - label.clone(), - String::from( "Link of feed that contains this frame." ), - ); - }, - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + "id" => + { + columns_desc.insert + ( + label.clone(), + String::from( "A unique identifier for this frame in the feed. " ), + ); + }, + "title" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Title of the frame" ), + ); + }, + "updated" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was fetched from source." ), + ); + }, + "authors" => + { + columns_desc.insert( + label.clone(), + String::from( "List of authors of the frame, optional." ) + ); + }, + "content" => + { + columns_desc.insert( + label.clone(), + String::from( "The content of the frame in html or plain text, optional." ), + ); + }, + "links" => + { + columns_desc.insert( + label.clone(), + String::from( "List of links associated with this item of related Web page and attachments." ), + ); + }, + "summary" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Short summary, abstract, or excerpt of the frame item, optional." ), + ); + }, + "categories" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies a list of categories that the item belongs to." ), + ); + }, + "published" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Time at which this item was first published or updated." ), + ); + }, + "source" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Specifies the source feed if the frame was copied from one feed into another feed, optional." ), + ); + }, + "rights" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Conveys information about copyrights over the feed, optional." ), + ); + }, + "media" => + { + columns_desc.insert + ( + label.clone(), + String::from( "List of media oblects, encountered in the frame, optional." ), + ); + }, + "language" => + { + columns_desc.insert + ( + label.clone(), + String::from( "The language specified on the item, optional." ), + ); + }, + "feed_link" => + { + columns_desc.insert + ( + label.clone(), + String::from( "Link of feed that contains this frame." ), + ); + }, + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } } } - } - "config" => - { - table_description = String::from( "Contains paths to feed config files." ); - for label in columns.get( "config" ).unwrap() + "config" => { - match label.as_str() + table_description = String::from( "Contains paths to feed config files." ); + for label in columns.get( "config" ).unwrap() { - "path" => { columns_desc.insert( label.clone(), String::from( "Path to configuration file" ) ); } - _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + match label.as_str() + { + "path" => { columns_desc.insert( label.clone(), String::from( "Path to configuration file" ) ); } + _ => { columns_desc.insert( label.clone(), String::from( "Desciption for this column hasn't been added yet!" ) ); } + } } - } - }, - _ => {}, + }, + _ => {}, + } + + reports.push( ColumnsReport::new( table_name, table_description, columns_desc ) ); } - Ok( ColumnsReport::new( table_name, table_description, columns_desc ) ) + Ok( TablesColumnsReport( reports ) ) } /// Get information about tables in storage. @@ -230,7 +245,26 @@ pub async fn tables_list const EMPTY_CELL : &'static str = ""; -/// Information about execution of tables commands. +/// Information about execution of table columns commands. +#[ derive( Debug ) ] +pub struct TablesColumnsReport( Vec< ColumnsReport > ); + +impl std::fmt::Display for TablesColumnsReport +{ + fn fmt( &self, f : &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + for report in &self.0 + { + writeln!( f, "{}", report )?; + } + + Ok( () ) + } +} + +impl Report for TablesColumnsReport {} + +/// Information about execution of columns commands. #[ derive( Debug ) ] pub struct ColumnsReport { diff --git a/module/move/unitore/src/entity/config.rs b/module/move/unitore/src/entity/config.rs index fe140c0ef9..cf3049acbe 100644 --- a/module/move/unitore/src/entity/config.rs +++ b/module/move/unitore/src/entity/config.rs @@ -1,16 +1,7 @@ //! Functionality for storing and retrieving config files. -use crate::*; -use error_tools::{ err, Result }; -use gluesql:: -{ - core:: - { - ast_builder::{ col, table, text, Execute }, - executor::Payload, - }, - sled_storage::SledStorage, -}; +use error_tools::Result; +use gluesql::core::executor::Payload; /// Config file path. #[ derive( Debug ) ] @@ -48,47 +39,8 @@ pub trait ConfigStore // qqq : port and adapters should not be in the same file // Ideally, they should be in different crates, but you should at least put them in different folders // there should be a `sled_adapter`` folder +// aaa : moved to separate folder -#[ async_trait::async_trait( ?Send ) ] -impl ConfigStore for storage::FeedStorage< SledStorage > -{ - async fn config_add( &mut self, config : &Config ) -> Result< Payload > - { - let res = table( "config" ) - .insert() - .columns - ( - "path", - ) - .values( vec![ vec![ text( config.path() ) ] ] ) - .execute( &mut *self.storage.lock().await ) - .await; - - Ok( res? ) - } - - async fn config_delete( &mut self, config : &Config ) -> Result< Payload > - { - let res = table( "config" ) - .delete() - .filter( col( "path" ).eq( format!( "'{}'", config.path() ) ) ) - .execute( &mut *self.storage.lock().await ) - .await?; - - if res == Payload::Delete( 0 ) - { - return Err( err!( format!( "Config file with path {} not found in storage", config.path() ) ) ) - } - - Ok( res ) - } - - async fn config_list( &mut self ) -> Result< Payload > - { - let res = table( "config" ).select().execute( &mut *self.storage.lock().await ).await?; - Ok( res ) - } -} // qqq : use AbsolutePath newtype from `path_tools` // qqq : normalize all paths with `path_tools::path::normalize` diff --git a/module/move/unitore/src/entity/feed.rs b/module/move/unitore/src/entity/feed.rs index 2d05d317ff..7084e841dd 100644 --- a/module/move/unitore/src/entity/feed.rs +++ b/module/move/unitore/src/entity/feed.rs @@ -2,27 +2,19 @@ use crate::*; use std::time::Duration; -use error_tools::{ for_app::Context, Result }; -use gluesql:: +use error_tools::Result; +use gluesql::core:: { - core:: - { - ast_builder::{ null, col, table, text, Execute, timestamp, ExprNode }, - data::Value, - executor::Payload, - chrono::{ Utc, DateTime, SecondsFormat }, - }, - sled_storage::SledStorage, + ast_builder::{ null, text, timestamp, ExprNode }, + executor::Payload, + chrono::{ Utc, DateTime, SecondsFormat }, }; use action:: { feed::FeedsReport, - frame::{ UpdateReport, SelectedEntries, FramesReport }, + frame::UpdateReport, }; -use storage::FeedStorage; -use entity::frame::FrameStore; -use wca::wtools::Itertools; /// Feed item. #[ derive( Debug ) ] @@ -92,174 +84,6 @@ pub trait FeedStore // qqq : poor description and probably naming. improve, please // aaa : updated description -#[ async_trait::async_trait( ?Send ) ] -impl FeedStore for FeedStorage< SledStorage > -{ - async fn feeds_list( &mut self ) -> Result< FeedsReport > - { - let res = table( "feed" ) - .select() - .project( "title, link, update_period, config_file" ) - .execute( &mut *self.storage.lock().await ) - .await? - ; - - let mut report = FeedsReport::new(); - match res - { - Payload::Select { labels: label_vec, rows: rows_vec } => - { - report.0 = SelectedEntries - { - selected_rows : rows_vec, - selected_columns : label_vec, - } - }, - _ => {}, - } - - Ok( report ) - } - - async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () > - { - for feed in feed - { - let _update = table( "feed" ) - .update() - .set( "title", feed.title.map( text ).unwrap_or( null() ) ) - .set( - "updated", - feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), - ) - .set( "authors", feed.authors.map( text ).unwrap_or( null() ) ) - .set( "description", feed.description.map( text ).unwrap_or( null() ) ) - .set( - "published", - feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), - ) - .filter( col( "link" ).eq( feed.link.to_string() ) ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to insert feed" )? - ; - } - - Ok( () ) - } - - async fn feeds_process - ( - &mut self, - feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) >, - ) -> Result< UpdateReport > - { - let mut new_entries = Vec::new(); - let mut modified_entries = Vec::new(); - let mut reports = Vec::new(); - - for feed in &feeds - { - let mut frames_report = FramesReport::new( feed.0.title.clone().unwrap().content ); - - let existing_frames = table( "frame" ) - .select() - .filter( col( "feed_link" ).eq( text( feed.2.to_string() ) ) ) - .project( "id, published" ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to get existing frames while saving new frames" )? - ; - - if let Some( rows ) = existing_frames.select() - { - let rows = rows.collect::< Vec< _ > >(); - frames_report.existing_frames = rows.len(); - let existing_entries = rows.iter() - .map( | r | ( r.get( "id" ).map( | &val | val.clone() ), r.get( "published" ).map( | &val | val.clone() ) ) ) - .flat_map( | ( id, published ) | - id.map( | id | - ( - id, - published.map( | date | - { - match date - { - Value::Timestamp( date_time ) => Some( date_time ), - _ => None, - } - } ) - .flatten() - ) - ) - ) - .flat_map( | ( id, published ) | match id { Value::Str( id ) => Some( ( id, published ) ), _ => None } ) - .collect_vec() - ; - - let existing_ids = existing_entries.iter().map( | ( id, _ ) | id ).collect_vec(); - for entry in &feed.0.entries - { - // if extry with same id is already in db, check if it is updated - if let Some( position ) = existing_ids.iter().position( | &id | id == &entry.id ) - { - if let Some( date ) = existing_entries[ position ].1 - { - if date.and_utc() != entry.published.unwrap() - { - frames_report.updated_frames += 1; - modified_entries.push( ( entry.clone(), feed.2.to_string() ).into() ); - } - } - } - else - { - frames_report.new_frames += 1; - new_entries.push( ( entry.clone(), feed.2.to_string() ).into() ); - } - } - } - reports.push( frames_report ); - } - - if !new_entries.is_empty() - { - let _saved_report = self.frames_save( new_entries ).await?; - } - if !modified_entries.is_empty() - { - let _updated_report = self.frames_update( modified_entries ).await?; - } - - Ok( UpdateReport( reports ) ) - } - - async fn feeds_save( &mut self, feed : Vec< Feed > ) -> Result< Payload > - { - let feeds_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | feed | feed.into() ).collect_vec(); - - let insert = table( "feed" ) - .insert() - .columns - ( - "link, - title, - updated, - authors, - description, - published, - update_period, - config_file", - ) - .values( feeds_rows ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to insert feeds" )? - ; - - Ok( insert ) - } -} /// Get convenient format of frame item for using with GlueSQL expression builder. /// Converts from Feed struct into vec of GlueSQL expression nodes. diff --git a/module/move/unitore/src/entity/frame.rs b/module/move/unitore/src/entity/frame.rs index 3d01cc2064..32ba98a4ca 100644 --- a/module/move/unitore/src/entity/frame.rs +++ b/module/move/unitore/src/entity/frame.rs @@ -1,23 +1,12 @@ //! Frame storing and retrieving functionality. use crate::*; -use std::collections::HashMap; -use error_tools::{ for_app::Context, Result }; -use gluesql:: +use error_tools::Result; +use gluesql::core:: { - core:: - { - ast_builder::{ null, col, table, text, Execute, timestamp, ExprNode }, - data::Value, - executor::Payload, - chrono::{ Utc, DateTime, SecondsFormat }, - }, - sled_storage::SledStorage, + ast_builder::{ null, text, timestamp, ExprNode }, chrono::{ DateTime, SecondsFormat, Utc }, executor::Payload }; - -use action::frame::{ FramesReport, ListReport, SelectedEntries }; -use storage::FeedStorage; -use wca::wtools::Itertools; +use action::frame::ListReport; /// Frame entity. #[ derive( Debug ) ] @@ -27,21 +16,22 @@ pub struct Frame pub id : String, /// Frame title. pub title : Option< String >, - stored_time : Option< DateTime< Utc > >, - authors : Option< String >, - content : Option< String >, - links : Option< String >, - summary : Option< String >, - categories : Option< String >, - published : Option< DateTime< Utc > >, - source : Option< String >, - rights : Option< String >, - media : Option< String >, - language : Option< String >, - feed_link : String, + pub stored_time : Option< DateTime< Utc > >, + pub authors : Option< Vec< String > >, + pub content : Option< String >, + pub links : Option< Vec< String > >, + pub summary : Option< String >, + pub categories : Option< Vec< String > >, + pub published : Option< DateTime< Utc > >, + pub source : Option< String >, + pub rights : Option< String >, + pub media : Option< Vec< String > >, + pub language : Option< String >, + pub feed_link : String, } // qqq : not obvious +// aaa : added explanation /// Convert from feed_rs feed entry and feed link to Frame struct for convenient use and storage. impl From< ( feed_rs::model::Entry, String ) > for Frame { @@ -59,9 +49,10 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame .clone() ; - let mut links = entry.links + let links = entry.links .iter() .map( | link | link.href.clone() ) + .collect::< Vec< _ > >() .clone() ; @@ -83,18 +74,18 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame id : entry.id, title : entry.title.map( | title | title.content ).clone(), stored_time : entry.updated, - authors : ( !authors.is_empty() ).then( || authors.join( ", " ) ), + authors: ( !authors.is_empty() ).then( || authors ), // qqq : why join? content, - links : ( !links.len() == 0 ).then( || links.join( ", " ) ), + links: ( !links.is_empty() ).then( || links ), // qqq : why join? summary : entry.summary.map( | c | c.content ).clone(), - categories : ( !categories.is_empty() ).then( || categories.join( ", " ) ), + categories: ( !categories.is_empty() ).then( || categories ), // qqq : why join? published : entry.published.clone(), source : entry.source.clone(), rights : entry.rights.map( | r | r.content ).clone(), - media : ( !media.is_empty() ).then( || media.join( ", " ) ), + media: ( !media.is_empty() ).then( || media ), // qqq : why join? language : entry.language.clone(), feed_link, @@ -120,97 +111,6 @@ pub trait FrameStore // qqq : what is update? what update? don't use word update without noun and explanation what deos it mean // aaa : fixed comments -#[ async_trait::async_trait( ?Send ) ] -impl FrameStore for FeedStorage< SledStorage > -{ - async fn frames_list( &mut self ) -> Result< ListReport > - { - let res = table( "frame" ).select().execute( &mut *self.storage.lock().await ).await?; - - let mut reports = Vec::new(); - let all_frames = - if let Payload::Select { labels: label_vec, rows: rows_vec } = res - { - SelectedEntries - { - selected_rows : rows_vec, - selected_columns : label_vec, - } - } - else - { - SelectedEntries::new() - }; - - let mut feeds_map = HashMap::new(); - - for row in all_frames.selected_rows - { - let title_val = row.last().unwrap().clone(); - let title = String::from( title_val ); - feeds_map.entry( title ) - .and_modify( | vec : &mut Vec< Vec< Value > > | vec.push( row.clone() ) ) - .or_insert( vec![ row ] ) - ; - } - - for ( title, frames ) in feeds_map - { - let mut report = FramesReport::new( title ); - report.existing_frames = frames.len(); - report.selected_frames = SelectedEntries - { - selected_rows : frames, - selected_columns : all_frames.selected_columns.clone(), - }; - reports.push( report ); - } - - Ok( ListReport( reports ) ) - } - - async fn frames_save( &mut self, frames : Vec< Frame > ) -> Result< Payload > - { - let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); - - let insert = table( "frame" ) - .insert() - .columns - ( - self.frame_fields.iter().map( | field | field[ 0 ] ).join( "," ).as_str() - ) - .values( entries_rows ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to insert frames" )? - ; - - Ok( insert ) - } - - async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () > - { - let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); - - for entry in entries_rows - { - let _update = table( "frame" ) - .update() - .set( "title", entry[ 1 ].to_owned() ) - .set( "content", entry[ 4 ].to_owned() ) - .set( "links", entry[ 5 ].to_owned() ) - .set( "summary", entry[ 6 ].to_owned() ) - .set( "published", entry[ 8 ].to_owned() ) - .set( "media", entry[ 9 ].to_owned() ) - .filter( col( "id" ).eq( entry[ 0 ].to_owned() ) ) - .execute( &mut *self.storage.lock().await ) - .await - .context( "Failed to update frames" )? - ; - } - Ok( () ) - } -} // qqq : what is it for and why? // aaa : added explanation @@ -232,7 +132,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > ; let authors = entry.authors - .map( | authors | text( authors ) ) + .map( | authors | text(authors[0].clone())) .unwrap_or( null() ) ; @@ -242,7 +142,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > ; let links = entry.links - .map( | links | text ( links ) ) + .map( | links | text ( links.join(", ") ) ) .unwrap_or( null() ) ; @@ -252,7 +152,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > ; let categories = entry.categories - .map( | categories | text ( categories ) ) + .map( | categories | text ( categories.join(", ") ) ) .unwrap_or( null() ) ; @@ -264,7 +164,7 @@ impl From< Frame > for Vec< ExprNode< 'static > > let source = entry.source.map( | s | text( s ) ).unwrap_or( null() ); let rights = entry.rights.map( | r | text( r ) ).unwrap_or( null() ); let media = entry.media - .map( | media | text ( media ) ) + .map( | media | text( media.join(", ") ) ) .unwrap_or( null() ) ; diff --git a/module/move/unitore/src/entity/table.rs b/module/move/unitore/src/entity/table.rs index b35ccdcac5..b177c3c934 100644 --- a/module/move/unitore/src/entity/table.rs +++ b/module/move/unitore/src/entity/table.rs @@ -2,14 +2,9 @@ use crate::*; use error_tools::Result; -use gluesql:: -{ - sled_storage::SledStorage, - prelude::Payload, -}; +use gluesql::prelude::Payload; use action::table::TablesReport; -use storage::FeedStorage; /// Functions for tables informantion. #[ async_trait::async_trait( ?Send ) ] @@ -21,26 +16,3 @@ pub trait TableStore /// List columns of table. async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > >; } - -#[ async_trait::async_trait( ?Send ) ] -impl TableStore for FeedStorage< SledStorage > -{ - async fn tables_list( &mut self ) -> Result< TablesReport > - { - let glue = &mut *self.storage.lock().await; - let payloads = glue.execute( "SELECT * FROM GLUE_TABLE_COLUMNS" ).await?; - - let report = TablesReport::new( payloads ); - - Ok( report ) - } - - async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > > - { - let glue = &mut *self.storage.lock().await; - let query_str = format!( "SELECT * FROM GLUE_TABLE_COLUMNS WHERE TABLE_NAME='{}'", table_name ); - let payloads = glue.execute( &query_str ).await?; - - Ok( payloads ) - } -} diff --git a/module/move/unitore/src/executor.rs b/module/move/unitore/src/executor.rs index 93e30efee1..88c5a85821 100644 --- a/module/move/unitore/src/executor.rs +++ b/module/move/unitore/src/executor.rs @@ -43,7 +43,6 @@ where /// Run feed updates. pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > { - //let ca = wca::CommandsAggregator::new(); let ca = wca::CommandsAggregator::former() .command( "frames.download" ) .hint( "Download frames from feed sources provided in config files." ) @@ -174,7 +173,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > "Subject: table name.\n", " Example: .table.list feed", )) - .subject().hint( "Name" ).kind( wca::Type::String ).optional( false ).end() + .subject().hint( "Name" ).kind( wca::Type::String ).optional( true ).end() .routine( | o : VerifiedCommand | { match action( table_list, &o.args ) diff --git a/module/move/unitore/src/lib.rs b/module/move/unitore/src/lib.rs index cc00707681..a6b66a5716 100644 --- a/module/move/unitore/src/lib.rs +++ b/module/move/unitore/src/lib.rs @@ -7,5 +7,6 @@ pub mod tool; pub mod command; pub mod action; pub mod entity; +pub mod sled_adapter; // qqq : src/Readmу.md with file structure please diff --git a/module/move/unitore/src/sled_adapter/config.rs b/module/move/unitore/src/sled_adapter/config.rs new file mode 100644 index 0000000000..a304bb47a8 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/config.rs @@ -0,0 +1,53 @@ +use crate::*; +use error_tools::{ err, Result }; +use gluesql:: +{ + core:: + { + ast_builder::{ col, table, text, Execute, }, + executor::Payload, + }, + sled_storage::SledStorage, +}; +use entity::config::{ Config, ConfigStore }; + +#[ async_trait::async_trait( ?Send ) ] +impl ConfigStore for storage::FeedStorage< SledStorage > +{ + async fn config_add( &mut self, config : &Config ) -> Result< Payload > + { + let res = table( "config" ) + .insert() + .columns + ( + "path", + ) + .values( vec![ vec![ text( config.path() ) ] ] ) + .execute( &mut *self.storage.lock().await ) + .await; + + Ok( res? ) + } + + async fn config_delete( &mut self, config : &Config ) -> Result< Payload > + { + let res = table( "config" ) + .delete() + .filter( col( "path" ).eq( format!( "'{}'", config.path() ) ) ) + .execute( &mut *self.storage.lock().await ) + .await?; + + if res == Payload::Delete( 0 ) + { + return Err( err!( format!( "Config file with path {} not found in storage", config.path() ) ) ) + } + + Ok( res ) + } + + async fn config_list( &mut self ) -> Result< Payload > + { + let res = table( "config" ).select().execute( &mut *self.storage.lock().await ).await?; + Ok( res ) + } +} diff --git a/module/move/unitore/src/sled_adapter/feed.rs b/module/move/unitore/src/sled_adapter/feed.rs new file mode 100644 index 0000000000..fc31f07af9 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/feed.rs @@ -0,0 +1,195 @@ +use crate::*; +use std::time::Duration; +use error_tools::{ Result, for_app::Context }; +use gluesql:: +{ + core:: + { + ast_builder::{ col, null, table, text, Execute, timestamp, ExprNode }, + executor::Payload, + data::Value, + chrono::SecondsFormat, + }, + sled_storage::SledStorage, +}; +use entity:: +{ + feed::{ Feed, FeedStore }, + frame::FrameStore, +}; +use action:: +{ + feed::FeedsReport, + frame::{ UpdateReport, SelectedEntries, FramesReport }, +}; +use storage::FeedStorage; +use wca::wtools::Itertools; + +#[ async_trait::async_trait( ?Send ) ] +impl FeedStore for FeedStorage< SledStorage > +{ + async fn feeds_list( &mut self ) -> Result< FeedsReport > + { + let res = table( "feed" ) + .select() + .project( "title, link, update_period, config_file" ) + .execute( &mut *self.storage.lock().await ) + .await? + ; + + let mut report = FeedsReport::new(); + match res + { + Payload::Select { labels: label_vec, rows: rows_vec } => + { + report.0 = SelectedEntries + { + selected_rows : rows_vec, + selected_columns : label_vec, + } + }, + _ => {}, + } + + Ok( report ) + } + + async fn feeds_update( &mut self, feed : Vec< Feed > ) -> Result< () > + { + for feed in feed + { + let _update = table( "feed" ) + .update() + .set( "title", feed.title.map( text ).unwrap_or( null() ) ) + .set( + "updated", + feed.updated.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) + .set( "authors", feed.authors.map( text ).unwrap_or( null() ) ) + .set( "description", feed.description.map( text ).unwrap_or( null() ) ) + .set( + "published", + feed.published.map( | d | timestamp( d.to_rfc3339_opts( SecondsFormat::Millis, true ) ) ).unwrap_or( null() ), + ) + .filter( col( "link" ).eq( feed.link.to_string() ) ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to insert feed" )? + ; + } + + Ok( () ) + } + + async fn feeds_process + ( + &mut self, + feeds : Vec< ( feed_rs::model::Feed, Duration, url::Url ) >, + ) -> Result< UpdateReport > + { + let mut new_entries = Vec::new(); + let mut modified_entries = Vec::new(); + let mut reports = Vec::new(); + + for feed in &feeds + { + let mut frames_report = FramesReport::new( feed.0.title.clone().unwrap().content ); + + let existing_frames = table( "frame" ) + .select() + .filter( col( "feed_link" ).eq( text( feed.2.to_string() ) ) ) + .project( "id, published" ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to get existing frames while saving new frames" )? + ; + + if let Some( rows ) = existing_frames.select() + { + let rows = rows.collect::< Vec< _ > >(); + frames_report.existing_frames = rows.len(); + let existing_entries = rows.iter() + .map( | r | ( r.get( "id" ).map( | &val | val.clone() ), r.get( "published" ).map( | &val | val.clone() ) ) ) + .flat_map( | ( id, published ) | + id.map( | id | + ( + id, + published.map( | date | + { + match date + { + Value::Timestamp( date_time ) => Some( date_time ), + _ => None, + } + } ) + .flatten() + ) + ) + ) + .flat_map( | ( id, published ) | match id { Value::Str( id ) => Some( ( id, published ) ), _ => None } ) + .collect_vec() + ; + + let existing_ids = existing_entries.iter().map( | ( id, _ ) | id ).collect_vec(); + for entry in &feed.0.entries + { + // if extry with same id is already in db, check if it is updated + if let Some( position ) = existing_ids.iter().position( | &id | id == &entry.id ) + { + if let Some( date ) = existing_entries[ position ].1 + { + if date.and_utc() != entry.published.unwrap() + { + frames_report.updated_frames += 1; + modified_entries.push( ( entry.clone(), feed.2.to_string() ).into() ); + } + } + } + else + { + frames_report.new_frames += 1; + new_entries.push( ( entry.clone(), feed.2.to_string() ).into() ); + } + } + } + reports.push( frames_report ); + } + + if !new_entries.is_empty() + { + let _saved_report = self.frames_save( new_entries ).await?; + } + if !modified_entries.is_empty() + { + let _updated_report = self.frames_update( modified_entries ).await?; + } + + Ok( UpdateReport( reports ) ) + } + + async fn feeds_save( &mut self, feed : Vec< Feed > ) -> Result< Payload > + { + let feeds_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | feed | feed.into() ).collect_vec(); + + let insert = table( "feed" ) + .insert() + .columns + ( + "link, + title, + updated, + authors, + description, + published, + update_period, + config_file", + ) + .values( feeds_rows ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to insert feeds" )? + ; + + Ok( insert ) + } +} diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs new file mode 100644 index 0000000000..7ce1f33641 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -0,0 +1,168 @@ +use crate::*; +use std::collections::HashMap; +use error_tools::{ Result, for_app::Context }; +use gluesql:: +{ + core:: + { + ast_builder::{ col, table, Execute, ExprNode }, + executor::Payload, + data::Value, + }, + sled_storage::SledStorage, +}; +use entity::frame::{ FrameStore, Frame }; +use action::frame::{ SelectedEntries, FramesReport, ListReport }; +use storage::FeedStorage; +use wca::wtools::Itertools; + +#[ async_trait::async_trait( ?Send ) ] +impl FrameStore for FeedStorage< SledStorage > +{ + async fn frames_list( &mut self ) -> Result< ListReport > + { + let res = table( "frame" ).select().execute( &mut *self.storage.lock().await ).await?; + + let mut reports = Vec::new(); + let all_frames = + if let Payload::Select { labels: label_vec, rows: rows_vec } = res + { + SelectedEntries + { + selected_rows : rows_vec, + selected_columns : label_vec, + } + } + else + { + SelectedEntries::new() + }; + + let mut feeds_map = HashMap::new(); + + for row in all_frames.selected_rows + { + let title_val = row.last().unwrap().clone(); + let title = String::from( title_val ); + feeds_map.entry( title ) + .and_modify( | vec : &mut Vec< Vec< Value > > | vec.push( row.clone() ) ) + .or_insert( vec![ row ] ) + ; + } + + for ( title, frames ) in feeds_map + { + let mut report = FramesReport::new( title ); + report.existing_frames = frames.len(); + report.selected_frames = SelectedEntries + { + selected_rows : frames, + selected_columns : all_frames.selected_columns.clone(), + }; + reports.push( report ); + } + + Ok( ListReport( reports ) ) + } + + async fn frames_save( &mut self, frames : Vec< Frame > ) -> Result< Payload > + { + let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); + + // let glue = &mut *self.storage.lock().await; + + // /// Frame id. + // pub id : String, + // /// Frame title. + // pub title : Option< String >, + // stored_time : Option< DateTime< Utc > >, + // authors : Option< Vec< String > >, + // content : Option< String >, + // links : Option< Vec< String > >, + // summary : Option< String >, + // categories : Option< Vec< String > >, + // published : Option< DateTime< Utc > >, + // source : Option< String >, + // rights : Option< String >, + // media : Option< Vec< String > >, + // language : Option< String >, + // feed_link : String, + + // use gluesql::core::ast_builder::text; + // let mut values_str = String::new(); + // let null = "NULL".to_string(); + // let values_str = frames.into_iter().map(|frame| format!( + // "('{}', {}, '{}', {}, {}, {}, '{}', {}, '{}')", + // frame.id, + // frame.title.map(|t|format!("'{}'", t)).unwrap_or( "Null".to_string() ), + // frame.stored_time.map(|d|d.to_string()).unwrap_or("Null".to_string()), + // frame.authors.map(|authors| {let authors = authors.into_iter().map(|a|format!("'[\"{}\"]'", a)).collect::>(); authors.join(", ")}).unwrap_or("'[]'".to_string()), + // null.clone(), + // frame.links.map(|links| {let links = links.into_iter().map(|a|format!("\"{}\"", a)).collect::>(); format!("'[{}]'", &links.join(", "))}).unwrap_or("'[]'".to_string()), + // frame.summary.unwrap_or(null.clone()), + // frame.categories.map(|categories| {let categories = categories.into_iter().map(|a|format!("{}", a)).collect::>(); dbg!(format!("'[\"{}\"]'", &categories.join(", ")))}).unwrap_or(null.clone()), + // frame.published.map(|d|d.to_string()).unwrap_or(null.clone()), + // frame.source.unwrap_or(null.clone()), + // frame.rights.unwrap_or(null.clone()), + // // frame.media.map(|media| {let media = media.into_iter().map(|a|format!("\"{}\"", a)).collect::>(); media.join(", ")}).unwrap_or(null.clone()), + // frame.language.unwrap_or(null.clone()), + // frame.feed_link, + // ) + // ) + // .collect::>(); + + // for frame in frames + // { + // let frame_str = format!( + // "({}, {}, {})", + // frame.id, frame.title.unwrap_or( "NULL".to_string() ), frame.stored_time.map(|d|d.to_string()).unwrap_or("NULL".to_string())); + // values_str.push_str(&format!("({}),", frame_str )); + // } + // let query_str = format!( "INSERT INTO frame(id, title, stored_time, authors, content, links, summary, categories, published) VALUES {};", values_str.join(", ") ); + //println!("{}", query_str); + // let mut payloads = glue.execute( &query_str ).await?; + + // INSERT INTO ListType VALUES + // (1, '[1, 2, 3]'), + // (2, '["hello", "world", 30, true, [9,8]]'), + // (3, '[{ "foo": 100, "bar": [true, 0, [10.5, false] ] }, 10, 20]'); + + let insert = table( "frame" ) + .insert() + .columns + ( + self.frame_fields.iter().map( | field | field[ 0 ] ).join( "," ).as_str() + ) + .values( entries_rows ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to insert frames" )? + ; + + Ok( insert ) + } + + async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () > + { + //let entries_rows : Vec< Vec< ExprNode< 'static > > > = Vec::new(); + let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); + + for entry in entries_rows + { + let _update = table( "frame" ) + .update() + .set( "title", entry[ 1 ].to_owned() ) + .set( "content", entry[ 4 ].to_owned() ) + .set( "links", entry[ 5 ].to_owned() ) + .set( "summary", entry[ 6 ].to_owned() ) + .set( "published", entry[ 8 ].to_owned() ) + .set( "media", entry[ 9 ].to_owned() ) + .filter( col( "id" ).eq( entry[ 0 ].to_owned() ) ) + .execute( &mut *self.storage.lock().await ) + .await + .context( "Failed to update frames" )? + ; + } + Ok( () ) + } +} diff --git a/module/move/unitore/src/sled_adapter/mod.rs b/module/move/unitore/src/sled_adapter/mod.rs new file mode 100644 index 0000000000..ea5e050ec3 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/mod.rs @@ -0,0 +1,4 @@ +mod frame; +mod table; +mod feed; +mod config; \ No newline at end of file diff --git a/module/move/unitore/src/sled_adapter/table.rs b/module/move/unitore/src/sled_adapter/table.rs new file mode 100644 index 0000000000..3aecdd79c9 --- /dev/null +++ b/module/move/unitore/src/sled_adapter/table.rs @@ -0,0 +1,33 @@ +use crate::*; +use error_tools::Result; +use gluesql:: +{ + core::executor::Payload, + sled_storage::SledStorage, +}; +use entity::table::TableStore; +use action::table::TablesReport; +use storage::FeedStorage; + +#[ async_trait::async_trait( ?Send ) ] +impl TableStore for FeedStorage< SledStorage > +{ + async fn tables_list( &mut self ) -> Result< TablesReport > + { + let glue = &mut *self.storage.lock().await; + let payloads = glue.execute( "SELECT * FROM GLUE_TABLE_COLUMNS" ).await?; + + let report = TablesReport::new( payloads ); + + Ok( report ) + } + + async fn table_list( &mut self, table_name : String ) -> Result< Vec< Payload > > + { + let glue = &mut *self.storage.lock().await; + let query_str = format!( "SELECT * FROM GLUE_TABLE_COLUMNS WHERE TABLE_NAME='{}'", table_name ); + let payloads = glue.execute( &query_str ).await?; + + Ok( payloads ) + } +} \ No newline at end of file diff --git a/module/move/unitore/tests/save_feed.rs b/module/move/unitore/tests/save_feed.rs index 2ac5cbcff3..9749d1e176 100644 --- a/module/move/unitore/tests/save_feed.rs +++ b/module/move/unitore/tests/save_feed.rs @@ -49,6 +49,8 @@ async fn test_save_feed_plain() -> Result< () > let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); + println!("{:#?}", entries); + assert_eq!( number_of_frames, 10 ); Ok( () ) From a3839e8428acbeb4ac208f9d3f2df1b2a617c170 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Mon, 8 Apr 2024 17:35:54 +0300 Subject: [PATCH 04/11] update file structure --- module/move/unitore/src/Readme.md | 6 + module/move/unitore/src/action/config.rs | 8 +- module/move/unitore/src/command/config.rs | 139 ++++++++++++ module/move/unitore/src/command/feed.rs | 53 +++++ module/move/unitore/src/command/frame.rs | 90 ++++++++ module/move/unitore/src/command/mod.rs | 6 +- module/move/unitore/src/command/query.rs | 59 +++++ module/move/unitore/src/command/table.rs | 54 ++++- module/move/unitore/src/executor.rs | 253 +++++----------------- 9 files changed, 460 insertions(+), 208 deletions(-) create mode 100644 module/move/unitore/src/command/config.rs create mode 100644 module/move/unitore/src/command/feed.rs create mode 100644 module/move/unitore/src/command/frame.rs create mode 100644 module/move/unitore/src/command/query.rs diff --git a/module/move/unitore/src/Readme.md b/module/move/unitore/src/Readme.md index e69de29bb2..be6a15dfc1 100644 --- a/module/move/unitore/src/Readme.md +++ b/module/move/unitore/src/Readme.md @@ -0,0 +1,6 @@ +## File structure + +- `command` - Contains commands for unitore cli. +- `action` - Contains functions that are executed when command are performed. +- `entity` - Contains entities that are used in commands execution. +- `tool` - Additional functions for convenient use of application. \ No newline at end of file diff --git a/module/move/unitore/src/action/config.rs b/module/move/unitore/src/action/config.rs index 952d997fbc..e9d65d176c 100644 --- a/module/move/unitore/src/action/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -21,6 +21,8 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args .into() ; + let path = proper_path_tools::path::normalize( path ); + let mut err_str = format!( "Invalid path for config file {:?}", path ); let start = path.components().next(); @@ -37,7 +39,11 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args err_str = format!( "Invalid path for config file {:?}", abs_path ); } } - let path = path.canonicalize().context( err_str )?; + + if !path.exists() + { + return Err( error_tools::for_app::Error::msg( err_str ) ); + } let config = Config::new( path.to_string_lossy().to_string() ); let mut manager = FeedManager::new( storage ); diff --git a/module/move/unitore/src/command/config.rs b/module/move/unitore/src/command/config.rs new file mode 100644 index 0000000000..3fef800b22 --- /dev/null +++ b/module/move/unitore/src/command/config.rs @@ -0,0 +1,139 @@ +//! Feed command. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use storage::FeedStorage; +use action::{ Report, config::{ config_add, config_delete, config_list } }; +use error_tools::Result; + +pub struct ConfigCommand; + +impl ConfigCommand +{ + pub fn add() -> Result< Command > + { + + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "config.add" ) + .long_hint( concat! + ( + "Add file with feeds configurations. Subject: path to config file.\n", + " Example: .config.add ./config/feeds.toml\n", + " The file should contain config entities with fields:\n", + " - `update_period` : update frequency for feed. Example values: `12h`, `1h 20min`, `2days 5h`;\n", + " - `link` : URL for feed source;\n\n", + " Example:\n", + " [[config]]\n", + " update_period = \"1min\"\n", + " link = \"https://feeds.bbci.co.uk/news/world/rss.xml\"\n", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_add( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } + + pub fn delete() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok( + Command::former() + .phrase( "config.delete" ) + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_delete( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } + + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "config.list" ) + .long_hint( concat! + ( + "List all config files saved in storage.\n", + " Example: .config.list", + )) + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_list( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } +} diff --git a/module/move/unitore/src/command/feed.rs b/module/move/unitore/src/command/feed.rs new file mode 100644 index 0000000000..c713e8b559 --- /dev/null +++ b/module/move/unitore/src/command/feed.rs @@ -0,0 +1,53 @@ +//! Feed command. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, VerifiedCommand }; +use storage::FeedStorage; +use action::{ Report, feed::feeds_list }; +use error_tools::Result; + +pub struct FeedCommand; + +impl FeedCommand +{ + pub fn list() -> Result< Command > + { + + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "feeds.list" ) + .long_hint( concat! + ( + "List all feeds from storage.\n", + " Example: .feeds.list", + )) + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + feeds_list( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/frame.rs b/module/move/unitore/src/command/frame.rs new file mode 100644 index 0000000000..75416b1d71 --- /dev/null +++ b/module/move/unitore/src/command/frame.rs @@ -0,0 +1,90 @@ +//! Frame commands. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, VerifiedCommand }; +use storage::FeedStorage; +use action::{ Report, frame::{ frames_list, frames_download } }; +use error_tools::Result; + +pub struct FrameCommand; + +impl FrameCommand +{ + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "frames.list" ) + .long_hint( concat! + ( + "List all frames saved in storage.\n", + " Example: .frames.list", + )) + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + frames_list( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } + + pub fn download() -> Result< Command > + { + + let rt = tokio::runtime::Runtime::new()?; + + Ok( + Command::former() + .phrase( "frames.download" ) + .hint( "Download frames from feed sources provided in config files." ) + .long_hint(concat! + ( + "Download frames from feed sources provided in config files.\n", + " Example: .frames.download", + )) + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + frames_download( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/mod.rs b/module/move/unitore/src/command/mod.rs index 790b2c4ac0..ea0a44e8a3 100644 --- a/module/move/unitore/src/command/mod.rs +++ b/module/move/unitore/src/command/mod.rs @@ -1 +1,5 @@ -pub mod table; \ No newline at end of file +pub mod table; +pub mod frame; +pub mod feed; +pub mod query; +pub mod config; \ No newline at end of file diff --git a/module/move/unitore/src/command/query.rs b/module/move/unitore/src/command/query.rs new file mode 100644 index 0000000000..21931d05af --- /dev/null +++ b/module/move/unitore/src/command/query.rs @@ -0,0 +1,59 @@ +//! Query command. + +use crate::*; +use gluesql::sled_storage::sled::Config; +use wca::{ Command, Type, VerifiedCommand }; +use storage::FeedStorage; +use action::{ Report, query::query_execute }; +use error_tools::Result; + +pub struct QueryCommand; + +impl QueryCommand +{ + pub fn list() -> Result< Command > + { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "query.execute" ) + .long_hint( concat! + ( + "Execute custom query. Subject: query string, with special characters escaped.\n", + " Example query:\n", + " - select all frames:\n", + r#" .query.execute \'SELECT \* FROM frame\'"#, + "\n", + " - select title and link to the most recent frame:\n", + r#" .query.execute \'SELECT title, links, MIN\(published\) FROM frame\'"#, + "\n\n", + )) + .subject().hint( "Query" ).kind( Type::String ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + query_execute( feed_storage, &o.args ).await + }); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + + }) + .end() + ) + } +} \ No newline at end of file diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs index c5d8f5ffab..f40387b4e0 100644 --- a/module/move/unitore/src/command/table.rs +++ b/module/move/unitore/src/command/table.rs @@ -4,21 +4,22 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, Type, VerifiedCommand }; use storage::FeedStorage; -use action::{ Report, table::table_list }; +use action::{ Report, table::{ table_list, tables_list } }; use error_tools::Result; -pub struct TableCommand( Command ); +pub struct TableCommand; impl TableCommand { - pub fn new() -> Result< Self > + pub fn list() -> Result< Command > { let rt = tokio::runtime::Runtime::new()?; - Ok( Self + Ok ( Command::former() + .phrase( "table.list" ) .long_hint( concat! ( "Delete file with feeds configuraiton. Subject: path to config file.\n", @@ -47,10 +48,51 @@ impl TableCommand } }) .end() - ) ) + ) } - pub fn command() +} + +pub struct TablesCommand; + +impl TablesCommand +{ + pub fn list() -> Result< Command > { + let rt = tokio::runtime::Runtime::new()?; + + Ok + ( + Command::former() + .phrase( "tables.list" ) + .long_hint( concat! + ( + "Delete file with feeds configuraiton. Subject: path to config file.\n", + " Example: .config.delete ./config/feeds.toml", + )) + .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() + .routine( move | o : VerifiedCommand | + { + let res = rt.block_on( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + tables_list( feed_storage, &o.args ).await + } ); + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } + }) + .end() + ) } } \ No newline at end of file diff --git a/module/move/unitore/src/executor.rs b/module/move/unitore/src/executor.rs index 88c5a85821..94cf1093eb 100644 --- a/module/move/unitore/src/executor.rs +++ b/module/move/unitore/src/executor.rs @@ -2,215 +2,68 @@ use crate::*; use feed_config::SubscriptionConfig; -use gluesql::sled_storage::{ sled::Config, SledStorage }; -use storage::{ Store, FeedStorage }; +use storage::Store; use entity::{ feed::FeedStore, config::ConfigStore, table::TableStore, frame::FrameStore }; -use wca::{ Args, Type, VerifiedCommand }; +use wca::{ Dictionary, Executor, Parser, Verifier }; use error_tools::Result; -use action:: -{ - Report, - frame::{ frames_list, frames_download }, - feed::feeds_list, - config::{ config_add, config_delete, config_list }, - query::query_execute, - table::{ table_list, tables_list }, -}; - -fn action< 'a, F, Fut, R >( async_endpoint : F, args : &'a Args ) -> Result< R > -where - F : FnOnce( FeedStorage< SledStorage >, &'a Args ) -> Fut, - Fut : std::future::Future< Output = Result< R > >, - R : action::Report, -{ - let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) - .unwrap_or( String::from( "./_data" ) ) - ; - - let config = Config::default() - .path( path_to_storage ) - ; - let rt = tokio::runtime::Runtime::new()?; - - rt.block_on( async move - { - let feed_storage = FeedStorage::init_storage( &config ).await?; - async_endpoint( feed_storage, args ).await - } ) -} +use action::Report; /// Run feed updates. pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > { - let ca = wca::CommandsAggregator::former() - .command( "frames.download" ) - .hint( "Download frames from feed sources provided in config files." ) - .long_hint(concat! - ( - "Download frames from feed sources provided in config files.\n", - " Example: .frames.download", - )) - .routine( | o : VerifiedCommand | - { - match action( frames_download, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "feeds.list" ) - .long_hint( concat! - ( - "List all feeds from storage.\n", - " Example: .feeds.list", - )) - .routine( | o : VerifiedCommand | - { - match action( feeds_list, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "frames.list" ) - .long_hint( concat! - ( - "List all frames saved in storage.\n", - " Example: .frames.list", - )) - .routine( | o : VerifiedCommand | - { - match action( frames_list, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.add" ) - .long_hint( concat! - ( - "Add file with feeds configurations. Subject: path to config file.\n", - " Example: .config.add ./config/feeds.toml\n", - " The file should contain config entities with fields:\n", - " - `update_period` : update frequency for feed. Example values: `12h`, `1h 20min`, `2days 5h`;\n", - " - `link` : URL for feed source;\n\n", - " Example:\n", - " [[config]]\n", - " update_period = \"1min\"\n", - " link = \"https://feeds.bbci.co.uk/news/world/rss.xml\"\n", - )) - .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( config_add, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.delete" ) - .long_hint( concat! - ( - "Delete file with feeds configuraiton. Subject: path to config file.\n", - " Example: .config.delete ./config/feeds.toml", - )) - .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( config_delete, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "config.list" ) - .long_hint( concat! - ( - "List all config files saved in storage.\n", - " Example: .config.list", - )) - .routine( | o : VerifiedCommand | - { - match action( config_list, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "tables.list" ) - .long_hint( concat! - ( - "List all tables saved in storage.\n", - " Example: .tables.list", - )) - .routine( | o : VerifiedCommand | - { - match action( tables_list, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "table.list" ) - .long_hint( concat! - ( - "List fields of specified table.\n", - "Subject: table name.\n", - " Example: .table.list feed", - )) - .subject().hint( "Name" ).kind( wca::Type::String ).optional( true ).end() - .routine( | o : VerifiedCommand | - { - match action( table_list, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - - .command( "query.execute" ) - .long_hint( concat! - ( - "Execute custom query. Subject: query string, with special characters escaped.\n", - " Example query:\n", - " - select all frames:\n", - r#" .query.execute \'SELECT \* FROM frame\'"#, - "\n", - " - select title and link to the most recent frame:\n", - r#" .query.execute \'SELECT title, links, MIN\(published\) FROM frame\'"#, - "\n\n", - )) - .subject().hint( "Query" ).kind( Type::String ).optional( false ).end() - .routine( | o : VerifiedCommand | - { - match action( query_execute, &o.args ) - { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), - } - }) - .end() - .help_variants( [ wca::HelpVariants::General, wca::HelpVariants::SubjectCommand ] ) - .perform(); + // init parser + let parser = Parser; + + // init converter + let dictionary = &Dictionary::former() + .command + ( + command::config::ConfigCommand::add()? + ) + .command + ( + command::config::ConfigCommand::delete()? + ) + .command + ( + command::config::ConfigCommand::list()? + ) + .command + ( + command::frame::FrameCommand::list()? + ) + .command + ( + command::frame::FrameCommand::download()? + ) + .command + ( + command::feed::FeedCommand::list()? + ) + .command + ( + command::table::TablesCommand::list()? + ) + .command + ( + command::table::TableCommand::list()? + ) + .command + ( + command::query::QueryCommand::list()? + ) + .form(); + let verifier = Verifier; + + // init executor + let executor = Executor::former().form(); let args = std::env::args().skip( 1 ).collect::< Vec< String > >(); - ca.perform( args )?; + let raw_program = parser.parse( args ).unwrap(); + let grammar_program = verifier.to_program( dictionary, raw_program ).unwrap(); + + executor.program( dictionary, grammar_program )?; Ok( () ) } From 61b8e672e7f8a2b95abc8010294705c09227de85 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Mon, 8 Apr 2024 17:49:32 +0300 Subject: [PATCH 05/11] fixed comments --- module/move/unitore/src/action/mod.rs | 5 +- module/move/unitore/src/entity/config.rs | 3 + .../move/unitore/src/sled_adapter/config.rs | 2 + module/move/unitore/src/sled_adapter/feed.rs | 2 + module/move/unitore/src/sled_adapter/frame.rs | 2 + module/move/unitore/src/sled_adapter/table.rs | 2 + .../move/unitore/tests/{frame.rs => basic.rs} | 0 .../unitore/tests/fixtures/plain_feed.xml | 4 +- ...pdate_newer_feed.rs => frames_download.rs} | 41 +++++++++++-- module/move/unitore/tests/save_feed.rs | 57 ------------------- 10 files changed, 54 insertions(+), 64 deletions(-) rename module/move/unitore/tests/{frame.rs => basic.rs} (100%) rename module/move/unitore/tests/{update_newer_feed.rs => frames_download.rs} (67%) delete mode 100644 module/move/unitore/tests/save_feed.rs diff --git a/module/move/unitore/src/action/mod.rs b/module/move/unitore/src/action/mod.rs index 80471e3650..5f958526f9 100644 --- a/module/move/unitore/src/action/mod.rs +++ b/module/move/unitore/src/action/mod.rs @@ -8,6 +8,8 @@ // entity - with all entities // tool - with something not directly related to the problem, but convenient to have as a separate function/structure +// aaa: added folders + pub mod frame; pub mod feed; pub mod config; @@ -15,7 +17,8 @@ pub mod query; pub mod table; // qqq : what is it for? purpose? -/// General report. +// aaa : added explanation +/// General report trait for commands return type. pub trait Report : std::fmt::Display + std::fmt::Debug { /// Print report of executed command. diff --git a/module/move/unitore/src/entity/config.rs b/module/move/unitore/src/entity/config.rs index cf3049acbe..b28a90a1dc 100644 --- a/module/move/unitore/src/entity/config.rs +++ b/module/move/unitore/src/entity/config.rs @@ -45,9 +45,12 @@ pub trait ConfigStore // qqq : use AbsolutePath newtype from `path_tools` // qqq : normalize all paths with `path_tools::path::normalize` // https://docs.rs/proper_path_tools/latest/proper_path_tools/path/fn.normalize.html +// added path normalization // unitore .query.execute \'SELECT \* FROM feed\' // qqq : something is broken in this table. also lack of association with config files +// aaa : added association with config // unitore .query.execute \'SELECT \* FROM x\' // qqq : it is not obvious where one record ends and another begins +// aaa : added id highlight diff --git a/module/move/unitore/src/sled_adapter/config.rs b/module/move/unitore/src/sled_adapter/config.rs index a304bb47a8..0691eb7d7f 100644 --- a/module/move/unitore/src/sled_adapter/config.rs +++ b/module/move/unitore/src/sled_adapter/config.rs @@ -1,3 +1,5 @@ +// Config file operation with Sled storage. + use crate::*; use error_tools::{ err, Result }; use gluesql:: diff --git a/module/move/unitore/src/sled_adapter/feed.rs b/module/move/unitore/src/sled_adapter/feed.rs index fc31f07af9..cabdeddd23 100644 --- a/module/move/unitore/src/sled_adapter/feed.rs +++ b/module/move/unitore/src/sled_adapter/feed.rs @@ -1,3 +1,5 @@ +// Feed operation with Sled storage. + use crate::*; use std::time::Duration; use error_tools::{ Result, for_app::Context }; diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs index 7ce1f33641..88f76512f2 100644 --- a/module/move/unitore/src/sled_adapter/frame.rs +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -1,3 +1,5 @@ +// Frames operation with Sled storage. + use crate::*; use std::collections::HashMap; use error_tools::{ Result, for_app::Context }; diff --git a/module/move/unitore/src/sled_adapter/table.rs b/module/move/unitore/src/sled_adapter/table.rs index 3aecdd79c9..b19b26a5bd 100644 --- a/module/move/unitore/src/sled_adapter/table.rs +++ b/module/move/unitore/src/sled_adapter/table.rs @@ -1,3 +1,5 @@ +// Table and columns info operations from Sled storage. + use crate::*; use error_tools::Result; use gluesql:: diff --git a/module/move/unitore/tests/frame.rs b/module/move/unitore/tests/basic.rs similarity index 100% rename from module/move/unitore/tests/frame.rs rename to module/move/unitore/tests/basic.rs diff --git a/module/move/unitore/tests/fixtures/plain_feed.xml b/module/move/unitore/tests/fixtures/plain_feed.xml index 53c32e9fd1..7048caabd0 100644 --- a/module/move/unitore/tests/fixtures/plain_feed.xml +++ b/module/move/unitore/tests/fixtures/plain_feed.xml @@ -69,7 +69,7 @@ - + @@ -328,7 +328,7 @@ Thu, 14 Mar 2024 13:00:00 +0000 - + diff --git a/module/move/unitore/tests/update_newer_feed.rs b/module/move/unitore/tests/frames_download.rs similarity index 67% rename from module/move/unitore/tests/update_newer_feed.rs rename to module/move/unitore/tests/frames_download.rs index 4702acfcf0..e97265210a 100644 --- a/module/move/unitore/tests/update_newer_feed.rs +++ b/module/move/unitore/tests/frames_download.rs @@ -8,20 +8,53 @@ use gluesql:: }, sled_storage::sled::Config, }; +use wca::wtools::Itertools; use unitore:: { feed_config::SubscriptionConfig, storage::FeedStorage, - entity::{ feed::FeedStore, frame::FrameStore }, + entity::{ frame::FrameStore, feed::FeedStore }, }; -use wca::wtools::Itertools; use error_tools::Result; +#[ tokio::test ] +async fn test_save() -> Result< () > +{ + let config = gluesql::sled_storage::sled::Config::default() + .path( "./test_save".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + + let feed_config = SubscriptionConfig + { + update_period : std::time::Duration::from_secs( 1000 ), + link : url::Url::parse( "https://www.nasa.gov/feed/" )?, + }; + + let mut feeds = Vec::new(); + + let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; + feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); + feed_storage.feeds_process( feeds ).await?; + + let entries = feed_storage.frames_list().await?; + + let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); + + println!("{:#?}", entries); + + assert_eq!( number_of_frames, 10 ); + + Ok( () ) +} + #[ tokio::test ] async fn test_update() -> Result< () > { let config = Config::default() - .path( "./test".to_owned() ) + .path( "./test_update".to_owned() ) .temporary( true ) ; @@ -77,4 +110,4 @@ async fn test_update() -> Result< () > assert!( updated.is_some() ); let _updated = updated.unwrap(); Ok( () ) -} \ No newline at end of file +} diff --git a/module/move/unitore/tests/save_feed.rs b/module/move/unitore/tests/save_feed.rs deleted file mode 100644 index 9749d1e176..0000000000 --- a/module/move/unitore/tests/save_feed.rs +++ /dev/null @@ -1,57 +0,0 @@ -use feed_rs::parser as feed_parser; -use unitore:: -{ - feed_config::SubscriptionConfig, - storage::FeedStorage, - entity::{ frame::FrameStore, feed::FeedStore }, -}; -use error_tools::Result; - -#[ tokio::test ] -async fn test_save_feed_plain() -> Result< () > -{ - // let mut f_store = MockFeedStore::new(); - // f_store - // .expect_process_feeds() - // .times( 1 ) - // .returning( | _ | Ok( UpdateReport( - // vec! [ FramesReport - // { - // new_frames : 2, - // updated_frames : 0, - // selected_frames : SelectedEntries::new(), - // existing_frames : 0, - // feed_link : String::new(), - // is_new_feed : false, - // } ] ) ) ) - // ; - - let config = gluesql::sled_storage::sled::Config::default() - .path( "./test".to_owned() ) - .temporary( true ) - ; - - let mut feed_storage = FeedStorage::init_storage( &config ).await?; - - let feed_config = SubscriptionConfig - { - update_period : std::time::Duration::from_secs( 1000 ), - link : url::Url::parse( "https://www.nasa.gov/feed/" )?, - }; - - let mut feeds = Vec::new(); - - let feed = feed_parser::parse( include_str!("./fixtures/plain_feed.xml").as_bytes() )?; - feeds.push( ( feed, feed_config.update_period.clone(), feed_config.link.clone() ) ); - feed_storage.feeds_process( feeds ).await?; - - let entries = feed_storage.frames_list().await?; - - let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); - - println!("{:#?}", entries); - - assert_eq!( number_of_frames, 10 ); - - Ok( () ) -} From c4e9d99e133cdf2ebc5472b4bef38cd9d8511916 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 11:11:44 +0300 Subject: [PATCH 06/11] fixed frames values --- module/move/unitore/src/entity/frame.rs | 40 +++++++++++++++++++++---- module/move/unitore/src/storage.rs | 8 ++--- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/module/move/unitore/src/entity/frame.rs b/module/move/unitore/src/entity/frame.rs index 32ba98a4ca..de687ec425 100644 --- a/module/move/unitore/src/entity/frame.rs +++ b/module/move/unitore/src/entity/frame.rs @@ -76,17 +76,21 @@ impl From< ( feed_rs::model::Entry, String ) > for Frame stored_time : entry.updated, authors: ( !authors.is_empty() ).then( || authors ), // qqq : why join? + // aaa : fixed, saved as list content, links: ( !links.is_empty() ).then( || links ), // qqq : why join? + // aaa : fixed, saved as list summary : entry.summary.map( | c | c.content ).clone(), categories: ( !categories.is_empty() ).then( || categories ), // qqq : why join? + // aaa : fixed, saved as list published : entry.published.clone(), source : entry.source.clone(), rights : entry.rights.map( | r | r.content ).clone(), media: ( !media.is_empty() ).then( || media ), // qqq : why join? + // aaa : fixed, saved as list language : entry.language.clone(), feed_link, } @@ -132,27 +136,46 @@ impl From< Frame > for Vec< ExprNode< 'static > > ; let authors = entry.authors - .map( | authors | text(authors[0].clone())) + .map( | authors | + text + ( + format!( "[{}]", authors.into_iter().map( | a | format!( "\"{}\"", a ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) .unwrap_or( null() ) ; let content = entry.content - .map( | content | text ( content ) ) + .map( | content | text( content ) ) .unwrap_or( null() ) ; let links = entry.links - .map( | links | text ( links.join(", ") ) ) + .map( | links | + text + ( + format!( "[{}]", links.into_iter().map( | link | format!( "\"{}\"", link ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) .unwrap_or( null() ) ; let summary = entry.summary - .map( | summary | text ( summary ) ) + .map( | summary | text( summary ) ) .unwrap_or( null() ) ; let categories = entry.categories - .map( | categories | text ( categories.join(", ") ) ) + .map( | categories | + text + ( + format! + ( + "[{}]", + categories.into_iter().map( | category | format!( "\"{}\"", category ) ).collect::< Vec< _ > >().join( ", " ), + ) + ) + ) .unwrap_or( null() ) ; @@ -164,7 +187,12 @@ impl From< Frame > for Vec< ExprNode< 'static > > let source = entry.source.map( | s | text( s ) ).unwrap_or( null() ); let rights = entry.rights.map( | r | text( r ) ).unwrap_or( null() ); let media = entry.media - .map( | media | text( media.join(", ") ) ) + .map( | media | + text + ( + format!( "[{}]", media.into_iter().map( | media | format!( "\"{}\"", media ) ).collect::< Vec< _ > >().join( ", " ) ) + ) + ) .unwrap_or( null() ) ; diff --git a/module/move/unitore/src/storage.rs b/module/move/unitore/src/storage.rs index 86295eb4f0..98c0fdd72b 100644 --- a/module/move/unitore/src/storage.rs +++ b/module/move/unitore/src/storage.rs @@ -73,15 +73,15 @@ impl FeedStorage< SledStorage > [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], [ "title", "TEXT", "Title of the frame" ], [ "stored_time", "TIMESTAMP", "Time at which this item was fetched from source." ], - [ "authors", "TEXT", "List of authors of the frame, optional." ], + [ "authors", "LIST", "List of authors of the frame, optional." ], [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], - [ "links", "TEXT", "List of links associated with this item of related Web page and attachments." ], + [ "links", "LIST", "List of links associated with this item of related Web page and attachments." ], [ "summary", "TEXT", "Short summary, abstract, or excerpt of the frame item, optional." ], - [ "categories", "TEXT", "Specifies a list of categories that the item belongs to." ], + [ "categories", "LIST", "Specifies a list of categories that the item belongs to." ], [ "published", "TIMESTAMP", "Time at which this item was first published or updated." ], [ "source", "TEXT", "Specifies the source feed if the frame was copied from one feed into another feed, optional." ], [ "rights", "TEXT", "Conveys information about copyrights over the feed, optional." ], - [ "media", "TEXT", "List of media oblects, encountered in the frame, optional." ], + [ "media", "LIST", "List of media oblects, encountered in the frame, optional." ], [ "language", "TEXT", "The language specified on the item, optional." ], [ "feed_link", "TEXT", "Link of feed that contains this frame." ], ]; From 595d760bef598c1240d2e178de4d4d3ddba455df Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 11:49:08 +0300 Subject: [PATCH 07/11] fix config path --- module/move/unitore/src/action/config.rs | 8 +++- .../tests/{config_add.rs => config.rs} | 42 ++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) rename module/move/unitore/tests/{config_add.rs => config.rs} (50%) diff --git a/module/move/unitore/src/action/config.rs b/module/move/unitore/src/action/config.rs index e9d65d176c..4c190fa45c 100644 --- a/module/move/unitore/src/action/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -43,9 +43,13 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args if !path.exists() { return Err( error_tools::for_app::Error::msg( err_str ) ); - } + } - let config = Config::new( path.to_string_lossy().to_string() ); + //let abs_path = proper_path_tools::path::canonicalize( path )?; + let abs_path = path.canonicalize()?; + println!("{}", abs_path.to_string_lossy().to_string() ); + + let config = Config::new( abs_path.to_string_lossy().to_string() ); let mut manager = FeedManager::new( storage ); let config_report = manager.storage diff --git a/module/move/unitore/tests/config_add.rs b/module/move/unitore/tests/config.rs similarity index 50% rename from module/move/unitore/tests/config_add.rs rename to module/move/unitore/tests/config.rs index c938eb5d99..a30448611b 100644 --- a/module/move/unitore/tests/config_add.rs +++ b/module/move/unitore/tests/config.rs @@ -1,23 +1,26 @@ use std::path::PathBuf; -use gluesql::sled_storage::sled::Config; +use gluesql::{ + sled_storage::sled::Config, + test_suite::data_type::list, + prelude::Payload::Select, +}; use unitore:: { executor::FeedManager, storage::FeedStorage, - entity::feed::FeedStore, + entity::{ feed::FeedStore, config::ConfigStore }, action::config, }; use error_tools::Result; #[ tokio::test ] -async fn add_config_file() -> Result< () > +async fn config_add() -> Result< () > { let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); - let path = path.canonicalize().expect( "Invalid path" ); let config = Config::default() - .path( "./test".to_owned() ) + .path( "./test_add".to_owned() ) .temporary( true ) ; @@ -39,3 +42,32 @@ async fn add_config_file() -> Result< () > Ok( () ) } + +#[ tokio::test ] +async fn config_delete() -> Result< () > +{ + let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); + + let config = Config::default() + .path( "./test_del".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path.clone() ) ] ) ).await?; + + config::config_delete( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; + + let list = feed_storage.config_list().await?; + + if let Select{ labels, rows } = list + { + assert!( rows.len() == 0 ) + } + else + { + assert!( false ); + } + + Ok( () ) +} From 695992bbf7377cb8315a18ae9480798b70ce2272 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 12:16:39 +0300 Subject: [PATCH 08/11] fix --- module/move/unitore/src/sled_adapter/frame.rs | 59 ------------------- 1 file changed, 59 deletions(-) diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs index 88f76512f2..325f7f1563 100644 --- a/module/move/unitore/src/sled_adapter/frame.rs +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -71,64 +71,6 @@ impl FrameStore for FeedStorage< SledStorage > { let entries_rows : Vec< Vec< ExprNode< 'static > > > = frames.into_iter().map( | entry | entry.into() ).collect_vec(); - // let glue = &mut *self.storage.lock().await; - - // /// Frame id. - // pub id : String, - // /// Frame title. - // pub title : Option< String >, - // stored_time : Option< DateTime< Utc > >, - // authors : Option< Vec< String > >, - // content : Option< String >, - // links : Option< Vec< String > >, - // summary : Option< String >, - // categories : Option< Vec< String > >, - // published : Option< DateTime< Utc > >, - // source : Option< String >, - // rights : Option< String >, - // media : Option< Vec< String > >, - // language : Option< String >, - // feed_link : String, - - // use gluesql::core::ast_builder::text; - // let mut values_str = String::new(); - // let null = "NULL".to_string(); - // let values_str = frames.into_iter().map(|frame| format!( - // "('{}', {}, '{}', {}, {}, {}, '{}', {}, '{}')", - // frame.id, - // frame.title.map(|t|format!("'{}'", t)).unwrap_or( "Null".to_string() ), - // frame.stored_time.map(|d|d.to_string()).unwrap_or("Null".to_string()), - // frame.authors.map(|authors| {let authors = authors.into_iter().map(|a|format!("'[\"{}\"]'", a)).collect::>(); authors.join(", ")}).unwrap_or("'[]'".to_string()), - // null.clone(), - // frame.links.map(|links| {let links = links.into_iter().map(|a|format!("\"{}\"", a)).collect::>(); format!("'[{}]'", &links.join(", "))}).unwrap_or("'[]'".to_string()), - // frame.summary.unwrap_or(null.clone()), - // frame.categories.map(|categories| {let categories = categories.into_iter().map(|a|format!("{}", a)).collect::>(); dbg!(format!("'[\"{}\"]'", &categories.join(", ")))}).unwrap_or(null.clone()), - // frame.published.map(|d|d.to_string()).unwrap_or(null.clone()), - // frame.source.unwrap_or(null.clone()), - // frame.rights.unwrap_or(null.clone()), - // // frame.media.map(|media| {let media = media.into_iter().map(|a|format!("\"{}\"", a)).collect::>(); media.join(", ")}).unwrap_or(null.clone()), - // frame.language.unwrap_or(null.clone()), - // frame.feed_link, - // ) - // ) - // .collect::>(); - - // for frame in frames - // { - // let frame_str = format!( - // "({}, {}, {})", - // frame.id, frame.title.unwrap_or( "NULL".to_string() ), frame.stored_time.map(|d|d.to_string()).unwrap_or("NULL".to_string())); - // values_str.push_str(&format!("({}),", frame_str )); - // } - // let query_str = format!( "INSERT INTO frame(id, title, stored_time, authors, content, links, summary, categories, published) VALUES {};", values_str.join(", ") ); - //println!("{}", query_str); - // let mut payloads = glue.execute( &query_str ).await?; - - // INSERT INTO ListType VALUES - // (1, '[1, 2, 3]'), - // (2, '["hello", "world", 30, true, [9,8]]'), - // (3, '[{ "foo": 100, "bar": [true, 0, [10.5, false] ] }, 10, 20]'); - let insert = table( "frame" ) .insert() .columns @@ -146,7 +88,6 @@ impl FrameStore for FeedStorage< SledStorage > async fn frames_update( &mut self, feed : Vec< Frame > ) -> Result< () > { - //let entries_rows : Vec< Vec< ExprNode< 'static > > > = Vec::new(); let entries_rows : Vec< Vec< ExprNode< 'static > > > = feed.into_iter().map( | entry | entry.into() ).collect_vec(); for entry in entries_rows From a1385de133ca9aae10c9bfdfdf8cc287ee083546 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 13:52:00 +0300 Subject: [PATCH 09/11] add documentation --- module/move/unitore/src/action/config.rs | 20 +++---- module/move/unitore/src/action/feed.rs | 6 +-- module/move/unitore/src/action/frame.rs | 14 ++--- module/move/unitore/src/action/query.rs | 7 ++- module/move/unitore/src/action/table.rs | 14 +++-- module/move/unitore/src/command/config.rs | 8 ++- module/move/unitore/src/command/feed.rs | 3 ++ module/move/unitore/src/command/frame.rs | 5 +- module/move/unitore/src/command/mod.rs | 2 + module/move/unitore/src/command/query.rs | 5 +- module/move/unitore/src/command/table.rs | 9 +++- module/move/unitore/src/entity/frame.rs | 12 +++++ module/move/unitore/src/executor.rs | 53 +------------------ module/move/unitore/src/feed_config.rs | 10 +++- module/move/unitore/src/lib.rs | 1 + module/move/unitore/src/retriever.rs | 9 ++++ .../move/unitore/src/sled_adapter/config.rs | 2 +- module/move/unitore/src/sled_adapter/feed.rs | 2 +- module/move/unitore/src/sled_adapter/frame.rs | 2 +- module/move/unitore/src/sled_adapter/mod.rs | 2 + module/move/unitore/src/sled_adapter/table.rs | 2 +- module/move/unitore/src/storage.rs | 1 + module/move/unitore/src/tool/mod.rs | 2 + module/move/unitore/tests/basic.rs | 4 -- module/move/unitore/tests/config.rs | 13 ++--- module/move/unitore/tests/frames_download.rs | 13 +++-- 26 files changed, 105 insertions(+), 116 deletions(-) diff --git a/module/move/unitore/src/action/config.rs b/module/move/unitore/src/action/config.rs index 4c190fa45c..951aabc2a9 100644 --- a/module/move/unitore/src/action/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -2,7 +2,6 @@ use crate::*; use error_tools::{ err, for_app::Context, BasicError, Result }; -use executor::FeedManager; use storage::FeedStorage; use entity:: { @@ -13,7 +12,7 @@ use action::Report; use gluesql::{ prelude::Payload, sled_storage::SledStorage }; /// Add configuration file with subscriptions to storage. -pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_add( mut storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -47,12 +46,9 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args //let abs_path = proper_path_tools::path::canonicalize( path )?; let abs_path = path.canonicalize()?; - println!("{}", abs_path.to_string_lossy().to_string() ); - let config = Config::new( abs_path.to_string_lossy().to_string() ); - let mut manager = FeedManager::new( storage ); - let config_report = manager.storage + let config_report = storage .config_add( &config ) .await .context( "Added 0 config files.\n Failed to add config file to storage." )? @@ -64,13 +60,13 @@ pub async fn config_add( storage : FeedStorage< SledStorage >, args : &wca::Args .collect::< Vec< _ > >() ; - let new_feeds = manager.storage.feeds_save( feeds ).await?; + let new_feeds = storage.feeds_save( feeds ).await?; Ok( ConfigReport{ payload : config_report, new_feeds : Some( new_feeds ) } ) } /// Remove configuration file from storage. -pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_delete( mut storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > { let path : std::path::PathBuf = args .get_owned::< wca::Value >( 0 ) @@ -81,9 +77,8 @@ pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::A let path = path.canonicalize().context( format!( "Invalid path for config file {:?}", path ) )?; let config = Config::new( path.to_string_lossy().to_string() ); - let mut manager = FeedManager::new( storage ); Ok( ConfigReport::new( - manager.storage + storage .config_delete( &config ) .await .context( "Failed to remove config from storage." )? @@ -91,10 +86,9 @@ pub async fn config_delete( storage : FeedStorage< SledStorage >, args : &wca::A } /// List all files with subscriptions that are currently in storage. -pub async fn config_list( storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > +pub async fn config_list( mut storage : FeedStorage< SledStorage >, _args : &wca::Args ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - Ok( ConfigReport::new( manager.storage.config_list().await? ) ) + Ok( ConfigReport::new( storage.config_list().await? ) ) } /// Information about result of command for subscription config. diff --git a/module/move/unitore/src/action/feed.rs b/module/move/unitore/src/action/feed.rs index 6b253ef0a2..ae5aa2152d 100644 --- a/module/move/unitore/src/action/feed.rs +++ b/module/move/unitore/src/action/feed.rs @@ -1,7 +1,6 @@ //! Endpoints and report for feed commands. use crate::*; -use executor::FeedManager; use action::{ Report, frame::SelectedEntries }; use storage::FeedStorage; use entity::feed::FeedStore; @@ -10,12 +9,11 @@ use error_tools::Result; /// List all feeds from storage. pub async fn feeds_list ( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, _args : &wca::Args, ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - manager.storage.feeds_list().await + storage.feeds_list().await } const EMPTY_CELL : &'static str = ""; diff --git a/module/move/unitore/src/action/frame.rs b/module/move/unitore/src/action/frame.rs index 88a445fed7..7e2825ddd5 100644 --- a/module/move/unitore/src/action/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -1,7 +1,6 @@ //! Frames commands actions. use crate::*; -use executor::FeedManager; use storage::FeedStorage; use entity:: { @@ -19,24 +18,21 @@ use action::Report; /// List all frames. pub async fn frames_list ( - storage : FeedStorage< SledStorage >, + mut storage : FeedStorage< SledStorage >, _args : &wca::Args, ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - manager.storage.frames_list().await + storage.frames_list().await } /// Update all frames from config files saved in storage. pub async fn frames_download ( - storage : FeedStorage< SledStorage >, + mut storage : FeedStorage< SledStorage >, _args : &wca::Args, ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - let payload = manager.storage.config_list().await?; - + let payload = storage.config_list().await?; let configs = match &payload { Payload::Select { labels: _, rows: rows_vec } => @@ -76,7 +72,7 @@ pub async fn frames_download let feed = client.fetch( subscription.link.clone() ).await?; feeds.push( ( feed, subscription.update_period.clone(), subscription.link ) ); } - manager.storage.feeds_process( feeds ).await + storage.feeds_process( feeds ).await } diff --git a/module/move/unitore/src/action/query.rs b/module/move/unitore/src/action/query.rs index fb004825cb..521a1d8cf6 100644 --- a/module/move/unitore/src/action/query.rs +++ b/module/move/unitore/src/action/query.rs @@ -5,14 +5,13 @@ use crate::*; use gluesql::core::executor::Payload; use storage::{ FeedStorage, Store }; -use executor::FeedManager; use action::Report; use error_tools::{ err, BasicError, Result }; /// Execute query specified in query string. pub async fn query_execute ( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, args : &wca::Args, ) -> Result< impl Report > { @@ -22,8 +21,7 @@ pub async fn query_execute .join( " " ) ; - let mut manager = FeedManager::new( storage ); - manager.storage.execute_query( query ).await + storage.execute_query( query ).await } const EMPTY_CELL : &'static str = ""; @@ -92,3 +90,4 @@ impl Report for QueryReport {} // qqq : good tests for query action // all tables should be touched by these tests +// aaa : added in https://github.com/Wandalen/wTools/pull/1284 diff --git a/module/move/unitore/src/action/table.rs b/module/move/unitore/src/action/table.rs index 11a34c21f4..643ff1119b 100644 --- a/module/move/unitore/src/action/table.rs +++ b/module/move/unitore/src/action/table.rs @@ -4,7 +4,6 @@ use crate::*; use gluesql::prelude::Payload; use std::collections::HashMap; use action::Report; -use executor::FeedManager; use storage::FeedStorage; use entity::table::TableStore; use error_tools::Result; @@ -12,13 +11,13 @@ use error_tools::Result; /// Get labels of column for specified table. pub async fn table_list ( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, args : &wca::Args, ) -> Result< impl Report > { let table_name = args.get_owned::< String >( 0 ); - let mut manager = FeedManager::new( storage ); + // let mut manager = FeedManager::new( storage ); let mut table_names = Vec::new(); if let Some( name ) = table_name { @@ -26,7 +25,7 @@ pub async fn table_list } else { - let tables = manager.storage.tables_list().await?; + let tables = storage.tables_list().await?; let names = tables.tables.keys().map( | k | k.clone() ).collect::< Vec< _ > >(); table_names.extend( names.into_iter() ); @@ -35,7 +34,7 @@ pub async fn table_list let mut reports = Vec::new(); for table_name in table_names { - let result = manager.storage.table_list( table_name.clone() ).await?; + let result = storage.table_list( table_name.clone() ).await?; let mut table_description = String::new(); let mut columns = HashMap::new(); @@ -235,12 +234,11 @@ pub async fn table_list /// Get information about tables in storage. pub async fn tables_list ( - storage : FeedStorage< gluesql::sled_storage::SledStorage >, + mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, _args : &wca::Args, ) -> Result< impl Report > { - let mut manager = FeedManager::new( storage ); - manager.storage.tables_list().await + storage.tables_list().await } const EMPTY_CELL : &'static str = ""; diff --git a/module/move/unitore/src/command/config.rs b/module/move/unitore/src/command/config.rs index 3fef800b22..2a09ed0423 100644 --- a/module/move/unitore/src/command/config.rs +++ b/module/move/unitore/src/command/config.rs @@ -1,4 +1,4 @@ -//! Feed command. +//! Config files commands. use crate::*; use gluesql::sled_storage::sled::Config; @@ -7,13 +7,15 @@ use storage::FeedStorage; use action::{ Report, config::{ config_add, config_delete, config_list } }; use error_tools::Result; +/// Struct that provides commands for config files. +#[ derive( Debug ) ] pub struct ConfigCommand; impl ConfigCommand { + /// Create command for adding config. pub fn add() -> Result< Command > { - let rt = tokio::runtime::Runtime::new()?; Ok @@ -59,6 +61,7 @@ impl ConfigCommand ) } + /// Create command for deleting config. pub fn delete() -> Result< Command > { let rt = tokio::runtime::Runtime::new()?; @@ -98,6 +101,7 @@ impl ConfigCommand ) } + /// Create command for listing all config files in storage. pub fn list() -> Result< Command > { let rt = tokio::runtime::Runtime::new()?; diff --git a/module/move/unitore/src/command/feed.rs b/module/move/unitore/src/command/feed.rs index c713e8b559..3fc620da3b 100644 --- a/module/move/unitore/src/command/feed.rs +++ b/module/move/unitore/src/command/feed.rs @@ -7,10 +7,13 @@ use storage::FeedStorage; use action::{ Report, feed::feeds_list }; use error_tools::Result; +/// Struct that provides commands for feed. +#[ derive( Debug ) ] pub struct FeedCommand; impl FeedCommand { + /// Create command that lists all feeds in storage. pub fn list() -> Result< Command > { diff --git a/module/move/unitore/src/command/frame.rs b/module/move/unitore/src/command/frame.rs index 75416b1d71..6ed103be34 100644 --- a/module/move/unitore/src/command/frame.rs +++ b/module/move/unitore/src/command/frame.rs @@ -7,10 +7,13 @@ use storage::FeedStorage; use action::{ Report, frame::{ frames_list, frames_download } }; use error_tools::Result; +/// Struct that provides commands for frames. +#[ derive( Debug ) ] pub struct FrameCommand; impl FrameCommand { + /// Create command that lists all frames in storage. pub fn list() -> Result< Command > { let rt = tokio::runtime::Runtime::new()?; @@ -50,9 +53,9 @@ impl FrameCommand ) } + /// Creates command that downloads frames from feeds specified in config files. pub fn download() -> Result< Command > { - let rt = tokio::runtime::Runtime::new()?; Ok( diff --git a/module/move/unitore/src/command/mod.rs b/module/move/unitore/src/command/mod.rs index ea0a44e8a3..5548eb2f17 100644 --- a/module/move/unitore/src/command/mod.rs +++ b/module/move/unitore/src/command/mod.rs @@ -1,3 +1,5 @@ +//! Commands for unitore executor. + pub mod table; pub mod frame; pub mod feed; diff --git a/module/move/unitore/src/command/query.rs b/module/move/unitore/src/command/query.rs index 21931d05af..e2c4d08d3e 100644 --- a/module/move/unitore/src/command/query.rs +++ b/module/move/unitore/src/command/query.rs @@ -7,11 +7,14 @@ use storage::FeedStorage; use action::{ Report, query::query_execute }; use error_tools::Result; +/// Struct that provides commands for queries. +#[ derive( Debug ) ] pub struct QueryCommand; impl QueryCommand { - pub fn list() -> Result< Command > + /// Creates command for custom query execution. + pub fn execute() -> Result< Command > { let rt = tokio::runtime::Runtime::new()?; diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs index f40387b4e0..e250af57fa 100644 --- a/module/move/unitore/src/command/table.rs +++ b/module/move/unitore/src/command/table.rs @@ -1,4 +1,4 @@ -//! +//! Table and columns commands. use crate::*; use gluesql::sled_storage::sled::Config; @@ -7,13 +7,15 @@ use storage::FeedStorage; use action::{ Report, table::{ table_list, tables_list } }; use error_tools::Result; +/// Struct that provides commands for table information. +#[ derive( Debug ) ] pub struct TableCommand; impl TableCommand { + /// Creates command to list info about tables in storage. pub fn list() -> Result< Command > { - let rt = tokio::runtime::Runtime::new()?; Ok @@ -52,10 +54,13 @@ impl TableCommand } } +/// Struct that provides commands for table columns information. +#[ derive( Debug ) ] pub struct TablesCommand; impl TablesCommand { + /// Creates command to list info about table columns in storage. pub fn list() -> Result< Command > { diff --git a/module/move/unitore/src/entity/frame.rs b/module/move/unitore/src/entity/frame.rs index de687ec425..8fb522ad58 100644 --- a/module/move/unitore/src/entity/frame.rs +++ b/module/move/unitore/src/entity/frame.rs @@ -16,17 +16,29 @@ pub struct Frame pub id : String, /// Frame title. pub title : Option< String >, + /// Time at which this item was fetched from source. pub stored_time : Option< DateTime< Utc > >, + /// List of authors of the frame. pub authors : Option< Vec< String > >, + /// The content of the frame in html or plain text. pub content : Option< String >, + /// List of links associated with this item of related Web page and attachments. pub links : Option< Vec< String > >, + /// Short summary, abstract, or excerpt of the frame item. pub summary : Option< String >, + /// A list of categories that the item belongs to. pub categories : Option< Vec< String > >, + /// Time at which this item was first published or updated. pub published : Option< DateTime< Utc > >, + /// Specifies the source feed if the frame was copied from one feed into another feed. pub source : Option< String >, + /// Information about copyrights over the feed. pub rights : Option< String >, + /// List of media oblects, encountered in the frame. pub media : Option< Vec< String > >, + /// The language of the frame. pub language : Option< String >, + /// Link to feed that contains this frame. pub feed_link : String, } diff --git a/module/move/unitore/src/executor.rs b/module/move/unitore/src/executor.rs index 94cf1093eb..8010dbd9cc 100644 --- a/module/move/unitore/src/executor.rs +++ b/module/move/unitore/src/executor.rs @@ -1,14 +1,9 @@ //! Execute plan. use crate::*; -use feed_config::SubscriptionConfig; -use storage::Store; -use entity::{ feed::FeedStore, config::ConfigStore, table::TableStore, frame::FrameStore }; use wca::{ Dictionary, Executor, Parser, Verifier }; use error_tools::Result; -use action::Report; - /// Run feed updates. pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > { @@ -51,7 +46,7 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > ) .command ( - command::query::QueryCommand::list()? + command::query::QueryCommand::execute()? ) .form(); let verifier = Verifier; @@ -67,49 +62,3 @@ pub fn execute() -> Result< (), Box< dyn std::error::Error + Send + Sync > > Ok( () ) } - -/// Manages feed subsriptions and updates. -pub struct FeedManager< S : FeedStore + ConfigStore + FrameStore + Store + Send > -{ - /// Subscription configuration with link and update period. - pub config : Vec< SubscriptionConfig >, - /// Storage for saving feed. - pub storage : S, -} - -impl< S : FeedStore + ConfigStore + FrameStore + Store + Send > std::fmt::Debug for FeedManager< S > -{ - fn fmt( &self, f: &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - writeln!(f, "Feed manager with storage and client" ) - } -} - -impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< S > -{ - /// Create new instance of FeedManager. - pub fn new( storage : S ) -> FeedManager< S > - { - Self - { - storage, - config : Vec::new(), - } - } -} - -impl< S : FeedStore + ConfigStore + FrameStore + TableStore + Store + Send > FeedManager< S > -{ - /// Set configurations for subscriptions. - pub fn set_config( &mut self, configs : Vec< SubscriptionConfig > ) - { - self.config = configs; - } - - /// Execute custom query, print result. - pub async fn execute_custom_query( &mut self, query : String ) -> Result< impl Report > - { - self.storage.execute_query( query ).await - } - -} diff --git a/module/move/unitore/src/feed_config.rs b/module/move/unitore/src/feed_config.rs index 71d852731f..88b5f8a791 100644 --- a/module/move/unitore/src/feed_config.rs +++ b/module/move/unitore/src/feed_config.rs @@ -23,7 +23,15 @@ pub struct Subscriptions pub config : Vec< SubscriptionConfig > } -/// Reads provided configuration file with list of subscriptions. +/// Get list of feed subscriptions from provided configuration file. +/// +/// # Arguments +/// +/// * `file_path` - Path to the configuration file. +/// +/// # Returns +/// +/// Result with list of feed subscriptions serialized as SubscriptionConfig. pub fn read( file_path : String ) -> Result< Vec< SubscriptionConfig > > { let read_file = OpenOptions::new() diff --git a/module/move/unitore/src/lib.rs b/module/move/unitore/src/lib.rs index a6b66a5716..88f8adc251 100644 --- a/module/move/unitore/src/lib.rs +++ b/module/move/unitore/src/lib.rs @@ -10,3 +10,4 @@ pub mod entity; pub mod sled_adapter; // qqq : src/Readmу.md with file structure please +// aaa : added Readme.md diff --git a/module/move/unitore/src/retriever.rs b/module/move/unitore/src/retriever.rs index 79eea6b407..ac4e94be11 100644 --- a/module/move/unitore/src/retriever.rs +++ b/module/move/unitore/src/retriever.rs @@ -20,6 +20,15 @@ pub struct FeedClient; impl FeedClient { + /// Fetch feed frames from provided url source. + /// + /// # Arguments + /// + /// * `source` - The link to feed source. + /// + /// # Returns + /// + /// Result with fetched feed as feed_rs Feed struct. pub async fn fetch( &self, source : url::Url ) -> Result< feed_rs::model::Feed > { let https = HttpsConnector::new(); diff --git a/module/move/unitore/src/sled_adapter/config.rs b/module/move/unitore/src/sled_adapter/config.rs index 0691eb7d7f..60f16a26a4 100644 --- a/module/move/unitore/src/sled_adapter/config.rs +++ b/module/move/unitore/src/sled_adapter/config.rs @@ -1,4 +1,4 @@ -// Config file operation with Sled storage. +//! Config file operation with Sled storage. use crate::*; use error_tools::{ err, Result }; diff --git a/module/move/unitore/src/sled_adapter/feed.rs b/module/move/unitore/src/sled_adapter/feed.rs index cabdeddd23..cfa73fdb42 100644 --- a/module/move/unitore/src/sled_adapter/feed.rs +++ b/module/move/unitore/src/sled_adapter/feed.rs @@ -1,4 +1,4 @@ -// Feed operation with Sled storage. +//! Feed operation with Sled storage. use crate::*; use std::time::Duration; diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs index 325f7f1563..5fe7dd1dbd 100644 --- a/module/move/unitore/src/sled_adapter/frame.rs +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -1,4 +1,4 @@ -// Frames operation with Sled storage. +//! Frames operation with Sled storage. use crate::*; use std::collections::HashMap; diff --git a/module/move/unitore/src/sled_adapter/mod.rs b/module/move/unitore/src/sled_adapter/mod.rs index ea5e050ec3..b17aa61c78 100644 --- a/module/move/unitore/src/sled_adapter/mod.rs +++ b/module/move/unitore/src/sled_adapter/mod.rs @@ -1,3 +1,5 @@ +//! Operations for using entities with Sled storage. + mod frame; mod table; mod feed; diff --git a/module/move/unitore/src/sled_adapter/table.rs b/module/move/unitore/src/sled_adapter/table.rs index b19b26a5bd..1c32dda965 100644 --- a/module/move/unitore/src/sled_adapter/table.rs +++ b/module/move/unitore/src/sled_adapter/table.rs @@ -1,4 +1,4 @@ -// Table and columns info operations from Sled storage. +//! Table and columns info operations from Sled storage. use crate::*; use error_tools::Result; diff --git a/module/move/unitore/src/storage.rs b/module/move/unitore/src/storage.rs index 98c0fdd72b..02d49d0364 100644 --- a/module/move/unitore/src/storage.rs +++ b/module/move/unitore/src/storage.rs @@ -22,6 +22,7 @@ pub struct FeedStorage< S : GStore + GStoreMut + Send > { /// GlueSQL storage. pub storage : Arc< Mutex< Glue< S > > >, + /// Frame table fields with description. pub frame_fields : Vec< [ &'static str; 3 ] >, } diff --git a/module/move/unitore/src/tool/mod.rs b/module/move/unitore/src/tool/mod.rs index 200a9b43be..749ba59a0e 100644 --- a/module/move/unitore/src/tool/mod.rs +++ b/module/move/unitore/src/tool/mod.rs @@ -1 +1,3 @@ +//! Tools for additional functionality. + pub mod table_display; \ No newline at end of file diff --git a/module/move/unitore/tests/basic.rs b/module/move/unitore/tests/basic.rs index a5d7b510e1..6e5df1ad4d 100644 --- a/module/move/unitore/tests/basic.rs +++ b/module/move/unitore/tests/basic.rs @@ -5,13 +5,9 @@ use error_tools::Result; async fn frame() -> Result< () > { let feed = feed_parser::parse( include_str!( "./fixtures/plain_feed.xml" ).as_bytes() )?; - let frame = unitore::entity::frame::Frame::from( ( feed.entries[ 0 ].clone(), String::new() ) ); - assert!( frame.id == feed.entries[ 0 ].id ); - println!( "{:#?}", feed.entries[ 0 ].media ); - println!( "{:#?}", frame ); Ok( () ) } diff --git a/module/move/unitore/tests/config.rs b/module/move/unitore/tests/config.rs index a30448611b..060078ac4e 100644 --- a/module/move/unitore/tests/config.rs +++ b/module/move/unitore/tests/config.rs @@ -1,13 +1,11 @@ use std::path::PathBuf; - -use gluesql::{ +use gluesql:: +{ sled_storage::sled::Config, - test_suite::data_type::list, prelude::Payload::Select, }; use unitore:: { - executor::FeedManager, storage::FeedStorage, entity::{ feed::FeedStore, config::ConfigStore }, action::config, @@ -24,11 +22,10 @@ async fn config_add() -> Result< () > .temporary( true ) ; - let feed_storage = FeedStorage::init_storage( &config ).await?; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; - let mut manager = FeedManager::new( feed_storage ); - let res = manager.storage.feeds_list().await?; + let res = feed_storage.feeds_list().await?; let feeds_links = res.0.selected_rows .iter() @@ -60,7 +57,7 @@ async fn config_delete() -> Result< () > let list = feed_storage.config_list().await?; - if let Select{ labels, rows } = list + if let Select{ labels : _, rows } = list { assert!( rows.len() == 0 ) } diff --git a/module/move/unitore/tests/frames_download.rs b/module/move/unitore/tests/frames_download.rs index e97265210a..9922a8c3f5 100644 --- a/module/move/unitore/tests/frames_download.rs +++ b/module/move/unitore/tests/frames_download.rs @@ -80,7 +80,12 @@ async fn test_update() -> Result< () > // check let payload = feed_storage.frames_list().await?; - let entries = payload.0.iter().map( | val | val.selected_frames.selected_rows.clone() ).flatten().collect::< Vec< _ > >(); + let entries = payload.0 + .iter() + .map( | val | val.selected_frames.selected_rows.clone() ) + .flatten() + .collect::< Vec< _ > >() + ; let entries = entries.iter().map( | entry | { @@ -105,8 +110,10 @@ async fn test_update() -> Result< () > assert_eq!( entries.len(), 10 ); // check date - println!( "{:?}", entries ); - let updated = entries.iter().find( | ( id, _published ) | id == "https://www.nasa.gov/?post_type=image-article&p=631537" ); + let updated = entries.iter().find + ( + | ( id, _published ) | id == "https://www.nasa.gov/?post_type=image-article&p=631537" + ); assert!( updated.is_some() ); let _updated = updated.unwrap(); Ok( () ) From b625f862e0e61e442c936c73e74a38e0c3e04a24 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 14:22:54 +0300 Subject: [PATCH 10/11] fix adapter --- module/move/unitore/src/action/config.rs | 2 +- module/move/unitore/src/action/feed.rs | 2 +- module/move/unitore/src/action/frame.rs | 2 +- module/move/unitore/src/action/query.rs | 2 +- module/move/unitore/src/action/table.rs | 2 +- module/move/unitore/src/command/config.rs | 2 +- module/move/unitore/src/command/feed.rs | 2 +- module/move/unitore/src/command/frame.rs | 2 +- module/move/unitore/src/command/query.rs | 2 +- module/move/unitore/src/command/table.rs | 2 +- module/move/unitore/src/lib.rs | 1 - .../move/unitore/src/sled_adapter/config.rs | 3 +- module/move/unitore/src/sled_adapter/feed.rs | 2 +- module/move/unitore/src/sled_adapter/frame.rs | 2 +- module/move/unitore/src/sled_adapter/mod.rs | 130 +++++++++++++++++- module/move/unitore/src/sled_adapter/table.rs | 2 +- module/move/unitore/src/storage.rs | 127 ----------------- module/move/unitore/tests/config.rs | 2 +- module/move/unitore/tests/frames_download.rs | 5 +- 19 files changed, 145 insertions(+), 149 deletions(-) delete mode 100644 module/move/unitore/src/storage.rs diff --git a/module/move/unitore/src/action/config.rs b/module/move/unitore/src/action/config.rs index 951aabc2a9..5fd8dd84a9 100644 --- a/module/move/unitore/src/action/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -2,7 +2,7 @@ use crate::*; use error_tools::{ err, for_app::Context, BasicError, Result }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use entity:: { feed::{ FeedStore, Feed }, diff --git a/module/move/unitore/src/action/feed.rs b/module/move/unitore/src/action/feed.rs index ae5aa2152d..a2eed760e9 100644 --- a/module/move/unitore/src/action/feed.rs +++ b/module/move/unitore/src/action/feed.rs @@ -2,7 +2,7 @@ use crate::*; use action::{ Report, frame::SelectedEntries }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use entity::feed::FeedStore; use error_tools::Result; diff --git a/module/move/unitore/src/action/frame.rs b/module/move/unitore/src/action/frame.rs index 7e2825ddd5..1374988347 100644 --- a/module/move/unitore/src/action/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -1,7 +1,7 @@ //! Frames commands actions. use crate::*; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use entity:: { feed::FeedStore, diff --git a/module/move/unitore/src/action/query.rs b/module/move/unitore/src/action/query.rs index 521a1d8cf6..c0ae448e0b 100644 --- a/module/move/unitore/src/action/query.rs +++ b/module/move/unitore/src/action/query.rs @@ -4,7 +4,7 @@ // aaa : fixed use crate::*; use gluesql::core::executor::Payload; -use storage::{ FeedStorage, Store }; +use sled_adapter::{ FeedStorage, Store }; use action::Report; use error_tools::{ err, BasicError, Result }; diff --git a/module/move/unitore/src/action/table.rs b/module/move/unitore/src/action/table.rs index 643ff1119b..7fecad6115 100644 --- a/module/move/unitore/src/action/table.rs +++ b/module/move/unitore/src/action/table.rs @@ -4,7 +4,7 @@ use crate::*; use gluesql::prelude::Payload; use std::collections::HashMap; use action::Report; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use entity::table::TableStore; use error_tools::Result; diff --git a/module/move/unitore/src/command/config.rs b/module/move/unitore/src/command/config.rs index 2a09ed0423..37bbfb7aaf 100644 --- a/module/move/unitore/src/command/config.rs +++ b/module/move/unitore/src/command/config.rs @@ -3,7 +3,7 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, Type, VerifiedCommand }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use action::{ Report, config::{ config_add, config_delete, config_list } }; use error_tools::Result; diff --git a/module/move/unitore/src/command/feed.rs b/module/move/unitore/src/command/feed.rs index 3fc620da3b..9255dbf3a4 100644 --- a/module/move/unitore/src/command/feed.rs +++ b/module/move/unitore/src/command/feed.rs @@ -3,7 +3,7 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, VerifiedCommand }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use action::{ Report, feed::feeds_list }; use error_tools::Result; diff --git a/module/move/unitore/src/command/frame.rs b/module/move/unitore/src/command/frame.rs index 6ed103be34..bdfe0d1174 100644 --- a/module/move/unitore/src/command/frame.rs +++ b/module/move/unitore/src/command/frame.rs @@ -3,7 +3,7 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, VerifiedCommand }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use action::{ Report, frame::{ frames_list, frames_download } }; use error_tools::Result; diff --git a/module/move/unitore/src/command/query.rs b/module/move/unitore/src/command/query.rs index e2c4d08d3e..b9ebbd67c5 100644 --- a/module/move/unitore/src/command/query.rs +++ b/module/move/unitore/src/command/query.rs @@ -3,7 +3,7 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, Type, VerifiedCommand }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use action::{ Report, query::query_execute }; use error_tools::Result; diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs index e250af57fa..34db044668 100644 --- a/module/move/unitore/src/command/table.rs +++ b/module/move/unitore/src/command/table.rs @@ -3,7 +3,7 @@ use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, Type, VerifiedCommand }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use action::{ Report, table::{ table_list, tables_list } }; use error_tools::Result; diff --git a/module/move/unitore/src/lib.rs b/module/move/unitore/src/lib.rs index 88f8adc251..f6e0df9632 100644 --- a/module/move/unitore/src/lib.rs +++ b/module/move/unitore/src/lib.rs @@ -2,7 +2,6 @@ pub mod retriever; pub mod feed_config; pub mod executor; -pub mod storage; pub mod tool; pub mod command; pub mod action; diff --git a/module/move/unitore/src/sled_adapter/config.rs b/module/move/unitore/src/sled_adapter/config.rs index 60f16a26a4..35ad1ae3cd 100644 --- a/module/move/unitore/src/sled_adapter/config.rs +++ b/module/move/unitore/src/sled_adapter/config.rs @@ -12,9 +12,10 @@ use gluesql:: sled_storage::SledStorage, }; use entity::config::{ Config, ConfigStore }; +use sled_adapter::FeedStorage; #[ async_trait::async_trait( ?Send ) ] -impl ConfigStore for storage::FeedStorage< SledStorage > +impl ConfigStore for FeedStorage< SledStorage > { async fn config_add( &mut self, config : &Config ) -> Result< Payload > { diff --git a/module/move/unitore/src/sled_adapter/feed.rs b/module/move/unitore/src/sled_adapter/feed.rs index cfa73fdb42..2659657103 100644 --- a/module/move/unitore/src/sled_adapter/feed.rs +++ b/module/move/unitore/src/sled_adapter/feed.rs @@ -24,7 +24,7 @@ use action:: feed::FeedsReport, frame::{ UpdateReport, SelectedEntries, FramesReport }, }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use wca::wtools::Itertools; #[ async_trait::async_trait( ?Send ) ] diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs index 5fe7dd1dbd..666db0921f 100644 --- a/module/move/unitore/src/sled_adapter/frame.rs +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -15,7 +15,7 @@ use gluesql:: }; use entity::frame::{ FrameStore, Frame }; use action::frame::{ SelectedEntries, FramesReport, ListReport }; -use storage::FeedStorage; +use sled_adapter::FeedStorage; use wca::wtools::Itertools; #[ async_trait::async_trait( ?Send ) ] diff --git a/module/move/unitore/src/sled_adapter/mod.rs b/module/move/unitore/src/sled_adapter/mod.rs index b17aa61c78..ff73740de2 100644 --- a/module/move/unitore/src/sled_adapter/mod.rs +++ b/module/move/unitore/src/sled_adapter/mod.rs @@ -1,6 +1,132 @@ -//! Operations for using entities with Sled storage. +//! Storage for frames, feeds and config files. + +use crate::*; +use std::sync::Arc; +use error_tools::{ for_app::Context, Result }; +use tokio::sync::Mutex; +use gluesql:: +{ + core:: + { + ast_builder::{ table, Build, Execute }, + store::{ GStore, GStoreMut }, + }, + prelude::Glue, + sled_storage::{ sled::Config, SledStorage }, +}; +use action::query::QueryReport; mod frame; mod table; mod feed; -mod config; \ No newline at end of file +mod config; + +/// Storage for feed frames. +#[ derive( Clone ) ] +pub struct FeedStorage< S : GStore + GStoreMut + Send > +{ + /// GlueSQL storage. + pub storage : Arc< Mutex< Glue< S > > >, + /// Frame table fields with description. + pub frame_fields : Vec< [ &'static str; 3 ] >, +} + +impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > +{ + fn fmt( &self, f: &mut std::fmt::Formatter<'_> ) -> std::fmt::Result + { + writeln!(f, "GlueSQL storage" ) + } +} + +impl FeedStorage< SledStorage > +{ + /// Initialize new storage from configuration, create feed table. + pub async fn init_storage( config : &Config ) -> Result< Self > + { + let storage = SledStorage::try_from( config.clone() ) + .context( format!( "Failed to initialize storage with config {:?}", config ) )? + ; + + let mut glue = Glue::new( storage ); + + let sub_table = table( "config" ) + .create_table_if_not_exists() + .add_column( "path TEXT PRIMARY KEY" ) + .build()? + ; + + sub_table.execute( &mut glue ).await?; + + let feed_table = table( "feed" ) + .create_table_if_not_exists() + .add_column( "link TEXT PRIMARY KEY" ) + .add_column( "type TEXT" ) + .add_column( "title TEXT" ) + .add_column( "updated TIMESTAMP" ) + .add_column( "authors TEXT" ) + .add_column( "description TEXT" ) + .add_column( "published TIMESTAMP" ) + .add_column( "update_period TEXT" ) + .add_column( "config_file TEXT FOREIGN KEY REFERENCES config(path)" ) + .build()? + ; + + feed_table.execute( &mut glue ).await?; + + let frame_fields = vec! + [ + [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], + [ "title", "TEXT", "Title of the frame" ], + [ "stored_time", "TIMESTAMP", "Time at which this item was fetched from source." ], + [ "authors", "LIST", "List of authors of the frame, optional." ], + [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], + [ "links", "LIST", "List of links associated with this item of related Web page and attachments." ], + [ "summary", "TEXT", "Short summary, abstract, or excerpt of the frame item, optional." ], + [ "categories", "LIST", "Specifies a list of categories that the item belongs to." ], + [ "published", "TIMESTAMP", "Time at which this item was first published or updated." ], + [ "source", "TEXT", "Specifies the source feed if the frame was copied from one feed into another feed, optional." ], + [ "rights", "TEXT", "Conveys information about copyrights over the feed, optional." ], + [ "media", "LIST", "List of media oblects, encountered in the frame, optional." ], + [ "language", "TEXT", "The language specified on the item, optional." ], + [ "feed_link", "TEXT", "Link of feed that contains this frame." ], + ]; + let mut table = table( "frame" ).create_table_if_not_exists().add_column( "id TEXT PRIMARY KEY" ); + + for column in frame_fields.iter().skip( 1 ).take( frame_fields.len() - 2 ) + { + table = table.add_column( format!( "{} {}", column[ 0 ], column[ 1 ] ).as_str() ); + } + + let table = table.add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) + .build()? + ; + + table.execute( &mut glue ).await?; + + Ok( Self{ storage : Arc::new( Mutex::new( glue ) ), frame_fields } ) + } +} + +/// Functionality of feed storage. +#[ mockall::automock ] +#[ async_trait::async_trait( ?Send ) ] +pub trait Store +{ + /// Execute custom query passed as String. + async fn execute_query( &mut self, query : String ) -> Result< QueryReport >; +} + +#[ async_trait::async_trait( ?Send ) ] +impl< S : GStore + GStoreMut + Send > Store for FeedStorage< S > +{ + async fn execute_query( &mut self, query : String ) -> Result< QueryReport > + { + let glue = &mut *self.storage.lock().await; + let payloads = glue.execute( &query ).await.context( "Failed to execute query" )?; + + let report = QueryReport ( payloads ); + + Ok( report ) + } +} diff --git a/module/move/unitore/src/sled_adapter/table.rs b/module/move/unitore/src/sled_adapter/table.rs index 1c32dda965..ddf0664bfc 100644 --- a/module/move/unitore/src/sled_adapter/table.rs +++ b/module/move/unitore/src/sled_adapter/table.rs @@ -9,7 +9,7 @@ use gluesql:: }; use entity::table::TableStore; use action::table::TablesReport; -use storage::FeedStorage; +use sled_adapter::FeedStorage; #[ async_trait::async_trait( ?Send ) ] impl TableStore for FeedStorage< SledStorage > diff --git a/module/move/unitore/src/storage.rs b/module/move/unitore/src/storage.rs deleted file mode 100644 index 02d49d0364..0000000000 --- a/module/move/unitore/src/storage.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Storage for frames, feeds and config files. - -use crate::*; -use std::sync::Arc; -use error_tools::{ for_app::Context, Result }; -use tokio::sync::Mutex; -use gluesql:: -{ - core:: - { - ast_builder::{ table, Build, Execute }, - store::{ GStore, GStoreMut }, - }, - prelude::Glue, - sled_storage::{ sled::Config, SledStorage }, -}; -use action::query::QueryReport; - -/// Storage for feed frames. -#[ derive( Clone ) ] -pub struct FeedStorage< S : GStore + GStoreMut + Send > -{ - /// GlueSQL storage. - pub storage : Arc< Mutex< Glue< S > > >, - /// Frame table fields with description. - pub frame_fields : Vec< [ &'static str; 3 ] >, -} - -impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > -{ - fn fmt( &self, f: &mut std::fmt::Formatter<'_> ) -> std::fmt::Result - { - writeln!(f, "GlueSQL storage" ) - } -} - -impl FeedStorage< SledStorage > -{ - /// Initialize new storage from configuration, create feed table. - pub async fn init_storage( config : &Config ) -> Result< Self > - { - let storage = SledStorage::try_from( config.clone() ) - .context( format!( "Failed to initialize storage with config {:?}", config ) )? - ; - - let mut glue = Glue::new( storage ); - - let sub_table = table( "config" ) - .create_table_if_not_exists() - .add_column( "path TEXT PRIMARY KEY" ) - .build()? - ; - - sub_table.execute( &mut glue ).await?; - - let feed_table = table( "feed" ) - .create_table_if_not_exists() - .add_column( "link TEXT PRIMARY KEY" ) - .add_column( "type TEXT" ) - .add_column( "title TEXT" ) - .add_column( "updated TIMESTAMP" ) - .add_column( "authors TEXT" ) - .add_column( "description TEXT" ) - .add_column( "published TIMESTAMP" ) - .add_column( "update_period TEXT" ) - .add_column( "config_file TEXT FOREIGN KEY REFERENCES config(path)" ) - .build()? - ; - - feed_table.execute( &mut glue ).await?; - - let frame_fields = vec! - [ - [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], - [ "title", "TEXT", "Title of the frame" ], - [ "stored_time", "TIMESTAMP", "Time at which this item was fetched from source." ], - [ "authors", "LIST", "List of authors of the frame, optional." ], - [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], - [ "links", "LIST", "List of links associated with this item of related Web page and attachments." ], - [ "summary", "TEXT", "Short summary, abstract, or excerpt of the frame item, optional." ], - [ "categories", "LIST", "Specifies a list of categories that the item belongs to." ], - [ "published", "TIMESTAMP", "Time at which this item was first published or updated." ], - [ "source", "TEXT", "Specifies the source feed if the frame was copied from one feed into another feed, optional." ], - [ "rights", "TEXT", "Conveys information about copyrights over the feed, optional." ], - [ "media", "LIST", "List of media oblects, encountered in the frame, optional." ], - [ "language", "TEXT", "The language specified on the item, optional." ], - [ "feed_link", "TEXT", "Link of feed that contains this frame." ], - ]; - let mut table = table( "frame" ).create_table_if_not_exists().add_column( "id TEXT PRIMARY KEY" ); - - for column in frame_fields.iter().skip( 1 ).take( frame_fields.len() - 2 ) - { - table = table.add_column( format!( "{} {}", column[ 0 ], column[ 1 ] ).as_str() ); - } - - let table = table.add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) - .build()? - ; - - table.execute( &mut glue ).await?; - - Ok( Self{ storage : Arc::new( Mutex::new( glue ) ), frame_fields } ) - } -} - -/// Functionality of feed storage. -#[ mockall::automock ] -#[ async_trait::async_trait( ?Send ) ] -pub trait Store -{ - /// Execute custom query passed as String. - async fn execute_query( &mut self, query : String ) -> Result< QueryReport >; -} - -#[ async_trait::async_trait( ?Send ) ] -impl< S : GStore + GStoreMut + Send > Store for FeedStorage< S > -{ - async fn execute_query( &mut self, query : String ) -> Result< QueryReport > - { - let glue = &mut *self.storage.lock().await; - let payloads = glue.execute( &query ).await.context( "Failed to execute query" )?; - - let report = QueryReport ( payloads ); - - Ok( report ) - } -} diff --git a/module/move/unitore/tests/config.rs b/module/move/unitore/tests/config.rs index 060078ac4e..d56c4fdfdc 100644 --- a/module/move/unitore/tests/config.rs +++ b/module/move/unitore/tests/config.rs @@ -6,7 +6,7 @@ use gluesql:: }; use unitore:: { - storage::FeedStorage, + sled_adapter::FeedStorage, entity::{ feed::FeedStore, config::ConfigStore }, action::config, }; diff --git a/module/move/unitore/tests/frames_download.rs b/module/move/unitore/tests/frames_download.rs index 9922a8c3f5..ad5c3c2ff6 100644 --- a/module/move/unitore/tests/frames_download.rs +++ b/module/move/unitore/tests/frames_download.rs @@ -12,7 +12,7 @@ use wca::wtools::Itertools; use unitore:: { feed_config::SubscriptionConfig, - storage::FeedStorage, + sled_adapter::FeedStorage, entity::{ frame::FrameStore, feed::FeedStore }, }; use error_tools::Result; @@ -42,9 +42,6 @@ async fn test_save() -> Result< () > let entries = feed_storage.frames_list().await?; let number_of_frames = entries.0[ 0 ].selected_frames.selected_rows.len(); - - println!("{:#?}", entries); - assert_eq!( number_of_frames, 10 ); Ok( () ) From 530d27f98cfadf081366220caf48eebef573fbd9 Mon Sep 17 00:00:00 2001 From: YuliaProkopovych Date: Tue, 9 Apr 2024 16:53:03 +0300 Subject: [PATCH 11/11] fix commands --- module/move/unitore/Readme.md | 2 +- module/move/unitore/src/action/config.rs | 23 ++--- module/move/unitore/src/action/feed.rs | 8 +- module/move/unitore/src/action/frame.rs | 11 +-- module/move/unitore/src/action/query.rs | 14 +-- module/move/unitore/src/action/table.rs | 46 ++++------ module/move/unitore/src/command/config.rs | 86 ++++++++++++------- module/move/unitore/src/command/feed.rs | 5 +- module/move/unitore/src/command/frame.rs | 8 +- module/move/unitore/src/command/query.rs | 41 +++++---- module/move/unitore/src/command/table.rs | 8 +- module/move/unitore/src/sled_adapter/frame.rs | 15 +++- module/move/unitore/src/sled_adapter/mod.rs | 51 +++++------ module/move/unitore/tests/config.rs | 70 --------------- module/move/unitore/tests/config_add.rs | 37 ++++++++ module/move/unitore/tests/config_delete.rs | 42 +++++++++ module/move/unitore/tests/table_list.rs | 45 ++++++++++ module/move/unitore/tests/tables_list.rs | 32 +++++++ 18 files changed, 316 insertions(+), 228 deletions(-) delete mode 100644 module/move/unitore/tests/config.rs create mode 100644 module/move/unitore/tests/config_add.rs create mode 100644 module/move/unitore/tests/config_delete.rs create mode 100644 module/move/unitore/tests/table_list.rs create mode 100644 module/move/unitore/tests/tables_list.rs diff --git a/module/move/unitore/Readme.md b/module/move/unitore/Readme.md index 3513ec3729..e631beb82b 100644 --- a/module/move/unitore/Readme.md +++ b/module/move/unitore/Readme.md @@ -47,7 +47,7 @@ cargo run .feeds.list ``` To get custom information about feeds or frames run SQL query to storage database using command `.query.execute` with query string: ```bash -cargo run .query.execute \'SELECT title, links, MIN\(published\) FROM frame\' +cargo run .query.execute 'SELECT title, links, MIN(published) FROM frame' ``` To remove config file from storage use command `.config.delete` with path to config file: ```bash diff --git a/module/move/unitore/src/action/config.rs b/module/move/unitore/src/action/config.rs index 5fd8dd84a9..49b63a4773 100644 --- a/module/move/unitore/src/action/config.rs +++ b/module/move/unitore/src/action/config.rs @@ -1,7 +1,9 @@ -//! Actions and report for commands for config files. +//! Actions and report for config files. + +use std::path::PathBuf; use crate::*; -use error_tools::{ err, for_app::Context, BasicError, Result }; +use error_tools::{ for_app::Context, Result }; use sled_adapter::FeedStorage; use entity:: { @@ -12,14 +14,8 @@ use action::Report; use gluesql::{ prelude::Payload, sled_storage::SledStorage }; /// Add configuration file with subscriptions to storage. -pub async fn config_add( mut storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_add( mut storage : FeedStorage< SledStorage >, path : &PathBuf ) -> Result< impl Report > { - let path : std::path::PathBuf = args - .get_owned::< wca::Value >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get path argument for command .config.add" ) )? - .into() - ; - let path = proper_path_tools::path::normalize( path ); let mut err_str = format!( "Invalid path for config file {:?}", path ); @@ -66,14 +62,9 @@ pub async fn config_add( mut storage : FeedStorage< SledStorage >, args : &wca:: } /// Remove configuration file from storage. -pub async fn config_delete( mut storage : FeedStorage< SledStorage >, args : &wca::Args ) -> Result< impl Report > +pub async fn config_delete( mut storage : FeedStorage< SledStorage >, path : &PathBuf ) -> Result< impl Report > { - let path : std::path::PathBuf = args - .get_owned::< wca::Value >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get path argument for command .config.delete" ) )? - .into() - ; - + let path = proper_path_tools::path::normalize( path ); let path = path.canonicalize().context( format!( "Invalid path for config file {:?}", path ) )?; let config = Config::new( path.to_string_lossy().to_string() ); diff --git a/module/move/unitore/src/action/feed.rs b/module/move/unitore/src/action/feed.rs index a2eed760e9..f7840a5f55 100644 --- a/module/move/unitore/src/action/feed.rs +++ b/module/move/unitore/src/action/feed.rs @@ -1,4 +1,4 @@ -//! Endpoints and report for feed commands. +//! Feed actions and reports. use crate::*; use action::{ Report, frame::SelectedEntries }; @@ -7,11 +7,7 @@ use entity::feed::FeedStore; use error_tools::Result; /// List all feeds from storage. -pub async fn feeds_list -( - mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > +pub async fn feeds_list( mut storage : FeedStorage< gluesql::sled_storage::SledStorage > ) -> Result< impl Report > { storage.feeds_list().await } diff --git a/module/move/unitore/src/action/frame.rs b/module/move/unitore/src/action/frame.rs index 1374988347..f26d538e20 100644 --- a/module/move/unitore/src/action/frame.rs +++ b/module/move/unitore/src/action/frame.rs @@ -1,4 +1,4 @@ -//! Frames commands actions. +//! Frames actions and reports. use crate::*; use sled_adapter::FeedStorage; @@ -16,11 +16,7 @@ use action::Report; // qqq : review the whole project and make sure all names are consitant: actions, commands, its tests /// List all frames. -pub async fn frames_list -( - mut storage : FeedStorage< SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > +pub async fn frames_list( mut storage : FeedStorage< SledStorage > ) -> Result< impl Report > { storage.frames_list().await } @@ -28,8 +24,7 @@ pub async fn frames_list /// Update all frames from config files saved in storage. pub async fn frames_download ( - mut storage : FeedStorage< SledStorage >, - _args : &wca::Args, + mut storage : FeedStorage< SledStorage > ) -> Result< impl Report > { let payload = storage.config_list().await?; diff --git a/module/move/unitore/src/action/query.rs b/module/move/unitore/src/action/query.rs index c0ae448e0b..0f5edd06a7 100644 --- a/module/move/unitore/src/action/query.rs +++ b/module/move/unitore/src/action/query.rs @@ -1,4 +1,4 @@ -//! Query command endpoint and report. +//! Query actions and report. // qqq : don't use both // aaa : fixed @@ -6,22 +6,16 @@ use crate::*; use gluesql::core::executor::Payload; use sled_adapter::{ FeedStorage, Store }; use action::Report; -use error_tools::{ err, BasicError, Result }; +use error_tools::Result; /// Execute query specified in query string. pub async fn query_execute ( mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, - args : &wca::Args, + query_str : String, ) -> Result< impl Report > { - let query = args - .get_owned::< Vec::< String > >( 0 ) - .ok_or_else::< BasicError, _ >( || err!( "Cannot get Query argument for command .query.execute" ) )? - .join( " " ) - ; - - storage.execute_query( query ).await + storage.execute_query( query_str ).await } const EMPTY_CELL : &'static str = ""; diff --git a/module/move/unitore/src/action/table.rs b/module/move/unitore/src/action/table.rs index 7fecad6115..5e0c92663b 100644 --- a/module/move/unitore/src/action/table.rs +++ b/module/move/unitore/src/action/table.rs @@ -1,4 +1,4 @@ -//! Tables metadata commands actions and reports. +//! Tables metadata actions and reports. use crate::*; use gluesql::prelude::Payload; @@ -12,12 +12,9 @@ use error_tools::Result; pub async fn table_list ( mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, - args : &wca::Args, + table_name : Option< String >, ) -> Result< impl Report > { - let table_name = args.get_owned::< String >( 0 ); - - // let mut manager = FeedManager::new( storage ); let mut table_names = Vec::new(); if let Some( name ) = table_name { @@ -27,7 +24,7 @@ pub async fn table_list { let tables = storage.tables_list().await?; - let names = tables.tables.keys().map( | k | k.clone() ).collect::< Vec< _ > >(); + let names = tables.0.keys().map( | k | k.clone() ).collect::< Vec< _ > >(); table_names.extend( names.into_iter() ); } @@ -123,21 +120,24 @@ pub async fn table_list }, "authors" => { - columns_desc.insert( + columns_desc.insert + ( label.clone(), String::from( "List of authors of the frame, optional." ) ); }, "content" => { - columns_desc.insert( + columns_desc.insert + ( label.clone(), String::from( "The content of the frame in html or plain text, optional." ), ); }, "links" => { - columns_desc.insert( + columns_desc.insert + ( label.clone(), String::from( "List of links associated with this item of related Web page and attachments." ), ); @@ -232,11 +232,7 @@ pub async fn table_list } /// Get information about tables in storage. -pub async fn tables_list -( - mut storage : FeedStorage< gluesql::sled_storage::SledStorage >, - _args : &wca::Args, -) -> Result< impl Report > +pub async fn tables_list( mut storage : FeedStorage< gluesql::sled_storage::SledStorage > ) -> Result< impl Report > { storage.tables_list().await } @@ -245,7 +241,7 @@ const EMPTY_CELL : &'static str = ""; /// Information about execution of table columns commands. #[ derive( Debug ) ] -pub struct TablesColumnsReport( Vec< ColumnsReport > ); +pub struct TablesColumnsReport( pub Vec< ColumnsReport > ); impl std::fmt::Display for TablesColumnsReport { @@ -262,7 +258,7 @@ impl std::fmt::Display for TablesColumnsReport impl Report for TablesColumnsReport {} -/// Information about execution of columns commands. +/// Information about execution of columns listing action. #[ derive( Debug ) ] pub struct ColumnsReport { @@ -336,11 +332,9 @@ impl std::fmt::Display for ColumnsReport impl Report for ColumnsReport {} /// Information about execution of tables commands. +/// Contains tables name, description and list of columns. #[ derive( Debug ) ] -pub struct TablesReport -{ - tables : std::collections::HashMap< String, ( String, Vec< String > ) > -} +pub struct TablesReport( pub HashMap< String, ( String, Vec< String > ) > ); impl TablesReport { @@ -348,9 +342,8 @@ impl TablesReport pub fn new( payload : Vec< Payload > ) -> Self { let mut result = std::collections::HashMap::new(); - match &payload[ 0 ] + if let Payload::Select { labels: _label_vec, rows: rows_vec } = &payload[ 0 ] { - Payload::Select { labels: _label_vec, rows: rows_vec } => { for row in rows_vec { @@ -367,10 +360,9 @@ impl TablesReport .or_insert( ( table_description, vec![ String::from( row[ 1 ].clone() ) ] ) ) ; } - }, - _ => {}, + } } - TablesReport{ tables : result } + TablesReport( result ) } } @@ -380,7 +372,7 @@ impl std::fmt::Display for TablesReport { writeln!( f, "Storage tables:" )?; let mut rows = Vec::new(); - for ( table_name, ( desc, columns ) ) in &self.tables + for ( table_name, ( desc, columns ) ) in &self.0 { let columns_str = if !columns.is_empty() { @@ -409,7 +401,7 @@ impl std::fmt::Display for TablesReport [ EMPTY_CELL.to_owned(), "name".to_owned(), - "description".to_owned(), + "description".to_owned(), "columns".to_owned(), ], rows, diff --git a/module/move/unitore/src/command/config.rs b/module/move/unitore/src/command/config.rs index 37bbfb7aaf..72eb063007 100644 --- a/module/move/unitore/src/command/config.rs +++ b/module/move/unitore/src/command/config.rs @@ -1,5 +1,7 @@ //! Config files commands. +use std::path::PathBuf; + use crate::*; use gluesql::sled_storage::sled::Config; use wca::{ Command, Type, VerifiedCommand }; @@ -37,25 +39,35 @@ impl ConfigCommand .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() .routine( move | o : VerifiedCommand | { - let res = rt.block_on( async move - { - let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) - .unwrap_or( String::from( "./_data" ) ) - ; - - let config = Config::default() - .path( path_to_storage ) - ; - - let feed_storage = FeedStorage::init_storage( &config ).await?; - config_add( feed_storage, &o.args ).await - }); + let path_arg = o.args + .get_owned::< wca::Value >( 0 ); + + if let Some( path ) = path_arg + { + let path : PathBuf = path.into(); + + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_add( feed_storage, &path ).await + } + ); + match res { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), } - + } }) .end() ) @@ -76,27 +88,37 @@ impl ConfigCommand )) .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() .routine( move | o : VerifiedCommand | - { - let res = rt.block_on( async move - { - let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) - .unwrap_or( String::from( "./_data" ) ) - ; - - let config = Config::default() - .path( path_to_storage ) - ; + { + let path_arg = o.args + .get_owned::< wca::Value >( 0 ); - let feed_storage = FeedStorage::init_storage( &config ).await?; - config_delete( feed_storage, &o.args ).await - }); - match res + if let Some( path ) = path_arg { - Ok( report ) => report.report(), - Err( err ) => println!( "{:?}", err ), + let path : PathBuf = path.into(); + + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + config_delete( feed_storage, &path ).await + } + ); + + match res + { + Ok( report ) => report.report(), + Err( err ) => println!( "{:?}", err ), + } } - - }) + }) .end() ) } diff --git a/module/move/unitore/src/command/feed.rs b/module/move/unitore/src/command/feed.rs index 9255dbf3a4..148d404952 100644 --- a/module/move/unitore/src/command/feed.rs +++ b/module/move/unitore/src/command/feed.rs @@ -16,7 +16,6 @@ impl FeedCommand /// Create command that lists all feeds in storage. pub fn list() -> Result< Command > { - let rt = tokio::runtime::Runtime::new()?; Ok @@ -28,7 +27,7 @@ impl FeedCommand "List all feeds from storage.\n", " Example: .feeds.list", )) - .routine( move | o : VerifiedCommand | + .routine( move | _o : VerifiedCommand | { let res = rt.block_on( async move { @@ -41,7 +40,7 @@ impl FeedCommand ; let feed_storage = FeedStorage::init_storage( &config ).await?; - feeds_list( feed_storage, &o.args ).await + feeds_list( feed_storage ).await }); match res { diff --git a/module/move/unitore/src/command/frame.rs b/module/move/unitore/src/command/frame.rs index bdfe0d1174..8a4f18a756 100644 --- a/module/move/unitore/src/command/frame.rs +++ b/module/move/unitore/src/command/frame.rs @@ -27,7 +27,7 @@ impl FrameCommand "List all frames saved in storage.\n", " Example: .frames.list", )) - .routine( move | o : VerifiedCommand | + .routine( move | _o : VerifiedCommand | { let res = rt.block_on( async move { @@ -40,7 +40,7 @@ impl FrameCommand ; let feed_storage = FeedStorage::init_storage( &config ).await?; - frames_list( feed_storage, &o.args ).await + frames_list( feed_storage ).await }); match res { @@ -67,7 +67,7 @@ impl FrameCommand "Download frames from feed sources provided in config files.\n", " Example: .frames.download", )) - .routine( move | o : VerifiedCommand | + .routine( move | _o : VerifiedCommand | { let res = rt.block_on( async move { @@ -80,7 +80,7 @@ impl FrameCommand ; let feed_storage = FeedStorage::init_storage( &config ).await?; - frames_download( feed_storage, &o.args ).await + frames_download( feed_storage ).await }); match res { diff --git a/module/move/unitore/src/command/query.rs b/module/move/unitore/src/command/query.rs index b9ebbd67c5..24519e1a86 100644 --- a/module/move/unitore/src/command/query.rs +++ b/module/move/unitore/src/command/query.rs @@ -24,36 +24,45 @@ impl QueryCommand .phrase( "query.execute" ) .long_hint( concat! ( - "Execute custom query. Subject: query string, with special characters escaped.\n", + "Execute custom query. Subject: query string.\n", " Example query:\n", " - select all frames:\n", - r#" .query.execute \'SELECT \* FROM frame\'"#, + r#" .query.execute 'SELECT * FROM frame'"#, "\n", " - select title and link to the most recent frame:\n", - r#" .query.execute \'SELECT title, links, MIN\(published\) FROM frame\'"#, + r#" .query.execute 'SELECT title, links, MIN( published ) FROM frame'"#, "\n\n", )) .subject().hint( "Query" ).kind( Type::String ).optional( false ).end() .routine( move | o : VerifiedCommand | { - let res = rt.block_on( async move - { - let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) - .unwrap_or( String::from( "./_data" ) ) - ; - - let config = Config::default() - .path( path_to_storage ) - ; - - let feed_storage = FeedStorage::init_storage( &config ).await?; - query_execute( feed_storage, &o.args ).await - }); + let query_arg = o.args + .get_owned::< String >( 0 ) + ; + + if let Some( query_str ) = query_arg + { + let res = rt.block_on + ( async move + { + let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) + .unwrap_or( String::from( "./_data" ) ) + ; + + let config = Config::default() + .path( path_to_storage ) + ; + + let feed_storage = FeedStorage::init_storage( &config ).await?; + query_execute( feed_storage, query_str ).await + } + ); match res { Ok( report ) => report.report(), Err( err ) => println!( "{:?}", err ), } + } }) .end() diff --git a/module/move/unitore/src/command/table.rs b/module/move/unitore/src/command/table.rs index 34db044668..67c82f23a0 100644 --- a/module/move/unitore/src/command/table.rs +++ b/module/move/unitore/src/command/table.rs @@ -30,6 +30,8 @@ impl TableCommand .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() .routine( move | o : VerifiedCommand | { + let table_name_arg = o.args.get_owned::< String >( 0 ); + let res = rt.block_on( async move { let path_to_storage = std::env::var( "UNITORE_STORAGE_PATH" ) @@ -41,7 +43,7 @@ impl TableCommand ; let feed_storage = FeedStorage::init_storage( &config ).await?; - table_list( feed_storage, &o.args ).await + table_list( feed_storage, table_name_arg ).await } ); match res { @@ -76,7 +78,7 @@ impl TablesCommand " Example: .config.delete ./config/feeds.toml", )) .subject().hint( "Path" ).kind( Type::Path ).optional( false ).end() - .routine( move | o : VerifiedCommand | + .routine( move | _o : VerifiedCommand | { let res = rt.block_on( async move { @@ -89,7 +91,7 @@ impl TablesCommand ; let feed_storage = FeedStorage::init_storage( &config ).await?; - tables_list( feed_storage, &o.args ).await + tables_list( feed_storage ).await } ); match res { diff --git a/module/move/unitore/src/sled_adapter/frame.rs b/module/move/unitore/src/sled_adapter/frame.rs index 666db0921f..e2224f4958 100644 --- a/module/move/unitore/src/sled_adapter/frame.rs +++ b/module/move/unitore/src/sled_adapter/frame.rs @@ -75,7 +75,20 @@ impl FrameStore for FeedStorage< SledStorage > .insert() .columns ( - self.frame_fields.iter().map( | field | field[ 0 ] ).join( "," ).as_str() + "id, + title, + stored_time, + authors, + content, + links, + summary, + categories, + published, + source, + rights, + media, + language, + feed_link" ) .values( entries_rows ) .execute( &mut *self.storage.lock().await ) diff --git a/module/move/unitore/src/sled_adapter/mod.rs b/module/move/unitore/src/sled_adapter/mod.rs index ff73740de2..511d866e8e 100644 --- a/module/move/unitore/src/sled_adapter/mod.rs +++ b/module/move/unitore/src/sled_adapter/mod.rs @@ -27,8 +27,6 @@ pub struct FeedStorage< S : GStore + GStoreMut + Send > { /// GlueSQL storage. pub storage : Arc< Mutex< Glue< S > > >, - /// Frame table fields with description. - pub frame_fields : Vec< [ &'static str; 3 ] >, } impl< S : GStore + GStoreMut + Send > std::fmt::Debug for FeedStorage< S > @@ -50,13 +48,13 @@ impl FeedStorage< SledStorage > let mut glue = Glue::new( storage ); - let sub_table = table( "config" ) + let config_table = table( "config" ) .create_table_if_not_exists() .add_column( "path TEXT PRIMARY KEY" ) .build()? ; - sub_table.execute( &mut glue ).await?; + config_table.execute( &mut glue ).await?; let feed_table = table( "feed" ) .create_table_if_not_exists() @@ -74,37 +72,28 @@ impl FeedStorage< SledStorage > feed_table.execute( &mut glue ).await?; - let frame_fields = vec! - [ - [ "id", "TEXT", "A unique identifier for this frame in the feed. " ], - [ "title", "TEXT", "Title of the frame" ], - [ "stored_time", "TIMESTAMP", "Time at which this item was fetched from source." ], - [ "authors", "LIST", "List of authors of the frame, optional." ], - [ "content", "TEXT", "The content of the frame in html or plain text, optional." ], - [ "links", "LIST", "List of links associated with this item of related Web page and attachments." ], - [ "summary", "TEXT", "Short summary, abstract, or excerpt of the frame item, optional." ], - [ "categories", "LIST", "Specifies a list of categories that the item belongs to." ], - [ "published", "TIMESTAMP", "Time at which this item was first published or updated." ], - [ "source", "TEXT", "Specifies the source feed if the frame was copied from one feed into another feed, optional." ], - [ "rights", "TEXT", "Conveys information about copyrights over the feed, optional." ], - [ "media", "LIST", "List of media oblects, encountered in the frame, optional." ], - [ "language", "TEXT", "The language specified on the item, optional." ], - [ "feed_link", "TEXT", "Link of feed that contains this frame." ], - ]; - let mut table = table( "frame" ).create_table_if_not_exists().add_column( "id TEXT PRIMARY KEY" ); - - for column in frame_fields.iter().skip( 1 ).take( frame_fields.len() - 2 ) - { - table = table.add_column( format!( "{} {}", column[ 0 ], column[ 1 ] ).as_str() ); - } - - let table = table.add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) + let frame_table = table( "frame" ) + .create_table_if_not_exists() + .add_column( "id TEXT PRIMARY KEY" ) + .add_column( "title TEXT" ) + .add_column( "stored_time TIMESTAMP" ) + .add_column( "authors LIST" ) + .add_column( "content TEXT" ) + .add_column( "links LIST" ) + .add_column( "summary TEXT" ) + .add_column( "categories LIST" ) + .add_column( "published TIMESTAMP" ) + .add_column( "source TEXT" ) + .add_column( "rights TEXT" ) + .add_column( "media LIST" ) + .add_column( "language TEXT" ) + .add_column( "feed_link TEXT FOREIGN KEY REFERENCES feed(link)" ) .build()? ; - table.execute( &mut glue ).await?; + frame_table.execute( &mut glue ).await?; - Ok( Self{ storage : Arc::new( Mutex::new( glue ) ), frame_fields } ) + Ok( Self{ storage : Arc::new( Mutex::new( glue ) ) } ) } } diff --git a/module/move/unitore/tests/config.rs b/module/move/unitore/tests/config.rs deleted file mode 100644 index d56c4fdfdc..0000000000 --- a/module/move/unitore/tests/config.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::path::PathBuf; -use gluesql:: -{ - sled_storage::sled::Config, - prelude::Payload::Select, -}; -use unitore:: -{ - sled_adapter::FeedStorage, - entity::{ feed::FeedStore, config::ConfigStore }, - action::config, -}; -use error_tools::Result; - -#[ tokio::test ] -async fn config_add() -> Result< () > -{ - let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); - - let config = Config::default() - .path( "./test_add".to_owned() ) - .temporary( true ) - ; - - let mut feed_storage = FeedStorage::init_storage( &config ).await?; - config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; - - let res = feed_storage.feeds_list().await?; - - let feeds_links = res.0.selected_rows - .iter() - .map( | feed | String::from( feed[ 1 ].clone() ) ) - .collect::< Vec< _ > >() - ; - - assert!( feeds_links.len() == 2 ); - assert!( feeds_links.contains( &format!( "https://feeds.bbci.co.uk/news/world/rss.xml" ) ) ); - assert!( feeds_links.contains( &format!( "https://rss.nytimes.com/services/xml/rss/nyt/World.xml" ) ) ); - - Ok( () ) -} - -#[ tokio::test ] -async fn config_delete() -> Result< () > -{ - let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); - - let config = Config::default() - .path( "./test_del".to_owned() ) - .temporary( true ) - ; - - let mut feed_storage = FeedStorage::init_storage( &config ).await?; - config::config_add( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path.clone() ) ] ) ).await?; - - config::config_delete( feed_storage.clone(), &wca::Args( vec![ wca::Value::Path( path ) ] ) ).await?; - - let list = feed_storage.config_list().await?; - - if let Select{ labels : _, rows } = list - { - assert!( rows.len() == 0 ) - } - else - { - assert!( false ); - } - - Ok( () ) -} diff --git a/module/move/unitore/tests/config_add.rs b/module/move/unitore/tests/config_add.rs new file mode 100644 index 0000000000..a3de7479b7 --- /dev/null +++ b/module/move/unitore/tests/config_add.rs @@ -0,0 +1,37 @@ +use std::path::PathBuf; +use gluesql::sled_storage::sled::Config; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::feed::FeedStore, + action::config, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn config_add() -> Result< () > +{ + let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); + + let config = Config::default() + .path( "./test_add".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &path ).await?; + + let res = feed_storage.feeds_list().await?; + + let feeds_links = res.0.selected_rows + .iter() + .map( | feed | String::from( feed[ 1 ].clone() ) ) + .collect::< Vec< _ > >() + ; + + assert!( feeds_links.len() == 2 ); + assert!( feeds_links.contains( &format!( "https://feeds.bbci.co.uk/news/world/rss.xml" ) ) ); + assert!( feeds_links.contains( &format!( "https://rss.nytimes.com/services/xml/rss/nyt/World.xml" ) ) ); + + Ok( () ) +} diff --git a/module/move/unitore/tests/config_delete.rs b/module/move/unitore/tests/config_delete.rs new file mode 100644 index 0000000000..95870d4700 --- /dev/null +++ b/module/move/unitore/tests/config_delete.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; +use gluesql:: +{ + sled_storage::sled::Config, + prelude::Payload::Select, +}; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::config::ConfigStore, + action::config, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn config_delete() -> Result< () > +{ + let path = PathBuf::from( "./tests/fixtures/test_config.toml" ); + + let config = Config::default() + .path( "./test_del".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + config::config_add( feed_storage.clone(), &path ).await?; + + config::config_delete( feed_storage.clone(), &path ).await?; + + let list = feed_storage.config_list().await?; + + if let Select{ labels : _, rows } = list + { + assert!( rows.len() == 0 ) + } + else + { + assert!( false ); + } + + Ok( () ) +} diff --git a/module/move/unitore/tests/table_list.rs b/module/move/unitore/tests/table_list.rs new file mode 100644 index 0000000000..dc840b3633 --- /dev/null +++ b/module/move/unitore/tests/table_list.rs @@ -0,0 +1,45 @@ +use gluesql:: +{ + sled_storage::sled::Config, + prelude::{ Payload, Value::Str }, +}; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::table::TableStore, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn table_list() -> Result< () > +{ + let config = Config::default() + .path( "./test_list".to_owned() ) + .temporary( true ) + ; + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + + let res = feed_storage.table_list( String::from( "feed" ) ).await?; + + if let Payload::Select { labels: _, rows } = &res[ 0 ] + { + let column_names = rows + .iter() + .map( | row | row[ 1 ].clone() ) + .collect::< Vec< _ > >() + ; + + assert_eq!( column_names.len(), 9 ); + assert!( column_names.contains( &Str( String::from( "published") ) ) ); + assert!( column_names.contains( &Str( String::from( "authors") ) ) ); + assert!( column_names.contains( &Str( String::from( "description") ) ) ); + assert!( column_names.contains( &Str( String::from( "type") ) ) ); + assert!( column_names.contains( &Str( String::from( "title") ) ) ); + assert!( column_names.contains( &Str( String::from( "updated") ) ) ); + assert!( column_names.contains( &Str( String::from( "link") ) ) ); + assert!( column_names.contains( &Str( String::from( "update_period" ) ) ) ); + assert!( column_names.contains( &Str( String::from( "config_file" ) ) ) ); + } + + Ok( () ) +} diff --git a/module/move/unitore/tests/tables_list.rs b/module/move/unitore/tests/tables_list.rs new file mode 100644 index 0000000000..6b306b1c19 --- /dev/null +++ b/module/move/unitore/tests/tables_list.rs @@ -0,0 +1,32 @@ +use gluesql::sled_storage::sled::Config; +use unitore:: +{ + sled_adapter::FeedStorage, + entity::table::TableStore, +}; +use error_tools::Result; + +#[ tokio::test ] +async fn tables_list() -> Result< () > +{ + let config = Config::default() + .path( "./test_list".to_owned() ) + .temporary( true ) + ; + + let mut feed_storage = FeedStorage::init_storage( &config ).await?; + let res = feed_storage.tables_list().await?; + + let table_names = res.0 + .iter() + .map( | ( table_name, _info ) | table_name ) + .collect::< Vec< _ > >() + ; + + assert_eq!( table_names.len(), 3 ); + assert!( table_names.contains( &&String::from( "config") ) ); + assert!( table_names.contains( &&String::from( "feed" ) ) ); + assert!( table_names.contains( &&String::from( "frame" ) ) ); + + Ok( () ) +}