Merge pull request #4 from telostat/vst/refactor

Make rrclone Configuration Self-Sufficient

config.yaml.tmpl

@@ -1,16 +1,60 @@
-- source_remote: source1
-  source_path: /
-  target_remote: target1
-  target_path: /data/backups/source1
-  filters:
-  - "+ /data/**"
-  - "+ /home/**"
-  - "- *"
-- source_remote: source2
-  source_path: /
-  target_remote: target1
-  target_path: /data/backups/source2
-  filters:
-  - "+ /data/**"
-  - "+ /home/**"
-  - "- *"
+tasks:
+  - name: "Backup remote to remote"
+    source:
+      backend:
+        type: "sftp"
+        vars:
+          host: "remote-source"
+          user: "user1"
+          md5sum_command: "md5sum"
+          sha1sum_command: "sha1sum"
+      path: "/"
+      filters:
+        - "+ /data/**"
+        - "+ /home/**"
+        - "- *"
+    target:
+      backend:
+        type: "sftp"
+        vars:
+          host: "remote-target"
+          user: "user2"
+          md5sum_command: "md5sum"
+          sha1sum_command: "sha1sum"
+      path: "/backups/remote-source/"
+  - name: "Backup remote to local"
+    source:
+      backend:
+        type: "sftp"
+        vars:
+          host: "remote-source"
+          user: "user1"
+          md5sum_command: "md5sum"
+          sha1sum_command: "sha1sum"
+      path: "/"
+      filters:
+        - "+ /data/**"
+        - "+ /home/**"
+        - "- *"
+    target:
+      backend:
+        type: "local"
+      path: "/backups/remote-source"
+  - name: "Backup local to remote"
+    source:
+      backend:
+        type: "local"
+      path: "/"
+      filters:
+        - "+ /data/**"
+        - "+ /home/**"
+        - "- *"
+    target:
+      backend:
+        type: "sftp"
+        vars:
+          host: "remote-target"
+          user: "user1"
+          md5sum_command: "md5sum"
+          sha1sum_command: "sha1sum"
+      path: "/backups/laptop"

src/config.rs

@@ -0,0 +1,45 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fs;
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Config {
+    pub tasks: Vec<Task>,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Task {
+    pub name: String,
+    pub source: Source,
+    pub target: Target,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Source {
+    pub backend: Backend,
+    pub path: String,
+    pub filters: Vec<String>,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Target {
+    pub backend: Backend,
+    pub path: String,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct Backend {
+    #[serde(rename = "type")]
+    pub ctype: String,
+    pub vars: Option<HashMap<String, String>>,
+}
+
+pub fn read_config(path: &String) -> Result<Config, String> {
+    match fs::read_to_string(path) {
+        Ok(content) => match serde_yaml::from_str(&content) {
+            Ok(config) => Ok(config),
+            Err(err) => Err(format!("Can not parse configuration. Error: {:?}", err)),
+        },
+        Err(err) => Err(format!("Can not read file {}. Error: {}", path, err)),
+    }
+}

src/main.rs

@@ -1,91 +1,24 @@
-use chrono::Utc;
-use serde::{Deserialize, Serialize};
 use std::env;
-use std::fs;
-use std::process::Command;
-use std::process::Stdio;
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-struct Task {
-    source_remote: String,
-    source_path: String,
-    target_remote: String,
-    target_path: String,
-    filters: Vec<String>,
-}
+mod config;
+mod rclone;
+mod utils;
 
-fn main() -> Result<(), serde_yaml::Error> {
+fn main() {
     let args: Vec<String> = env::args().collect();
     let dryrun = args.len() == 3 && &args[2] == "--dry-run";
 
-    let contents =
-        fs::read_to_string(&args[1]).expect("Something went wrong while reading the YAML file");
-
-    let result: Result<Vec<Task>, _> = serde_yaml::from_str(&contents);
-
-    match result {
-        Ok(tasks) => run_tasks(&tasks, dryrun),
-        Err(x) => log(format!("Something went wrong while decoding the YAML file {:?}:", x)),
-    }
-
-    Ok(())
-}
-
-fn log(msg: String) -> () {
-    eprintln!(
-        "RRCLONE>> [{}] {}",
-        Utc::now().format("%Y-%m-%d %H:%M:%S").to_string(),
-        msg
-    );
-}
-
-fn run_tasks(ts: &Vec<Task>, dryrun: bool) -> () {
-    for t in ts {
-        run_task(t, dryrun)
+    match config::read_config(&args[1]) {
+        Ok(config) => run(&config, dryrun),
+        Err(err) => utils::log(format!(
+            "Something went wrong while reading configuration file. Error was: {}",
+            err
+        )),
     }
 }
 
-fn run_task(t: &Task, dryrun: bool) -> () {
-    let tstart = Utc::now();
-    let source = format!("{}:{}", t.source_remote, t.source_path);
-    let target = format!("{}:{}", t.target_remote, t.target_path);
-
-    log(format!("Syncing from \"{}\" to \"{}\"", &source, &target));
-
-    let mut args = vec![
-        "-v",
-        "--stats-log-level",
-        "NOTICE",
-        "--stats",
-        "10000m",
-        "sync",
-        &source,
-        &target,
-    ];
-
-    if dryrun {
-        args.push("--dry-run");
-    }
-
-    for f in &t.filters {
-        args.push("--filter");
-        args.push(f);
-    }
-
-    let mut child = Command::new("rclone")
-        .args(&args)
-        .stdout(Stdio::inherit())
-        .stderr(Stdio::inherit())
-        .spawn()
-        .expect("Failed to execute process");
-
-    let output = child.wait().expect("Failed to read stdout from the task");
-
-    let elapsed = Utc::now() - tstart;
-
-    match output.code() {
-        Some(0) => log(format!("Tasks finished successfully in {:?} second(s).", elapsed.num_seconds())),
-        Some(code) => log(format!("Task finished with errors in {:?} second(s). Error code: {}", elapsed.num_seconds(), code)),
-        None => log(format!("Task terminated by signal in {:?} second(s).", elapsed.num_seconds())),
+fn run(config: &config::Config, dryrun: bool) -> () {
+    for task in &config.tasks {
+        rclone::run_task(&task, dryrun)
     }
 }

src/rclone.rs

@@ -0,0 +1,86 @@
+use chrono::Utc;
+use std::process::Command;
+use std::process::Stdio;
+
+use crate::config;
+use crate::utils;
+
+pub fn run_task(task: &config::Task, dryrun: bool) -> () {
+    let tstart = Utc::now();
+    utils::log(format!("Running task: {}", &task.name));
+
+    let mut args = task_to_args(&task);
+
+    if dryrun {
+        args.push("--dry-run".to_string());
+        utils::log(format!("Running rclone with args {}", args.join(" ")))
+    }
+
+    let mut child = Command::new("rclone")
+        .args(&args)
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .expect("Failed to execute process");
+
+    let output = child.wait().expect("Failed to read stdout from the task");
+
+    let elapsed = (Utc::now() - tstart).num_seconds();
+
+    match output.code() {
+        Some(0) => utils::log(format!(
+            "Tasks finished successfully in {:?} second(s).",
+            elapsed
+        )),
+        Some(code) => utils::log(format!(
+            "Task finished with errors in {:?} second(s). Error code: {}",
+            elapsed, code
+        )),
+        None => utils::log(format!(
+            "Task terminated by signal in {:?} second(s).",
+            elapsed
+        )),
+    }
+}
+
+pub fn task_to_args(task: &config::Task) -> Vec<String> {
+    let mut args = vec![
+        "-v".to_string(), // Run in verbose mode.
+        "--stats-log-level".to_string(),
+        "NOTICE".to_string(),
+        "--stats".to_string(),
+        "10000m".to_string(),
+        "sync".to_string(),
+        format!(
+            "{}{}",
+            backend_to_args(&task.source.backend),
+            &task.source.path
+        ),
+        format!(
+            "{}{}",
+            backend_to_args(&task.target.backend),
+            &task.target.path
+        ),
+    ];
+
+    for f in &task.source.filters {
+        args.push("--filter".to_string());
+        args.push(f.to_string());
+    }
+
+    args
+}
+
+pub fn backend_to_args(backend: &config::Backend) -> String {
+    let vars: Vec<String> = match &backend.vars {
+        Some(xs) => xs.iter().map(|(x, y)| format!("{}={}", x, y)).collect(),
+        None => vec![],
+    };
+
+    return format!(
+        ":{}{}{}:",
+        backend.ctype,
+        if vars.len() == 0 { "" } else { "," },
+        vars.join(","),
+    );
+}

src/utils.rs

@@ -0,0 +1,9 @@
+use chrono::Utc;
+
+pub fn log(msg: String) -> () {
+    eprintln!("RRCLONE>> [{}] {}", isonow(), msg);
+}
+
+pub fn isonow() -> String {
+    Utc::now().format("%Y-%m-%d %H:%M:%S").to_string()
+}