feat(sync): moved logic in sync-test scripts into rust, enabling sync with a single command (#430)

* feat(sync): moved logic in `test-sync-pull/push.sh` scripts into rust code, enabling easy sync with a single command

* fix: fixes to sync dirs

* fix: moved --sync-dir arg into sync-advanced subcommand, fixed warnings

* fix: pass around client instead of port/testing

* fix: fixed log dir for aw-sync, misc sync fixes and refactor

Author: ErikBjare

Cargo.lock

@@ -53,15 +53,15 @@ pub fn get_cache_dir() -> Result<PathBuf, ()> {
 }
 
 #[cfg(not(target_os = "android"))]
-pub fn get_log_dir() -> Result<PathBuf, ()> {
+pub fn get_log_dir(module: &str) -> Result<PathBuf, ()> {
     let mut dir = appdirs::user_log_dir(Some("activitywatch"), None)?;
-    dir.push("aw-server-rust");
+    dir.push(module);
     fs::create_dir_all(dir.clone()).expect("Unable to create log dir");
     Ok(dir)
 }
 
 #[cfg(target_os = "android")]
-pub fn get_log_dir() -> Result<PathBuf, ()> {
+pub fn get_log_dir(module: &str) -> Result<PathBuf, ()> {
     panic!("not implemented on Android");
 }
 
@@ -87,7 +87,7 @@ fn test_get_dirs() {
     set_android_data_dir("/test");
 
     get_cache_dir().unwrap();
-    get_log_dir().unwrap();
+    get_log_dir("aw-server-rust").unwrap();
     db_path(true).unwrap();
     db_path(false).unwrap();
 }

aw-server/src/dirs.rs

@@ -5,19 +5,17 @@ use fern::colors::{Color, ColoredLevelConfig};
 
 use crate::dirs;
 
-pub fn setup_logger(testing: bool, verbose: bool) -> Result<(), fern::InitError> {
+pub fn setup_logger(module: &str, testing: bool, verbose: bool) -> Result<(), fern::InitError> {
     let mut logfile_path: PathBuf =
-        dirs::get_log_dir().expect("Unable to get log dir to store logs in");
+        dirs::get_log_dir(module).expect("Unable to get log dir to store logs in");
     fs::create_dir_all(logfile_path.clone()).expect("Unable to create folder for logs");
-    logfile_path.push(
-        chrono::Local::now()
-            .format(if !testing {
-                "aw-server_%Y-%m-%dT%H-%M-%S%z.log"
-            } else {
-                "aw-server-testing_%Y-%m-%dT%H-%M-%S%z.log"
-            })
-            .to_string(),
-    );
+    let filename = if !testing {
+        format!("{}_%Y-%m-%dT%H-%M-%S%z.log", module)
+    } else {
+        format!("{}-testing_%Y-%m-%dT%H-%M-%S%z.log", module)
+    };
+
+    logfile_path.push(chrono::Local::now().format(&filename).to_string());
 
     log_panics::init();
 
@@ -93,6 +91,6 @@ mod tests {
     #[ignore]
     #[test]
     fn test_setup_logger() {
-        setup_logger(true, true).unwrap();
+        setup_logger("aw-server-rust", true, true).unwrap();
     }
 }

aw-server/src/logging.rs

@@ -70,7 +70,8 @@ async fn main() -> Result<(), rocket::Error> {
         testing = true;
     }
 
-    logging::setup_logger(testing, opts.verbose).expect("Failed to setup logging");
+    logging::setup_logger("aw-server-rust", testing, opts.verbose)
+        .expect("Failed to setup logging");
 
     if testing {
         info!("Running server in Testing mode");

aw-server/src/main.rs

@@ -14,12 +14,16 @@ path = "src/main.rs"
 
 [dependencies]
 log = "0.4"
+toml = "0.7"
 chrono = { version = "0.4", features = ["serde"] }
 serde = "1.0"
 serde_json = "1.0"
 reqwest = { version = "0.11", features = ["json", "blocking"] }
 clap = { version = "4.1", features = ["derive"] }
+appdirs = "0.2.0"
+dirs = "3.0.2"
 aw-server = { path = "../aw-server" }
 aw-models = { path = "../aw-models" }
 aw-datastore = { path = "../aw-datastore" }
 aw-client-rust = { path = "../aw-client-rust" }
+gethostname = "0.4.3"

aw-sync/Cargo.toml

@@ -0,0 +1,27 @@
+use dirs::home_dir;
+use std::fs;
+use std::path::PathBuf;
+
+// TODO: This could be refactored to share logic with aw-server/src/dirs.rs
+// TODO: add proper config support
+#[allow(dead_code)]
+pub fn get_config_dir() -> Result<PathBuf, ()> {
+    let mut dir = appdirs::user_config_dir(Some("activitywatch"), None, false)?;
+    dir.push("aw-sync");
+    fs::create_dir_all(dir.clone()).expect("Unable to create config dir");
+    Ok(dir)
+}
+
+pub fn get_server_config_path(testing: bool) -> Result<PathBuf, ()> {
+    let dir = aw_server::dirs::get_config_dir()?;
+    Ok(dir.join(if testing {
+        "config-testing.toml"
+    } else {
+        "config.toml"
+    }))
+}
+
+pub fn get_sync_dir() -> Result<PathBuf, ()> {
+    // TODO: make this configurable
+    home_dir().map(|p| p.join("ActivityWatchSync")).ok_or(())
+}

aw-sync/src/dirs.rs

@@ -10,5 +10,12 @@ pub use sync::sync_datastores;
 pub use sync::sync_run;
 pub use sync::SyncSpec;
 
+mod sync_wrapper;
+pub use sync_wrapper::push;
+pub use sync_wrapper::{pull, pull_all};
+
 mod accessmethod;
 pub use accessmethod::AccessMethod;
+
+mod dirs;
+mod util;

aw-sync/src/lib.rs

@@ -23,9 +23,10 @@ use clap::{Parser, Subcommand};
 use aw_client_rust::blocking::AwClient;
 
 mod accessmethod;
+mod dirs;
 mod sync;
-
-const DEFAULT_PORT: &str = "5600";
+mod sync_wrapper;
+mod util;
 
 #[derive(Parser)]
 #[clap(version = "0.1", author = "Erik Bjäreholt")]
@@ -38,8 +39,8 @@ struct Opts {
     host: String,
 
     /// Port of instance to connect to.
-    #[clap(long, default_value = DEFAULT_PORT)]
-    port: String,
+    #[clap(long)]
+    port: Option<String>,
 
     /// Convenience option for using the default testing host and port.
     #[clap(long)]
@@ -48,42 +49,53 @@ struct Opts {
     /// Enable debug logging.
     #[clap(long)]
     verbose: bool,
-
-    /// Full path to sync directory.
-    /// If not specified, exit.
-    #[clap(long)]
-    sync_dir: String,
-
-    /// Full path to sync db file
-    /// Useful for syncing buckets from a specific db file in the sync directory.
-    /// Must be a valid absolute path to a file in the sync directory.
-    #[clap(long)]
-    sync_db: Option<String>,
 }
 
 #[derive(Subcommand)]
 enum Commands {
-    /// Sync subcommand.
+    /// Sync subcommand (basic)
+    ///
+    /// Pulls remote buckets then pushes local buckets.
+    Sync {
+        /// Host(s) to pull from, comma separated. Will pull from all hosts if not specified.
+        #[clap(long)]
+        host: Option<String>,
+    },
+
+    /// Sync subcommand (advanced)
     ///
     /// Pulls remote buckets then pushes local buckets.
     /// First pulls remote buckets in the sync directory to the local aw-server.
     /// Then pushes local buckets from the aw-server to the local sync directory.
     #[clap(arg_required_else_help = true)]
-    Sync {
+    SyncAdvanced {
         /// Date to start syncing from.
         /// If not specified, start from beginning.
         /// NOTE: might be unstable, as count cannot be used to verify integrity of sync.
         /// Format: YYYY-MM-DD
         #[clap(long)]
         start_date: Option<String>,
+
         /// Specify buckets to sync using a comma-separated list.
         /// If not specified, all buckets will be synced.
         #[clap(long)]
         buckets: Option<String>,
+
         /// Mode to sync in. Can be "push", "pull", or "both".
         /// Defaults to "both".
         #[clap(long, default_value = "both")]
         mode: String,
+
+        /// Full path to sync directory.
+        /// If not specified, exit.
+        #[clap(long)]
+        sync_dir: String,
+
+        /// Full path to sync db file
+        /// Useful for syncing buckets from a specific db file in the sync directory.
+        /// Must be a valid absolute path to a file in the sync directory.
+        #[clap(long)]
+        sync_db: Option<String>,
     },
     /// List buckets and their sync status.
     List {},
@@ -95,35 +107,59 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     info!("Started aw-sync...");
 
-    aw_server::logging::setup_logger(true, verbose).expect("Failed to setup logging");
-
-    let sync_directory = if opts.sync_dir.is_empty() {
-        println!("No sync directory specified, exiting...");
-        std::process::exit(1);
-    } else {
-        Path::new(&opts.sync_dir)
-    };
-    info!("Using sync dir: {}", sync_directory.display());
-
-    if let Some(sync_db) = &opts.sync_db {
-        info!("Using sync db: {}", sync_db);
-    }
+    aw_server::logging::setup_logger("aw-sync", opts.testing, verbose)
+        .expect("Failed to setup logging");
 
-    let port = if opts.testing && opts.port == DEFAULT_PORT {
-        "5666"
-    } else {
-        &opts.port
-    };
+    let port = opts
+        .port
+        .or_else(|| Some(crate::util::get_server_port(opts.testing).ok()?.to_string()))
+        .unwrap();
 
-    let client = AwClient::new(opts.host.as_str(), port, "aw-sync");
+    let client = AwClient::new(opts.host.as_str(), port.as_str(), "aw-sync");
 
     match &opts.command {
+        // Perform basic sync
+        Commands::Sync { host } => {
+            // Pull
+            match host {
+                Some(host) => {
+                    let hosts: Vec<&str> = host.split(',').collect();
+                    for host in hosts.iter() {
+                        info!("Pulling from host: {}", host);
+                        sync_wrapper::pull(host, &client)?;
+                    }
+                }
+                None => {
+                    info!("Pulling from all hosts");
+                    sync_wrapper::pull_all(&client)?;
+                }
+            }
+
+            // Push
+            info!("Pushing local data");
+            sync_wrapper::push(&client)?;
+            Ok(())
+        }
         // Perform two-way sync
-        Commands::Sync {
+        Commands::SyncAdvanced {
             start_date,
             buckets,
             mode,
+            sync_dir,
+            sync_db,
         } => {
+            let sync_directory = if sync_dir.is_empty() {
+                error!("No sync directory specified, exiting...");
+                std::process::exit(1);
+            } else {
+                Path::new(&sync_dir)
+            };
+            info!("Using sync dir: {}", sync_directory.display());
+
+            if let Some(sync_db) = &sync_db {
+                info!("Using sync db: {}", sync_db);
+            }
+
             let start: Option<DateTime<Utc>> = start_date.as_ref().map(|date| {
                 println!("{}", date.clone());
                 chrono::NaiveDate::parse_from_str(&date.clone(), "%Y-%m-%d")
@@ -140,7 +176,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .as_ref()
                 .map(|b| b.split(',').map(|s| s.to_string()).collect());
 
-            let sync_db: Option<PathBuf> = opts.sync_db.as_ref().map(|db| {
+            let sync_db: Option<PathBuf> = sync_db.as_ref().map(|db| {
                 let db_path = Path::new(db);
                 if !db_path.is_absolute() {
                     panic!("Sync db path must be absolute");
@@ -165,11 +201,11 @@ fn main() -> Result<(), Box<dyn Error>> {
                 _ => panic!("Invalid mode"),
             };
 
-            sync::sync_run(client, &sync_spec, mode_enum)
+            sync::sync_run(&client, &sync_spec, mode_enum)
         }
 
         // List all buckets
-        Commands::List {} => sync::list_buckets(&client, sync_directory),
+        Commands::List {} => sync::list_buckets(&client),
     }?;
 
     // Needed to give the datastores some time to commit before program is shut down.