Merge pull request #618 from Nukesor/compression-benchmarks

Compression benchmarks

CHANGELOG.md

@@ -110,8 +110,6 @@ Upon updating Pueue and restarting the daemon, the previous state will be wiped,
 - **Breaking**: Ported from `anyhow` to `color_eyre` for prettier log output.
 - **Breaking**: Switch `cbor` handling library, breaking backwards-compatible communication on a data format level.
 - **Breaking**: Switch protocol message representation, completely breaking backwards compatibility.
-- Option to save the state in compressed form. This can be toggled with the `daemon.compress_state_file` config file.
-  Preliminary testing shows significant compression ratios (up to x15), which helps with large states in embedded and I/O bound environments.
 
 ### Add
 
@@ -128,6 +126,12 @@ Upon updating Pueue and restarting the daemon, the previous state will be wiped,
 - Add `queued_count` and `stashed_count` to callback template variables. This allows users to fire callbacks when whole groups are finished. [#578](https://github.com/Nukesor/pueue/issues/578)
 - Add new subcommand to set or unset environment variables for tasks. [#503](https://github.com/Nukesor/pueue/issues/503)
 - Add `add --follow` flag that may be called in combination with `--immediate` [#592](https://github.com/Nukesor/pueue/issues/592)
+- Add option to save the state in compressed form. This can be toggled with the `daemon.compress_state_file` config file.
+  Preliminary testing shows significant compression ratios (up to x15), which helps with large states in embedded and I/O bound environments.
+  On my local machine with a state of 400 tasks, state file size **shrinks** from \~2MB to \~120KB and save time **increases** from \~8ms to \~20ms.
+  Due to the very repetitive nature of the state's data (mostly environment variables), the `gzip` compression algorithm with the `flate2` implementation has been chosen.
+  It shows similar compression rations to `zstd` on level `7`, which is more than enough and the dependency is significantly lighter than `zstd`.
+  `snappy`, which is already a dependency, has also been considered, but it has much worse compression ratios (~2MB -> ~300KB).
 
 ### Fixed
 

codecov.yml

@@ -7,6 +7,7 @@ ignore:
   - "LICENSE"
   - ".github"
   - ".gitignore"
+  - "pueue/benches"
 
 coverage:
   status:

pueue/Cargo.toml

@@ -14,6 +14,14 @@ rust-version.workspace = true
 [badges]
 maintenance = { status = "actively-developed" }
 
+[[bench]]
+harness = false
+name = "save_compressed_state"
+
+[[bench]]
+harness = false
+name = "save_state"
+
 [dependencies]
 chrono.workspace = true
 clap = { version = "4.5.30", features = [
@@ -55,6 +63,7 @@ tracing-subscriber = { version = "0.3.19", features = [
 assert_cmd = "2"
 assert_matches = "1"
 better-panic.workspace = true
+criterion = "0.5"
 pretty_assertions.workspace = true
 rstest = "0.24"
 serde_yaml.workspace = true

pueue/benches/save_compressed_state.rs

@@ -0,0 +1,43 @@
+use std::{collections::HashMap, env::vars, path::PathBuf};
+
+use chrono::Local;
+use criterion::{Criterion, criterion_group, criterion_main};
+use pueue::daemon::internal_state::state::InternalState;
+use pueue_lib::{Settings, Task, state::PUEUE_DEFAULT_GROUP};
+
+/// Create a large state file with a few hundred tasks.
+/// Save it to disk in compressed state
+fn save_compressed_state() {
+    let dir = tempfile::tempdir().unwrap();
+    let mut settings = Settings::default();
+    settings.shared.pueue_directory = Some(dir.path().to_owned());
+    settings.daemon.compress_state_file = true;
+
+    let mut state = InternalState::new();
+
+    for _ in 0..400 {
+        let task = Task::new(
+            "ls".into(),
+            PathBuf::from("/tmp"),
+            HashMap::from_iter(vars()),
+            PUEUE_DEFAULT_GROUP.to_owned(),
+            pueue_lib::TaskStatus::Queued {
+                enqueued_at: Local::now(),
+            },
+            Vec::new(),
+            0,
+            None,
+        );
+
+        state.add_task(task);
+    }
+
+    state.save(&settings).unwrap();
+}
+
+pub fn state(crit: &mut Criterion) {
+    crit.bench_function("Save compressed state", |b| b.iter(save_compressed_state));
+}
+
+criterion_group!(benches, state);
+criterion_main!(benches);