add db-gen program to create DBs 3x to 20x as fast (powered by gitoxide)

It produces diffs in parallel, and despite wasting quite a bit of CPU
due to less-than-stellar object access performance for diffs, it still
manages to create a linux kernel database in ~21 minutes (M1 Pro).

Author: Byron

db-gen/.gitignore

@@ -0,0 +1,20 @@
+# Created by https://www.toptal.com/developers/gitignore/api/rust
+# Edit at https://www.toptal.com/developers/gitignore?templates=rust
+
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# End of https://www.toptal.com/developers/gitignore/api/rust

db-gen/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "db-gen"
+version = "0.1.0"
+edition = "2021"
+authors = ["Sebastian Thiel"]
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.68"
+gix = { version = "0.37.2", features = ["max-performance"] }
+prodash = { version = "23.0.0", features = ["render-line-autoconfigure", "render-line", "render-line-crossterm", "signal-hook"] }
+crossbeam-channel = "0.5.6"
+clap = { version = "4.1.6", features = ["derive"] }
+rusqlite = "0.28.0"

db-gen/README.md

@@ -0,0 +1,22 @@
+Produce a repository database fast with `gitoxide`.
+
+### How to run
+
+Assuming an existing `Rust` installation (which can be done [with rustup][https://rustup.rs]),
+run the following:
+
+```sh
+cargo run --release -- /path/to/repository
+```
+
+This will drop the required database files in the current directory.
+As the database location matters, one might want to do the following instead:
+
+```sh
+cargo build --release
+cd ..
+./db-gen/target/release/db-gen /path/to/repository
+```
+
+That way directory files are placed where they would be expected by the visualizer.
+

db-gen/rustfmt.toml

@@ -0,0 +1,2 @@
+max_width = 120
+

db-gen/src/db.rs

@@ -0,0 +1,61 @@
+pub fn create_indices(con: &rusqlite::Connection) -> rusqlite::Result<()> {
+    con.execute_batch("CREATE INDEX if not exists commitFileID ON commitFile (fileID)")?;
+    con.execute_batch("CREATE INDEX if not exists commitAuthorEmail ON commitAuthor (authorEmail)")
+}
+
+pub fn create(name: &str) -> rusqlite::Result<rusqlite::Connection> {
+    let con = rusqlite::Connection::open(format!("{name}.db"))?;
+    con.execute_batch(
+        r#"
+    CREATE TABLE if not exists commits(
+        hash character(40) NOT NULL PRIMARY KEY,
+        authorDate text NOT NULL,
+        committerName text NOT NULL,
+        committerEmail text NOT NULL,
+        committerDate text NOT NULL
+    )
+"#,
+    )?;
+    con.execute_batch(
+        r#"
+    CREATE TABLE if not exists files(
+        fileID integer NOT NULL PRIMARY KEY,
+        filePath text UNIQUE
+    )
+"#,
+    )?;
+    con.execute_batch(
+        r#"
+   CREATE TABLE if not exists commitFile(
+        hash character(40),
+        fileID text,
+        linesAdded integer,
+        linesRemoved integer,
+        FOREIGN KEY (hash) REFERENCES commits (hash),
+        FOREIGN KEY (fileID) REFERENCES files (fileID),
+        PRIMARY KEY (hash, fileID)
+    )
+    "#,
+    )?;
+    con.execute_batch(
+        r#"
+    CREATE TABLE if not exists author(
+        authorEmail text NOT NULL PRIMARY KEY,
+        authorName text NOT NULL
+    )
+"#,
+    )?;
+    con.execute_batch(
+        r#"
+    CREATE TABLE if not exists commitAuthor(
+        hash character(40),
+        authorEmail text,
+        FOREIGN KEY (hash) REFERENCES commits (hash),
+        FOREIGN KEY (authorEmail) REFERENCES author (authorEmail),
+        PRIMARY KEY (hash, authorEmail)
+    )
+"#,
+    )?;
+
+    Ok(con)
+}