Merge pull request #20 from VolumeGraphics/new_csv_tokenizer

New csv tokenizer - 0.1.5-RC2

Author: ChrisRega

Cargo.toml

@@ -1,14 +1,16 @@
 [package]
 name = "havocompare"
-description = "A flexible folder comparison tool / crate with html reporting."
+description = "A flexible rule-based file and folder comparison tool and crate including nice html reporting. Compares CSVs, text files, pdf-texts and images."
 repository = "https://github.com/VolumeGraphics/havocompare"
 homepage = "https://github.com/VolumeGraphics/havocompare"
 documentation = "https://docs.rs/havocompare"
-version = "0.1.5-RC1"
+version = "0.2.0"
 edition = "2021"
 license = "MIT"
 authors = ["Volume Graphics GmbH"]
 exclude = ["tests/integ", "tests/html", "target", "tests/csv", ".github", "test_report"]
+keywords = ["diff" ,"compare", "csv", "image", "difference", "csv-diff", "image-diff", "pdf", "pdf-diff", "plain-text", "plain-text-diff"]
+categories = ["filesystem"]
 
 [dependencies]
 clap = {version= "4.0", features=["derive"]}
@@ -19,7 +21,7 @@ schemars_derive = "0.8"
 thiserror = "1.0"
 regex = "1.6"
 image = "0.24.4"
-image-compare = "0.2.3"
+image-compare = "0.2.4"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 serde_json = "1.0"
@@ -35,6 +37,7 @@ data-encoding = "2.3.2"
 permutation = "0.4.1"
 pdf-extract = "0.6.4"
 vg_errortools = "0.1.0"
+rayon = "1.6"
 
 [target.'cfg(windows)'.dependencies]
 ansi_term = "0.12"

README.md

@@ -91,16 +91,17 @@ rules:
       exclude_field_regex: "Excluded"
       # optional: preprocessing of the csv files
       preprocessing:
-        # extracts the headers to the header-fields, makes reportings more legible and allows for further processing "ByName"
+        # extracts the headers to the header-fields, makes reports more legible and allows for further processing "ByName".
+        # While it may fail, there's no penalty for it, as long as you don't rely on it.
         - ExtractHeaders
         # Sort the table by column 0, beware that the column must only contain numbers / quantities
         - SortByColumnNumber: 0
-        # Delete a column by name, needs `ExtractHeaders` first - delete sets all values to 'DELETED's
+        # Delete a column by name, needs `ExtractHeaders` first - delete sets all values to 'DELETED'
         - DeleteColumnByName: "Column to delete"
         - DeleteColumnByNumber: 1
         # Sorts are stable, so a second sort will keep the first sort as sub-order.
         - SortByColumnName: "Sort by column name blabla"
-        # Deletes the first row by setting all values to 'DELETED's - meaning that numbering stays constant 
+        # Deletes the first row by setting all values to 'DELETED' - meaning that numbering stays constant 
         - DeleteRowByNumber: 0
         # Deletes rows having any element matching the given regex (may delete different lines in nom / act!
         - DeleteRowByRegex: "Vertex_Count"
@@ -177,12 +178,18 @@ Currently we only support SHA-256 but more checks can be added easily.
 
 ## Changelog
 
-### 0.1.5
+### 0.2.0
 - Deletion of columns will no longer really delete them but replace every value with "DELETED"
 - Expose config struct to library API
 - Fixed a bug regarding wrong handling of multiple empty lines
 - Reworked CSV reporting to have an interleaved and more compact view 
-- Display the relative path of compared files instead of file name in the report index.html
+  - Display the relative path of compared files instead of file name in the report index.html
+  - Made header-extraction fallible but uncritical - can now always be enabled
+- Wrote a completely new csv parser:
+  - Respects escaping with '\'
+  - Allows string-literals containing unescaped field separators (field1, "field2, but as literal", field3)
+  - Allows multi-line string literals with quotes
+- CSVs with non-rectangular format will now fail
 
 ### 0.1.4
 - Add multiple includes and excludes - warning, this will break yamls from 0.1.3 and earlier