Merge pull request #89 from wagga40/2.30.0

v2.30.0

Dockerfile

@@ -4,7 +4,7 @@
 #
 ARG PYTHON_VERSION="3.11-slim"
 
-FROM "python:${PYTHON_VERSION}" as stage
+FROM "python:${PYTHON_VERSION}" AS stage
 
 ARG ZIRCOLITE_INSTALL_PREFIX="/opt"
 ARG ZIRCOLITE_REPOSITORY_URI="https://github.com/wagga40/Zircolite.git"
@@ -30,7 +30,7 @@ RUN chmod 0755 \
 FROM "python:${PYTHON_VERSION}"
 
 LABEL author="wagga40"
-LABEL description="A standalone SIGMA-based detection tool for EVTX."
+LABEL description="A standalone SIGMA-based detection tool for EVTX, Auditd and Sysmon for Linux logs."
 LABEL maintainer="wagga40"
 
 ARG ZIRCOLITE_INSTALL_PREFIX="/opt"
@@ -46,6 +46,8 @@ WORKDIR "${ZIRCOLITE_INSTALL_PREFIX}/zircolite"
 RUN python3 -m pip install \
         --requirement requirements.full.txt
 
+RUN python3 zircolite.py -U
+
 ENTRYPOINT [ "python3", "zircolite.py" ]
 
 CMD [ "--help" ]

README.md

@@ -1,24 +1,33 @@
 # <p align="center">![](pics/zircolite_400.png)</p>
 
-## Standalone SIGMA-based detection tool for EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs
+## Standalone SIGMA-based detection tool for EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs 
 ![](pics/Zircolite_v2.9.gif)
 
-[![python](https://img.shields.io/badge/python-3.8-blue)](https://www.python.org/)
+[![python](https://img.shields.io/badge/python-3.10-blue)](https://www.python.org/)
 ![version](https://img.shields.io/badge/Architecture-64bit-red)
 
-> **Zircolite is a standalone tool written in Python 3. It allows to use SIGMA rules on : MS Windows EVTX (EVTX, XML and JSONL format), Auditd logs, Sysmon for Linux, EVTXtract, CSV and XML logs**
+**Zircolite** is a standalone tool written in Python 3 that allows you to use SIGMA rules on:
 
-- **Zircolite** is relatively fast and can parse large datasets in just seconds
-- **Zircolite** is based on a Sigma backend (SQLite) and do not use internal sigma to "something" conversion
-- **Zircolite** can export results to multiple format with using Jinja [templates](templates) : JSON, CSV, JSONL, Splunk, Elastic, Zinc, Timesketch...
+- MS Windows EVTX (EVTX, XML, and JSONL formats)
+- Auditd logs
+- Sysmon for Linux
+- EVTXtract
+- CSV and XML logs
 
-**Zircolite can be used directly in Python or you can use the binaries provided in [releases](https://github.com/wagga40/Zircolite/releases).**
+### Key Features
 
-**Documentation is [here](https://wagga40.github.io/Zircolite/) (dedicated site) or [here](docs) (repo directory).**
+- **Fast Processing**: Zircolite is relatively fast and can parse large datasets in just seconds.
+- **SIGMA Backend**: It is based on a SIGMA backend (SQLite) and does not use internal SIGMA-to-something conversion.
+- **Advanced Log Manipulation**: It can manipulate input logs by splitting fields and applying transformations, allowing for more flexible and powerful log analysis.
+- **Flexible Export**: Zircolite can export results to multiple formats using Jinja [templates](templates), including JSON, CSV, JSONL, Splunk, Elastic, Zinc, Timesketch, and more.
+
+**You can use Zircolite directly in Python or use the binaries provided in the [releases](https://github.com/wagga40/Zircolite/releases).** 
+
+**Documentation is available [here](https://wagga40.github.io/Zircolite/) (dedicated site) or [here](docs) (repo directory).**
 
 ## Requirements / Installation
 
-Python 3.8 minimum is required. If you only want to use base functionalities of Zircolite, you can install dependencies with : `pip3 install -r requirements.txt`. But `pip3 install -r requirements.full.txt` is strongly recommended.
+The project has only beek tested with Python 3.10. If you only want to use base functionnalities of Zircolite, you can install dependencies with : `pip3 install -r requirements.txt`. But `pip3 install -r requirements.full.txt` is strongly recommended.
 
 The use of [evtx_dump](https://github.com/omerbenamram/evtx) is **optional but required by default (because it is -for now- much faster)**, If you do not want to use it you have to use the `--noexternal` option. The tool is provided if you clone the Zircolite repository (the official repository is [here](https://github.com/omerbenamram/evtx)).
 
@@ -28,28 +37,41 @@ The use of [evtx_dump](https://github.com/omerbenamram/evtx) is **optional but r
 
 Check tutorials made by other (EN, SP and FR) [here](#tutorials).
 
-### EVTX files
+### EVTX files : 
+
+Help is available with:
+
+```shell
+python3 zircolite.py -h
+```
 
-Help is available with `zircolite.py -h`. If your EVTX files have the extension ".evtx" :
+If your EVTX files have the extension ".evtx" :
 
 ```shell
 # python3 zircolite.py --evtx <EVTX FOLDER or EVTX FILE> --ruleset <SIGMA RULESET> [--ruleset <OTHER RULESET>]
 python3 zircolite.py --evtx sysmon.evtx --ruleset rules/rules_windows_sysmon_pysigma.json
 ```
 
-The SYSMON ruleset employed is a default one, intended for analyzing logs from endpoints with SYSMON installed.
+- The `--evtx` argument can be a file or a folder. If it is a folder, all EVTX files in the current folder and subfolders will be selected.
+- The SYSMON ruleset used is a default one, intended for analyzing logs from endpoints with SYSMON installed.
 
-### Auditd / Sysmon for Linux / JSONL or NDJSON logs
+### Auditd / Sysmon for Linux / JSONL or NDJSON logs : 
 
 ```shell
+# For Auditd logs
 python3 zircolite.py --events auditd.log --ruleset rules/rules_linux.json --auditd
+# For Sysmon for Linux logs
 python3 zircolite.py --events sysmon.log --ruleset rules/rules_linux.json --sysmon4linux
-python3 zircolite.py --events <JSON_FOLDER or JSON_FILE> --ruleset rules/rules_windows_sysmon_pysigma.json --jsononly
+# For JSONL or NDJSON logs
+python3 zircolite.py --events <JSON_FOLDER_OR_FILE> --ruleset rules/rules_windows_sysmon_pysigma.json --jsononly
 ```
 
-:information_source: If you want to try the tool you can test with [EVTX-ATTACK-SAMPLES](https://github.com/sbousseaden/EVTX-ATTACK-SAMPLES) (EVTX Files).
+- The `--events` argument can be a file or a folder. If it is a folder, all event files in the current folder and subfolders will be selected.
+
+> [!TIP]
+> If you want to try the tool you can test with [EVTX-ATTACK-SAMPLES](https://github.com/sbousseaden/EVTX-ATTACK-SAMPLES) (EVTX Files).
 
-### Run with Docker
+### Running with Docker
 
 ```bash
 # Pull docker image
@@ -64,54 +86,67 @@ docker run --rm --tty \
     -r /case/input/a_sigma_rule.yml
 ```
 
-You can replace `$PWD` with the directory (absolute path only) where your logs and rules/rulesets are stored.
+- Replace `$PWD` with the directory (absolute path only) where your logs and rules/rulesets are stored.
 
 ### Updating default rulesets
 
 ```shell
 python3 zircolite.py -U
 ```
 
-:information_source: Please note these rulesets are provided to use Zircolite out-of-the-box but [you should generate your own rulesets](#why-you-should-build-your-own-rulesets) but they can be very noisy or slow. These auto-updated rulesets are available on the dedicated repository : [Zircolite-Rules](https://github.com/wagga40/Zircolite-Rules).
+> [!IMPORTANT]  
+> Please note these rulesets are provided to use Zircolite out-of-the-box, but [you should generate your own rulesets](#why-you-should-build-your-own-rulesets) as they can be very noisy or slow. These auto-updated rulesets are available in the dedicated repository: [Zircolite-Rules](https://github.com/wagga40/Zircolite-Rules).
 
 ## Docs
 
-Everything is [here](docs).
+Complete documentation is available [here](docs).
 
 ## Mini-Gui
 
-The Mini-GUI can be used totally offline, it allows the user to display and search results. You can automatically generate a Mini-Gui "package" with the `--package` option. To know how to use the Mini-GUI, check docs [here](docs/Advanced.md#mini-gui).
+The Mini-GUI can be used totally offline. It allows you to display and search results. You can automatically generate a Mini-GUI "package" with the `--package` option. To learn how to use the Mini-GUI, check the docs [here](docs/Advanced.md#mini-gui).
 
-### Detected events by Mitre Att&ck (c) techniques and criticality levels
+### Detected events by Mitre Att&ck (c) techniques and criticity levels
 
 ![](pics/gui.webp)
 
 ### Detected events Timeline
 
 ![](pics/gui-timeline.webp)
 
-### Detected events by Mitre Att&ck (c) techniques displayed on the Matrix
+### Detected events by Mitre Att&ck (c) techniques displayed on the Matrix 
 
 ![](pics/gui-matrix.webp)
 
 ## Tutorials, references and related projects
 
 ### Tutorials
 
-- (EN) [Russ McRee](https://holisticinfosec.io) has published a pretty good [tutorial](https://holisticinfosec.io/post/2021-09-28-zircolite/) on SIGMA and **Zircolite** in his [blog](https://holisticinfosec.io/post/2021-09-28-zircolite/)
+- **English**: [Russ McRee](https://holisticinfosec.io) has published a detailed [tutorial](https://holisticinfosec.io/post/2021-09-28-zircolite/) on SIGMA and Zircolite on his blog.
+
+- **Spanish**: **César Marín** has published a tutorial in Spanish [here](https://derechodelared.com/zircolite-ejecucion-de-reglas-sigma-en-ficheros-evtx/).
 
-- (SP) **César Marín** has published a tutorial in **spanish** [here](https://derechodelared.com/zircolite-ejecucion-de-reglas-sigma-en-ficheros-evtx/)
+- **French**: [IT-connect.fr](https://www.it-connect.fr/) has published [an extensive tutorial](https://www.it-connect.fr/) on Zircolite in French.
 
-- (FR) [IT-connect.fr](https://www.it-connect.fr/) has published [a very extensive tutorial](https://www.it-connect.fr/) in **French** on Zircolite
+- **French**: [IT-connect.fr](https://www.it-connect.fr/) has also published a [Hack the Box challenge Write-Up](https://www.it-connect.fr/hack-the-box-sherlocks-tracer-solution/) using Zircolite.
 
-### References
+### References 
 
-- [Florian Roth](https://github.com/Neo23x0/) cited **Zircolite** in his [**SIGMA Hall of fame**](https://github.com/Neo23x0/Talks/blob/master/Sigma_Hall_of_Fame_20211022.pdf) in its talk during the October 2021 EU ATT&CK Workshop in October 2021
-- Zircolite has been cited and used in the research work of the CIDRE team : [PWNJUSTSU - Website](https://pwnjutsu.irisa.fr) and [PWNJUSTSU - Academic paper](https://hal.inria.fr/hal-03694719/document)
-- Zircolite has been cited and presented during [JSAC 2023](https://jsac.jpcert.or.jp/archive/2023/pdf/JSAC2023_workshop_sigma_jp.pdf)
+- [Florian Roth](https://github.com/Neo23x0/) cited Zircolite in his [**SIGMA Hall of Fame**](https://github.com/Neo23x0/Talks/blob/master/Sigma_Hall_of_Fame_20211022.pdf) during his talk at the October 2021 EU ATT&CK Workshop.
+- Zircolite has been cited and presented during [JSAC 2023](https://jsac.jpcert.or.jp/archive/2023/pdf/JSAC2023_workshop_sigma_jp.pdf).
+- Zircolite has been cited and used in multiple research papers:
+  - **CIDRE Team**:
+    - [PWNJUTSU - Website](https://pwnjutsu.irisa.fr)
+    - [PWNJUTSU - Academic Paper](https://hal.inria.fr/hal-03694719/document)
+    - [CERBERE: Cybersecurity Exercise for Red and Blue Team Entertainment, Reproducibility](https://centralesupelec.hal.science/hal-04285565/file/CERBERE_final.pdf)
+  - **Universidad de la República**:
+    - [A Process Mining-Based Method for Attacker Profiling Using the MITRE ATT&CK Taxonomy](https://journals-sol.sbc.org.br/index.php/jisa/article/view/3902/2840)
+
+---
 
 ## License
 
-- All the **code** of the project is licensed under the [GNU Lesser General Public License](https://www.gnu.org/licenses/lgpl-3.0.en.html)
-- `evtx_dump` is under the MIT license
-- The rules are released under the [Detection Rule License (DRL)](https://github.com/SigmaHQ/Detection-Rule-License)
+- All the **code** of the project is licensed under the [GNU Lesser General Public License](https://www.gnu.org/licenses/lgpl-3.0.en.html).
+- `evtx_dump` is under the MIT license.
+- The rules are released under the [Detection Rule License (DRL) 1.0](https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md).
+
+---

config/fieldMappings.json

@@ -268,5 +268,68 @@
 		"Hash": {"separator":",", "equal":"="},
 		"Hashes": {"separator":",", "equal":"="},
 		"ConfigurationFileHash": {"separator":",", "equal":"="}
+	},
+	"transforms_enabled": true,
+	"transforms":{
+		"proctitle": [{
+			"info": "Proctitle HEX to ASCII",
+			"type": "python",
+            "code": "def transform(param):\n\treturn bytes.fromhex(param).decode('ascii').replace('\\x00',' ')",
+			"alias": false,
+			"alias_name": "",
+            "source_condition": ["auditd_input"],
+			"enabled": true
+		}],
+		"cmd": [{
+			"info": "Cmd HEX to ASCII",
+			"type": "python",
+            "code": "def transform(param):\n\treturn bytes.fromhex(param).decode('ascii').replace('\\x00',' ')",
+			"alias": false,
+			"alias_name": "",
+            "source_condition": ["auditd_input"],
+			"enabled": true
+		}],
+		"CommandLine": [
+			{
+				"info": "Base64 decoded CommandLine",
+				"type": "python",
+				"code": "\ndef transform(param):\n    decoded_values = []\n    concatenated_result = ''\n    data = param\n\n    base64_pattern = r'(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'\n    matches = re.findall(base64_pattern, data)\n    for match in matches:\n        decoded = base64.b64decode(match)\n        encoding = chardet.detect(decoded)['encoding']\n        if encoding and encoding in ['utf-8', 'ascii', 'utf-16le', 'ISO-8859-1']:\n            decoded = decoded.decode(encoding)\n            decoded = decoded.strip()\n            if decoded.isprintable() and len(decoded) > 10 :\n                decoded_values.append(decoded)\n    \n    concatenated_result = '|'.join(decoded_values)\n    return concatenated_result\n",
+				"alias": true,
+				"alias_name": "CommandLine_b64decoded",
+				"source_condition": ["evtx_input", "json_array_input", "json_input", "evtxtract_input", "db_input"],
+				"enabled": false
+			},
+			{
+				"info": "CommandLine credentials extraction - Regex by Practical Security Analytics - https://practicalsecurityanalytics.com/extracting-credentials-from-windows-logs/",
+                "type": "python",
+                "code": "\ndef transform(param):\n    import re\n    regex_patterns = [\n        r'net.+user\\s+(?P<username>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))\\s+(?P<password>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))',\n        r'net.+use\\s+(?P<share>\\\\\\\\\\S+)\\s+/USER:(?P<username>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))\\s+(?P<password>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))',\n        r'schtasks.+/U\\s+(?P<username>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+)).+/P\\s+(?P<password>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))',\n        r'wmic.+/user:\\s*(?P<username>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+)).+/password:\\s*(?P<password>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))',\n        r'psexec.+-u\\s+(?P<username>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+)).+-p\\s+(?P<password>(?:\"((?:\\\\.|[^\"\\\\])*)\")|(?:[^\\s\"]+))'\n    ]\n\n    matches = []\n    \n    for pattern in regex_patterns:\n        found = re.findall(pattern, param)\n        if len(found) > 0:\n            for match in list(found[0]):\n                if len(match) > 0:  \n                    matches.append(match) \n\n    concatenated_result = '|'.join(matches)\n    if concatenated_result == None:\n        return ''\n    return concatenated_result\n",
+                "alias": true,
+                "alias_name": "CommandLine_Extracted_Creds",
+				"source_condition": ["evtx_input", "json_array_input", "json_input", "evtxtract_input", "db_input"],
+                "enabled": false
+			}
+		],
+		"Payload": [
+			{
+				"info": "Base64 decoded Payload",
+				"type": "python",
+				"code": "\ndef transform(param):\n    decoded_values = []\n    concatenated_result = ''\n    data = param\n\n    base64_pattern = r'(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'\n    matches = re.findall(base64_pattern, data)\n    for match in matches:\n        decoded = base64.b64decode(match)\n        encoding = chardet.detect(decoded)['encoding']\n        if encoding and encoding in ['utf-8', 'ascii', 'utf-16le', 'ISO-8859-1']:\n            decoded = decoded.decode(encoding)\n            decoded = decoded.strip()\n            if decoded.isprintable() and len(decoded) > 10 :\n                decoded_values.append(decoded)\n    \n    concatenated_result = '|'.join(decoded_values)\n    return concatenated_result\n",
+				"alias": true,
+				"alias_name": "Payload_b64decoded",
+				"source_condition": ["evtx_input", "json_array_input", "json_input", "evtxtract_input", "db_input"],
+				"enabled": false
+			}
+		],
+		"ServiceFileName":[
+			{
+				"info": "Base64 decoded ServiceFileName",
+				"type": "python",
+				"code": "\ndef transform(param):\n    decoded_values = []\n    concatenated_result = ''\n    data = param\n\n    base64_pattern = r'(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'\n    matches = re.findall(base64_pattern, data)\n    for match in matches:\n        decoded = base64.b64decode(match)\n        encoding = chardet.detect(decoded)['encoding']\n        if encoding and encoding in ['utf-8', 'ascii', 'utf-16le', 'ISO-8859-1']:\n            decoded = decoded.decode(encoding)\n            decoded = decoded.strip()\n            if decoded.isprintable() and len(decoded) > 10 :\n                decoded_values.append(decoded)\n    \n    concatenated_result = '|'.join(decoded_values)\n    return concatenated_result\n",
+				"alias": true,
+				"alias_name": "ServiceFileName_b64decoded",
+				"source_condition": ["evtx_input", "json_array_input", "json_input", "evtxtract_input", "db_input"],
+				"enabled": false
+			}
+		]
 	}
 }