From 833ee1da71ba6a2cdeb29142eee85500ce83ce94 Mon Sep 17 00:00:00 2001
From: Yunchu Lee <yunchu.lee@intel.com>
Date: Thu, 14 Dec 2023 10:34:51 +0900
Subject: [PATCH] Enable fuzzing to the CLI (#1222)

---
 tests/fuzzing/__init__.py                |  3 ++
 tests/fuzzing/assets/cli_operations.dict | 15 ++++++++
 tests/fuzzing/cli_fuzzing.py             | 44 ++++++++++++++++++++++++
 tests/fuzzing/helper.py                  | 16 +++++++++
 tox.ini                                  | 14 ++++++++
 5 files changed, 92 insertions(+)
 create mode 100644 tests/fuzzing/__init__.py
 create mode 100644 tests/fuzzing/assets/cli_operations.dict
 create mode 100644 tests/fuzzing/cli_fuzzing.py
 create mode 100644 tests/fuzzing/helper.py

diff --git a/tests/fuzzing/__init__.py b/tests/fuzzing/__init__.py
new file mode 100644
index 0000000000..ff847f0120
--- /dev/null
+++ b/tests/fuzzing/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
diff --git a/tests/fuzzing/assets/cli_operations.dict b/tests/fuzzing/assets/cli_operations.dict
new file mode 100644
index 0000000000..2a64321718
--- /dev/null
+++ b/tests/fuzzing/assets/cli_operations.dict
@@ -0,0 +1,15 @@
+"convert"
+"detect"
+"compare"
+"dinfo"
+"download"
+"explain"
+"explore"
+"filter"
+"generate"
+"merge"
+"patch"
+"prune"
+"stats"
+"transform"
+"validate"
diff --git a/tests/fuzzing/cli_fuzzing.py b/tests/fuzzing/cli_fuzzing.py
new file mode 100644
index 0000000000..976f7ef62e
--- /dev/null
+++ b/tests/fuzzing/cli_fuzzing.py
@@ -0,0 +1,44 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+import sys
+
+import atheris
+from helper import FuzzingHelper
+
+from datumaro.__main__ import main as cli_main
+from datumaro.cli.util.errors import CliException
+
+
+@atheris.instrument_func
+def fuzz_datum(input_bytes):
+    # create a FuzzingHelper instance to get suitable data type from the randomly generated 'input_bytes'
+    helper = FuzzingHelper(input_bytes)
+    backup_argv = sys.argv
+
+    # get 'operation' arguments from 'input_bytes'
+    operation = helper.get_string()
+    sys.argv = ["datum", operation]
+    try:
+        _ = cli_main()
+    except SystemExit as e:
+        # argparser will throw SystemExit with code 2 when some required arguments are missing
+        if e.code != 2:
+            raise
+    except CliException:
+        pass
+    # some known exceptions can be catched here
+    finally:
+        sys.argv = backup_argv
+
+
+def main():
+    # 'sys.argv' used to passing options to atheris.Setup()
+    # available options can be found https://llvm.org/docs/LibFuzzer.html#options
+    atheris.Setup(sys.argv, fuzz_datum)
+    # Fuzz() will
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/fuzzing/helper.py b/tests/fuzzing/helper.py
new file mode 100644
index 0000000000..579f4da5c3
--- /dev/null
+++ b/tests/fuzzing/helper.py
@@ -0,0 +1,16 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+import atheris
+
+
+class FuzzingHelper(object):
+    """Helper to make required data from input_bytes for the fuzzing tests"""
+
+    def __init__(self, input_bytes):
+        """Init"""
+        self.provider = atheris.FuzzedDataProvider(input_bytes)
+
+    def get_string(self, byte_conut=256):
+        """Consume a string"""
+        return self.provider.ConsumeString(byte_conut)
diff --git a/tox.ini b/tox.ini
index a9539baff2..ac0346143e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,3 +88,17 @@ commands_pre =
     python -m pip uninstall pytest-stress -y
 commands =
     python -m pytest -v --csv={toxworkdir}/results-{envname}.csv -x {posargs:--loop 5}
+
+
+[testenv:fuzzing]
+deps=
+    {[testenv]deps}
+    atheris
+allowlist_externals =
+    bash
+commands_pre =
+    bash -c 'cargo -V; echo "cargo (rust) version checking exit code = $?"'
+commands =
+    coverage erase
+    - coverage run tests/fuzzing/cli_fuzzing.py {posargs:-dict=tests/fuzzing/assets/cli_operations.dict -artifact_prefix={toxworkdir}/ -print_final_stats=1 -atheris_runs=500000}
+    coverage report --precision=2