Enable fuzzing to the CLI (#1222)

tests/fuzzing/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT

tests/fuzzing/assets/cli_operations.dict

@@ -0,0 +1,15 @@
+"convert"
+"detect"
+"compare"
+"dinfo"
+"download"
+"explain"
+"explore"
+"filter"
+"generate"
+"merge"
+"patch"
+"prune"
+"stats"
+"transform"
+"validate"

tests/fuzzing/cli_fuzzing.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+import sys
+
+import atheris
+from helper import FuzzingHelper
+
+from datumaro.__main__ import main as cli_main
+from datumaro.cli.util.errors import CliException
+
+
+@atheris.instrument_func
+def fuzz_datum(input_bytes):
+    # create a FuzzingHelper instance to get suitable data type from the randomly generated 'input_bytes'
+    helper = FuzzingHelper(input_bytes)
+    backup_argv = sys.argv
+
+    # get 'operation' arguments from 'input_bytes'
+    operation = helper.get_string()
+    sys.argv = ["datum", operation]
+    try:
+        _ = cli_main()
+    except SystemExit as e:
+        # argparser will throw SystemExit with code 2 when some required arguments are missing
+        if e.code != 2:
+            raise
+    except CliException:
+        pass
+    # some known exceptions can be catched here
+    finally:
+        sys.argv = backup_argv
+
+
+def main():
+    # 'sys.argv' used to passing options to atheris.Setup()
+    # available options can be found https://llvm.org/docs/LibFuzzer.html#options
+    atheris.Setup(sys.argv, fuzz_datum)
+    # Fuzz() will
+    atheris.Fuzz()
+
+
+if __name__ == "__main__":
+    main()

tests/fuzzing/helper.py

@@ -0,0 +1,16 @@
+# Copyright (C) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+import atheris
+
+
+class FuzzingHelper(object):
+    """Helper to make required data from input_bytes for the fuzzing tests"""
+
+    def __init__(self, input_bytes):
+        """Init"""
+        self.provider = atheris.FuzzedDataProvider(input_bytes)
+
+    def get_string(self, byte_conut=256):
+        """Consume a string"""
+        return self.provider.ConsumeString(byte_conut)

tox.ini

@@ -88,3 +88,17 @@ commands_pre =
     python -m pip uninstall pytest-stress -y
 commands =
     python -m pytest -v --csv={toxworkdir}/results-{envname}.csv -x {posargs:--loop 5}
+
+
+[testenv:fuzzing]
+deps=
+    {[testenv]deps}
+    atheris
+allowlist_externals =
+    bash
+commands_pre =
+    bash -c 'cargo -V; echo "cargo (rust) version checking exit code = $?"'
+commands =
+    coverage erase
+    - coverage run tests/fuzzing/cli_fuzzing.py {posargs:-dict=tests/fuzzing/assets/cli_operations.dict -artifact_prefix={toxworkdir}/ -print_final_stats=1 -atheris_runs=500000}
+    coverage report --precision=2