feat: add pytest for e2e testing

Author: namchuai

engine/e2e-test/main.py

@@ -0,0 +1,7 @@
+import pytest
+
+from test_cli_engine_list import TestCliEngineList
+from test_cli_engine_get import TestCliEngineGet
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

engine/e2e-test/test_cli_engine_get.py

@@ -0,0 +1,27 @@
+import platform
+
+import pytest
+from test_runner import run
+
+
+class TestCliEngineGet:
+    @pytest.mark.skipif(platform.system() != "Windows", reason="Windows-specific test")
+    def test_engines_list_run_successfully_on_windows(self):
+        # TODO: implement
+        assert 0 == 0
+
+    @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
+    def test_engines_get_run_successfully_on_macos(self):
+        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.llamacpp"])
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+
+    @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
+    def test_engines_get_llamacpp_on_macos(self):
+        exit_code, output, error = run("Get engine", ["engines", "get", "cortex.llamacpp"])
+        assert exit_code == 0, f"Get engine failed with error: {error}"
+        assert "Incompatible" not in output, f"cortex.llamacpp can only be Ready or Not Installed"
+
+    @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-specific test")
+    def test_engines_list_run_successfully_on_linux(self):
+        # TODO: implement
+        assert 0 == 0

engine/e2e-test/test_cli_engine_list.py

@@ -0,0 +1,24 @@
+import platform
+
+import pytest
+from test_runner import run
+
+
+class TestCliEngineList:
+    @pytest.mark.skipif(platform.system() != "Windows", reason="Windows-specific test")
+    def test_engines_list_run_successfully_on_windows(self):
+        exit_code, output, error = run("List engines", ["engines", "list"])
+        assert exit_code == 0, f"List engines failed with error: {error}"
+        assert "llama.cpp" in output
+
+    @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test")
+    def test_engines_list_run_successfully_on_macos(self):
+        exit_code, output, error = run("List engines", ["engines", "list"])
+        assert exit_code == 0, f"List engines failed with error: {error}"
+        assert "llama.cpp" in output
+
+    @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-specific test")
+    def test_engines_list_run_successfully_on_linux(self):
+        exit_code, output, error = run("List engines", ["engines", "list"])
+        assert exit_code == 0, f"List engines failed with error: {error}"
+        assert "llama.cpp" in output

engine/e2e-test/test_runner.py

@@ -0,0 +1,13 @@
+import platform
+import subprocess
+import os
+from typing import List, Tuple
+
+
+def run(name: str, arguments: List[str]):
+    if platform.system() == "Windows":
+        executable = "build\\cortex-cpp.exe"
+    else:
+        executable = "build/cortex-cpp"
+    result = subprocess.run([executable] + arguments, capture_output=True, text=True)
+    return result.returncode, result.stdout, result.stderr