diff --git a/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json b/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json deleted file mode 100644 index 476a6ec2ba..0000000000 --- a/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json +++ /dev/null @@ -1,364 +0,0 @@ -{ - "machine_info": { - "node": "ada-01", - "processor": "x86_64", - "machine": "x86_64", - "python_compiler": "GCC 9.4.0", - "python_implementation": "CPython", - "python_implementation_version": "3.8.10", - "python_version": "3.8.10", - "python_build": [ - "default", - "Jun 22 2022 20:18:18" - ], - "release": "5.11.0-25-generic", - "system": "Linux", - "cpu": { - "python_version": "3.8.10.final.0 (64 bit)", - "cpuinfo_version": [ - 8, - 0, - 0 - ], - "cpuinfo_version_string": "8.0.0", - "arch": "X86_64", - "bits": 64, - "count": 56, - "arch_string_raw": "x86_64", - "vendor_id_raw": "GenuineIntel", - "brand_raw": "Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz", - "hz_advertised_friendly": "2.6000 GHz", - "hz_actual_friendly": "1.2000 GHz", - "hz_advertised": [ - 2600000000, - 0 - ], - "hz_actual": [ - 1200000000, - 0 - ], - "stepping": 1, - "model": 79, - "family": 6, - "flags": [ - "3dnowprefetch", - "abm", - "acpi", - "adx", - "aes", - "aperfmperf", - "apic", - "arat", - "arch_perfmon", - "avx", - "avx2", - "bmi1", - "bmi2", - "bts", - "cat_l3", - "cdp_l3", - "clflush", - "cmov", - "constant_tsc", - "cpuid", - "cpuid_fault", - "cqm", - "cqm_llc", - "cqm_mbm_local", - "cqm_mbm_total", - "cqm_occup_llc", - "cx16", - "cx8", - "dca", - "de", - "ds_cpl", - "dtes64", - "dtherm", - "dts", - "epb", - "ept", - "ept_ad", - "erms", - "est", - "f16c", - "flexpriority", - "flush_l1d", - "fma", - "fpu", - "fsgsbase", - "fxsr", - "hle", - "ht", - "ibpb", - "ibrs", - "ida", - "intel_ppin", - "intel_pt", - "invpcid", - "invpcid_single", - "lahf_lm", - "lm", - "mca", - "mce", - "md_clear", - "mmx", - "monitor", - "movbe", - "msr", - "mtrr", - "nonstop_tsc", - "nopl", - "nx", - "osxsave", - "pae", - "pat", - "pbe", - "pcid", - "pclmulqdq", - "pdcm", - "pdpe1gb", - "pebs", - "pge", - "pln", - "pni", - "popcnt", - "pqe", - "pqm", - "pse", - "pse36", - "pti", - "pts", - "rdrand", - "rdrnd", - "rdseed", - "rdt_a", - "rdtscp", - "rep_good", - "rtm", - "sdbg", - "sep", - "smap", - "smep", - "smx", - "ss", - "ssbd", - "sse", - "sse2", - "sse4_1", - "sse4_2", - "ssse3", - "stibp", - "syscall", - "tm", - "tm2", - "tpr_shadow", - "tsc", - "tsc_adjust", - "tsc_deadline_timer", - "tscdeadline", - "vme", - "vmx", - "vnmi", - "vpid", - "x2apic", - "xsave", - "xsaveopt", - "xtopology", - "xtpr" - ], - "l3_cache_size": 36700160, - "l2_cache_size": "7 MiB", - "l1_data_cache_size": "896 KiB", - "l1_instruction_cache_size": "896 KiB", - "l2_cache_line_size": 256, - "l2_cache_associativity": 6 - } - }, - "commit_info": { - "id": "befa1483a94f1dfd86be7d1577242ee7db392e6a", - "time": "2022-09-23T12:28:19-04:00", - "author_time": "2022-09-23T21:58:19+05:30", - "dirty": true, - "project": "eva", - "branch": "master" - }, - "benchmarks": [ - { - "group": null, - "name": "test_should_run_pytorch_and_fastrcnn", - "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_fastrcnn", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 5.5531277088448405, - "max": 6.837524174712598, - "mean": 6.083840253204107, - "stddev": 0.5935614321232195, - "rounds": 5, - "median": 5.850675313733518, - "iqr": 1.0749499255325645, - "q1": 5.57805929845199, - "q3": 6.653009223984554, - "iqr_outliers": 0, - "stddev_outliers": 1, - "outliers": "1;0", - "ld15iqr": 5.5531277088448405, - "hd15iqr": 6.837524174712598, - "ops": 0.16436986481907398, - "total": 30.419201266020536, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_should_run_pytorch_and_ssd", - "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ssd", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 2.6702366108074784, - "max": 3.0154475858435035, - "mean": 2.844165700674057, - "stddev": 0.15508371222011108, - "rounds": 5, - "median": 2.842389179393649, - "iqr": 0.28922474291175604, - "q1": 2.700881325872615, - "q3": 2.990106068784371, - "iqr_outliers": 0, - "stddev_outliers": 2, - "outliers": "2;0", - "ld15iqr": 2.6702366108074784, - "hd15iqr": 3.0154475858435035, - "ops": 0.3515969550448497, - "total": 14.220828503370285, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_should_run_pytorch_and_facenet", - "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_facenet", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 0.3764120349660516, - "max": 0.4438629001379013, - "mean": 0.4140555374324322, - "stddev": 0.02878907926075933, - "rounds": 5, - "median": 0.42684925626963377, - "iqr": 0.047489251708611846, - "q1": 0.3874723019544035, - "q3": 0.43496155366301537, - "iqr_outliers": 0, - "stddev_outliers": 2, - "outliers": "2;0", - "ld15iqr": 0.3764120349660516, - "hd15iqr": 0.4438629001379013, - "ops": 2.415134950738789, - "total": 2.070277687162161, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_should_run_pytorch_and_ocr", - "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ocr", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 2.5097263120114803, - "max": 2.5920280972495675, - "mean": 2.5351627355441453, - "stddev": 0.033354057821116334, - "rounds": 5, - "median": 2.530061839148402, - "iqr": 0.035450790310278535, - "q1": 2.51149294199422, - "q3": 2.5469437323044986, - "iqr_outliers": 0, - "stddev_outliers": 1, - "outliers": "1;0", - "ld15iqr": 2.5097263120114803, - "hd15iqr": 2.5920280972495675, - "ops": 0.3944519955186864, - "total": 12.675813677720726, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_should_run_pytorch_and_resnet50", - "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_resnet50", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 1.4770564455538988, - "max": 1.5711254822090268, - "mean": 1.5154685793444513, - "stddev": 0.04252216822718916, - "rounds": 5, - "median": 1.4958880990743637, - "iqr": 0.07408787799067795, - "q1": 1.4814561281818897, - "q3": 1.5555440061725676, - "iqr_outliers": 0, - "stddev_outliers": 1, - "outliers": "1;0", - "ld15iqr": 1.4770564455538988, - "hd15iqr": 1.5711254822090268, - "ops": 0.6598619157333975, - "total": 7.577342896722257, - "iterations": 1 - } - } - ], - "datetime": "2022-09-23T19:03:10.673591", - "version": "3.4.1" -} \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index bf6deabc6b..33ea6f9e7f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,4 +5,3 @@ filterwarnings = markers = torchtest: marks tests that rely on torch (deselect with '-m "not torchtest"') - benchmark: marks tests that need to be benchmarked (deselect with '-m "not benchmark"') diff --git a/setup.py b/setup.py index 5ad953eff8..abbbd87ff2 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,6 @@ def read(path, encoding="utf-8"): ] benchmark_libs = [ - "pytest-benchmark" ] doc_libs = [ diff --git a/test/integration_tests/conftest.py b/test/integration_tests/conftest.py deleted file mode 100644 index 2e571e1434..0000000000 --- a/test/integration_tests/conftest.py +++ /dev/null @@ -1,42 +0,0 @@ -# coding=utf-8 -# Copyright 2018-2022 EVA -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs - -import mock -import pytest - -from eva.catalog.catalog_manager import CatalogManager -from eva.server.command_handler import execute_query_fetch_all - - -@pytest.fixture(scope="session") -def tearDownClass(): - yield None - file_remove("ua_detrac.mp4") - - -@pytest.fixture(scope="session") -def setup_pytorch_tests(): - - CatalogManager().reset() - copy_sample_videos_to_upload_dir() - query = """LOAD FILE 'ua_detrac.mp4' - INTO MyVideo;""" - execute_query_fetch_all(query) - query = """LOAD FILE 'mnist.mp4' - INTO MNIST;""" - execute_query_fetch_all(query) - load_inbuilt_udfs() diff --git a/test/integration_tests/test_benchmark_pytorch.py b/test/integration_tests/test_benchmark_pytorch.py deleted file mode 100644 index ac28f805e0..0000000000 --- a/test/integration_tests/test_benchmark_pytorch.py +++ /dev/null @@ -1,119 +0,0 @@ -import pytest -import sys -from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs - -import mock -import pytest - -from eva.catalog.catalog_manager import CatalogManager -from eva.server.command_handler import execute_query_fetch_all - - -@pytest.mark.torchtest -@pytest.mark.benchmark -def test_should_run_pytorch_and_fastrcnn(benchmark): - - select_query = """SELECT FastRCNNObjectDetector(data) FROM MyVideo - WHERE id < 5;""" - actual_batch = benchmark(execute_query_fetch_all, select_query) - assert(len(actual_batch)==5) - - -@pytest.mark.torchtest -@pytest.mark.benchmark -def test_should_run_pytorch_and_ssd(benchmark): - create_udf_query = """CREATE UDF IF NOT EXISTS SSDObjectDetector - INPUT (Frame_Array NDARRAY UINT8(3, 256, 256)) - OUTPUT (label NDARRAY STR(10)) - TYPE Classification - IMPL 'eva/udfs/ssd_object_detector.py'; - """ - execute_query_fetch_all(create_udf_query) - - select_query = """SELECT SSDObjectDetector(data) FROM MyVideo - WHERE id < 5;""" - actual_batch = benchmark(execute_query_fetch_all, select_query) - assert(len(actual_batch)==5) - # non-trivial test case - res = actual_batch.frames - for idx in res.index: - assert("car" in res["ssdobjectdetector.label"][idx]) - - -@pytest.mark.torchtest -@pytest.mark.benchmark -def test_should_run_pytorch_and_facenet(benchmark): - create_udf_query = """CREATE UDF IF NOT EXISTS FaceDetector - INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) - OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4), - scores NDARRAY FLOAT32(ANYDIM)) - TYPE FaceDetection - IMPL 'eva/udfs/face_detector.py'; - """ - execute_query_fetch_all(create_udf_query) - - select_query = """SELECT FaceDetector(data) FROM MyVideo - WHERE id < 5;""" - actual_batch = benchmark(execute_query_fetch_all, select_query) - assert(len(actual_batch)==5) - - - - -@pytest.mark.torchtest -@pytest.mark.benchmark -def test_should_run_pytorch_and_ocr(benchmark): - create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor - INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) - OUTPUT (labels NDARRAY STR(10), - bboxes NDARRAY FLOAT32(ANYDIM, 4), - scores NDARRAY FLOAT32(ANYDIM)) - TYPE OCRExtraction - IMPL 'eva/udfs/ocr_extractor.py'; - """ - execute_query_fetch_all(create_udf_query) - - select_query = """SELECT OCRExtractor(data) FROM MNIST - WHERE id >= 150 AND id < 155;""" - actual_batch = benchmark(execute_query_fetch_all, select_query) - assert(len(actual_batch)==5) - - # non-trivial test case for MNIST - res = actual_batch.frames - assert(res["ocrextractor.labels"][0][0] == "4") - assert(res["ocrextractor.scores"][2][0] > 0.9) - -@pytest.mark.torchtest -@pytest.mark.benchmark -def test_should_run_pytorch_and_resnet50(benchmark): - create_udf_query = """CREATE UDF IF NOT EXISTS FeatureExtractor - INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) - OUTPUT (features NDARRAY FLOAT32(ANYDIM)) - TYPE Classification - IMPL 'eva/udfs/feature_extractor.py'; - """ - execute_query_fetch_all(create_udf_query) - - select_query = """SELECT FeatureExtractor(data) FROM MyVideo - WHERE id < 5;""" - actual_batch = benchmark(execute_query_fetch_all, select_query) - assert(len(actual_batch)==5) - - # non-trivial test case for Resnet50 - res = actual_batch.frames - assert(res["featureextractor.features"][0].shape==(1, 2048)) - assert(res["featureextractor.features"][0][0][0] > 0.3) - -def test_should_raise_import_error_with_missing_torch(benchmark): - with pytest.raises(ImportError): - with mock.patch.dict(sys.modules, {"torch": None}): - from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 - - pass - -def test_should_raise_import_error_with_missing_torchvision(benchmark): - with pytest.raises(ImportError): - with mock.patch.dict(sys.modules, {"torchvision.transforms": None}): - from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 - - pass