diff --git a/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json b/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json new file mode 100644 index 0000000000..476a6ec2ba --- /dev/null +++ b/.benchmarks/Linux-CPython-3.8-64bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json @@ -0,0 +1,364 @@ +{ + "machine_info": { + "node": "ada-01", + "processor": "x86_64", + "machine": "x86_64", + "python_compiler": "GCC 9.4.0", + "python_implementation": "CPython", + "python_implementation_version": "3.8.10", + "python_version": "3.8.10", + "python_build": [ + "default", + "Jun 22 2022 20:18:18" + ], + "release": "5.11.0-25-generic", + "system": "Linux", + "cpu": { + "python_version": "3.8.10.final.0 (64 bit)", + "cpuinfo_version": [ + 8, + 0, + 0 + ], + "cpuinfo_version_string": "8.0.0", + "arch": "X86_64", + "bits": 64, + "count": 56, + "arch_string_raw": "x86_64", + "vendor_id_raw": "GenuineIntel", + "brand_raw": "Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz", + "hz_advertised_friendly": "2.6000 GHz", + "hz_actual_friendly": "1.2000 GHz", + "hz_advertised": [ + 2600000000, + 0 + ], + "hz_actual": [ + 1200000000, + 0 + ], + "stepping": 1, + "model": 79, + "family": 6, + "flags": [ + "3dnowprefetch", + "abm", + "acpi", + "adx", + "aes", + "aperfmperf", + "apic", + "arat", + "arch_perfmon", + "avx", + "avx2", + "bmi1", + "bmi2", + "bts", + "cat_l3", + "cdp_l3", + "clflush", + "cmov", + "constant_tsc", + "cpuid", + "cpuid_fault", + "cqm", + "cqm_llc", + "cqm_mbm_local", + "cqm_mbm_total", + "cqm_occup_llc", + "cx16", + "cx8", + "dca", + "de", + "ds_cpl", + "dtes64", + "dtherm", + "dts", + "epb", + "ept", + "ept_ad", + "erms", + "est", + "f16c", + "flexpriority", + "flush_l1d", + "fma", + "fpu", + "fsgsbase", + "fxsr", + "hle", + "ht", + "ibpb", + "ibrs", + "ida", + "intel_ppin", + "intel_pt", + "invpcid", + "invpcid_single", + "lahf_lm", + "lm", + "mca", + "mce", + "md_clear", + "mmx", + "monitor", + "movbe", + "msr", + "mtrr", + "nonstop_tsc", + "nopl", + "nx", + "osxsave", + "pae", + "pat", + "pbe", + "pcid", + "pclmulqdq", + "pdcm", + "pdpe1gb", + "pebs", + "pge", + "pln", + "pni", + "popcnt", + "pqe", + "pqm", + "pse", + "pse36", + "pti", + "pts", + "rdrand", + "rdrnd", + "rdseed", + "rdt_a", + "rdtscp", + "rep_good", + "rtm", + "sdbg", + "sep", + "smap", + "smep", + "smx", + "ss", + "ssbd", + "sse", + "sse2", + "sse4_1", + "sse4_2", + "ssse3", + "stibp", + "syscall", + "tm", + "tm2", + "tpr_shadow", + "tsc", + "tsc_adjust", + "tsc_deadline_timer", + "tscdeadline", + "vme", + "vmx", + "vnmi", + "vpid", + "x2apic", + "xsave", + "xsaveopt", + "xtopology", + "xtpr" + ], + "l3_cache_size": 36700160, + "l2_cache_size": "7 MiB", + "l1_data_cache_size": "896 KiB", + "l1_instruction_cache_size": "896 KiB", + "l2_cache_line_size": 256, + "l2_cache_associativity": 6 + } + }, + "commit_info": { + "id": "befa1483a94f1dfd86be7d1577242ee7db392e6a", + "time": "2022-09-23T12:28:19-04:00", + "author_time": "2022-09-23T21:58:19+05:30", + "dirty": true, + "project": "eva", + "branch": "master" + }, + "benchmarks": [ + { + "group": null, + "name": "test_should_run_pytorch_and_fastrcnn", + "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_fastrcnn", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 5.5531277088448405, + "max": 6.837524174712598, + "mean": 6.083840253204107, + "stddev": 0.5935614321232195, + "rounds": 5, + "median": 5.850675313733518, + "iqr": 1.0749499255325645, + "q1": 5.57805929845199, + "q3": 6.653009223984554, + "iqr_outliers": 0, + "stddev_outliers": 1, + "outliers": "1;0", + "ld15iqr": 5.5531277088448405, + "hd15iqr": 6.837524174712598, + "ops": 0.16436986481907398, + "total": 30.419201266020536, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_should_run_pytorch_and_ssd", + "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ssd", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 2.6702366108074784, + "max": 3.0154475858435035, + "mean": 2.844165700674057, + "stddev": 0.15508371222011108, + "rounds": 5, + "median": 2.842389179393649, + "iqr": 0.28922474291175604, + "q1": 2.700881325872615, + "q3": 2.990106068784371, + "iqr_outliers": 0, + "stddev_outliers": 2, + "outliers": "2;0", + "ld15iqr": 2.6702366108074784, + "hd15iqr": 3.0154475858435035, + "ops": 0.3515969550448497, + "total": 14.220828503370285, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_should_run_pytorch_and_facenet", + "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_facenet", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.3764120349660516, + "max": 0.4438629001379013, + "mean": 0.4140555374324322, + "stddev": 0.02878907926075933, + "rounds": 5, + "median": 0.42684925626963377, + "iqr": 0.047489251708611846, + "q1": 0.3874723019544035, + "q3": 0.43496155366301537, + "iqr_outliers": 0, + "stddev_outliers": 2, + "outliers": "2;0", + "ld15iqr": 0.3764120349660516, + "hd15iqr": 0.4438629001379013, + "ops": 2.415134950738789, + "total": 2.070277687162161, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_should_run_pytorch_and_ocr", + "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ocr", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 2.5097263120114803, + "max": 2.5920280972495675, + "mean": 2.5351627355441453, + "stddev": 0.033354057821116334, + "rounds": 5, + "median": 2.530061839148402, + "iqr": 0.035450790310278535, + "q1": 2.51149294199422, + "q3": 2.5469437323044986, + "iqr_outliers": 0, + "stddev_outliers": 1, + "outliers": "1;0", + "ld15iqr": 2.5097263120114803, + "hd15iqr": 2.5920280972495675, + "ops": 0.3944519955186864, + "total": 12.675813677720726, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_should_run_pytorch_and_resnet50", + "fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_resnet50", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 1.4770564455538988, + "max": 1.5711254822090268, + "mean": 1.5154685793444513, + "stddev": 0.04252216822718916, + "rounds": 5, + "median": 1.4958880990743637, + "iqr": 0.07408787799067795, + "q1": 1.4814561281818897, + "q3": 1.5555440061725676, + "iqr_outliers": 0, + "stddev_outliers": 1, + "outliers": "1;0", + "ld15iqr": 1.4770564455538988, + "hd15iqr": 1.5711254822090268, + "ops": 0.6598619157333975, + "total": 7.577342896722257, + "iterations": 1 + } + } + ], + "datetime": "2022-09-23T19:03:10.673591", + "version": "3.4.1" +} \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 33ea6f9e7f..bf6deabc6b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,4 @@ filterwarnings = markers = torchtest: marks tests that rely on torch (deselect with '-m "not torchtest"') + benchmark: marks tests that need to be benchmarked (deselect with '-m "not benchmark"') diff --git a/setup.py b/setup.py index abbbd87ff2..5ad953eff8 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ def read(path, encoding="utf-8"): ] benchmark_libs = [ + "pytest-benchmark" ] doc_libs = [ diff --git a/test/integration_tests/conftest.py b/test/integration_tests/conftest.py new file mode 100644 index 0000000000..2e571e1434 --- /dev/null +++ b/test/integration_tests/conftest.py @@ -0,0 +1,42 @@ +# coding=utf-8 +# Copyright 2018-2022 EVA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs + +import mock +import pytest + +from eva.catalog.catalog_manager import CatalogManager +from eva.server.command_handler import execute_query_fetch_all + + +@pytest.fixture(scope="session") +def tearDownClass(): + yield None + file_remove("ua_detrac.mp4") + + +@pytest.fixture(scope="session") +def setup_pytorch_tests(): + + CatalogManager().reset() + copy_sample_videos_to_upload_dir() + query = """LOAD FILE 'ua_detrac.mp4' + INTO MyVideo;""" + execute_query_fetch_all(query) + query = """LOAD FILE 'mnist.mp4' + INTO MNIST;""" + execute_query_fetch_all(query) + load_inbuilt_udfs() diff --git a/test/integration_tests/test_benchmark_pytorch.py b/test/integration_tests/test_benchmark_pytorch.py new file mode 100644 index 0000000000..ac28f805e0 --- /dev/null +++ b/test/integration_tests/test_benchmark_pytorch.py @@ -0,0 +1,119 @@ +import pytest +import sys +from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs + +import mock +import pytest + +from eva.catalog.catalog_manager import CatalogManager +from eva.server.command_handler import execute_query_fetch_all + + +@pytest.mark.torchtest +@pytest.mark.benchmark +def test_should_run_pytorch_and_fastrcnn(benchmark): + + select_query = """SELECT FastRCNNObjectDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert(len(actual_batch)==5) + + +@pytest.mark.torchtest +@pytest.mark.benchmark +def test_should_run_pytorch_and_ssd(benchmark): + create_udf_query = """CREATE UDF IF NOT EXISTS SSDObjectDetector + INPUT (Frame_Array NDARRAY UINT8(3, 256, 256)) + OUTPUT (label NDARRAY STR(10)) + TYPE Classification + IMPL 'eva/udfs/ssd_object_detector.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT SSDObjectDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert(len(actual_batch)==5) + # non-trivial test case + res = actual_batch.frames + for idx in res.index: + assert("car" in res["ssdobjectdetector.label"][idx]) + + +@pytest.mark.torchtest +@pytest.mark.benchmark +def test_should_run_pytorch_and_facenet(benchmark): + create_udf_query = """CREATE UDF IF NOT EXISTS FaceDetector + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4), + scores NDARRAY FLOAT32(ANYDIM)) + TYPE FaceDetection + IMPL 'eva/udfs/face_detector.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT FaceDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert(len(actual_batch)==5) + + + + +@pytest.mark.torchtest +@pytest.mark.benchmark +def test_should_run_pytorch_and_ocr(benchmark): + create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (labels NDARRAY STR(10), + bboxes NDARRAY FLOAT32(ANYDIM, 4), + scores NDARRAY FLOAT32(ANYDIM)) + TYPE OCRExtraction + IMPL 'eva/udfs/ocr_extractor.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT OCRExtractor(data) FROM MNIST + WHERE id >= 150 AND id < 155;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert(len(actual_batch)==5) + + # non-trivial test case for MNIST + res = actual_batch.frames + assert(res["ocrextractor.labels"][0][0] == "4") + assert(res["ocrextractor.scores"][2][0] > 0.9) + +@pytest.mark.torchtest +@pytest.mark.benchmark +def test_should_run_pytorch_and_resnet50(benchmark): + create_udf_query = """CREATE UDF IF NOT EXISTS FeatureExtractor + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (features NDARRAY FLOAT32(ANYDIM)) + TYPE Classification + IMPL 'eva/udfs/feature_extractor.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT FeatureExtractor(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert(len(actual_batch)==5) + + # non-trivial test case for Resnet50 + res = actual_batch.frames + assert(res["featureextractor.features"][0].shape==(1, 2048)) + assert(res["featureextractor.features"][0][0][0] > 0.3) + +def test_should_raise_import_error_with_missing_torch(benchmark): + with pytest.raises(ImportError): + with mock.patch.dict(sys.modules, {"torch": None}): + from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 + + pass + +def test_should_raise_import_error_with_missing_torchvision(benchmark): + with pytest.raises(ImportError): + with mock.patch.dict(sys.modules, {"torchvision.transforms": None}): + from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 + + pass