Skip to content

Commit

Permalink
Benchmarking for pytorch tests using Pytest benchmark (#396)
Browse files Browse the repository at this point in the history 
* Benchmarking for pytorch tests complete

* linting and markers donef

Co-authored-by: Gaurav Tarlok Kakkar <gaurav21776@gmail.com>
Co-authored-by: Kaushik <108662423+kaushikravichandran@users.noreply.github.com>
  • Loading branch information
@aavhad1910 @gaurav274 @kaushikravichandran
3 people authored Nov 8, 2022
1 parent 821d59d commit d593ae9
Show file tree
Hide file tree
Showing 5 changed files with 527 additions and 0 deletions.
364 changes: 364 additions & 0 deletions ...bit/0001_befa1483a94f1dfd86be7d1577242ee7db392e6a_20220923_190122_uncommited-changes.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,364 @@
{
"machine_info": {
"node": "ada-01",
"processor": "x86_64",
"machine": "x86_64",
"python_compiler": "GCC 9.4.0",
"python_implementation": "CPython",
"python_implementation_version": "3.8.10",
"python_version": "3.8.10",
"python_build": [
"default",
"Jun 22 2022 20:18:18"
],
"release": "5.11.0-25-generic",
"system": "Linux",
"cpu": {
"python_version": "3.8.10.final.0 (64 bit)",
"cpuinfo_version": [
8,
0,
0
],
"cpuinfo_version_string": "8.0.0",
"arch": "X86_64",
"bits": 64,
"count": 56,
"arch_string_raw": "x86_64",
"vendor_id_raw": "GenuineIntel",
"brand_raw": "Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz",
"hz_advertised_friendly": "2.6000 GHz",
"hz_actual_friendly": "1.2000 GHz",
"hz_advertised": [
2600000000,
0
],
"hz_actual": [
1200000000,
0
],
"stepping": 1,
"model": 79,
"family": 6,
"flags": [
"3dnowprefetch",
"abm",
"acpi",
"adx",
"aes",
"aperfmperf",
"apic",
"arat",
"arch_perfmon",
"avx",
"avx2",
"bmi1",
"bmi2",
"bts",
"cat_l3",
"cdp_l3",
"clflush",
"cmov",
"constant_tsc",
"cpuid",
"cpuid_fault",
"cqm",
"cqm_llc",
"cqm_mbm_local",
"cqm_mbm_total",
"cqm_occup_llc",
"cx16",
"cx8",
"dca",
"de",
"ds_cpl",
"dtes64",
"dtherm",
"dts",
"epb",
"ept",
"ept_ad",
"erms",
"est",
"f16c",
"flexpriority",
"flush_l1d",
"fma",
"fpu",
"fsgsbase",
"fxsr",
"hle",
"ht",
"ibpb",
"ibrs",
"ida",
"intel_ppin",
"intel_pt",
"invpcid",
"invpcid_single",
"lahf_lm",
"lm",
"mca",
"mce",
"md_clear",
"mmx",
"monitor",
"movbe",
"msr",
"mtrr",
"nonstop_tsc",
"nopl",
"nx",
"osxsave",
"pae",
"pat",
"pbe",
"pcid",
"pclmulqdq",
"pdcm",
"pdpe1gb",
"pebs",
"pge",
"pln",
"pni",
"popcnt",
"pqe",
"pqm",
"pse",
"pse36",
"pti",
"pts",
"rdrand",
"rdrnd",
"rdseed",
"rdt_a",
"rdtscp",
"rep_good",
"rtm",
"sdbg",
"sep",
"smap",
"smep",
"smx",
"ss",
"ssbd",
"sse",
"sse2",
"sse4_1",
"sse4_2",
"ssse3",
"stibp",
"syscall",
"tm",
"tm2",
"tpr_shadow",
"tsc",
"tsc_adjust",
"tsc_deadline_timer",
"tscdeadline",
"vme",
"vmx",
"vnmi",
"vpid",
"x2apic",
"xsave",
"xsaveopt",
"xtopology",
"xtpr"
],
"l3_cache_size": 36700160,
"l2_cache_size": "7 MiB",
"l1_data_cache_size": "896 KiB",
"l1_instruction_cache_size": "896 KiB",
"l2_cache_line_size": 256,
"l2_cache_associativity": 6
}
},
"commit_info": {
"id": "befa1483a94f1dfd86be7d1577242ee7db392e6a",
"time": "2022-09-23T12:28:19-04:00",
"author_time": "2022-09-23T21:58:19+05:30",
"dirty": true,
"project": "eva",
"branch": "master"
},
"benchmarks": [
{
"group": null,
"name": "test_should_run_pytorch_and_fastrcnn",
"fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_fastrcnn",
"params": null,
"param": null,
"extra_info": {},
"options": {
"disable_gc": false,
"timer": "perf_counter",
"min_rounds": 5,
"max_time": 1.0,
"min_time": 5e-06,
"warmup": false
},
"stats": {
"min": 5.5531277088448405,
"max": 6.837524174712598,
"mean": 6.083840253204107,
"stddev": 0.5935614321232195,
"rounds": 5,
"median": 5.850675313733518,
"iqr": 1.0749499255325645,
"q1": 5.57805929845199,
"q3": 6.653009223984554,
"iqr_outliers": 0,
"stddev_outliers": 1,
"outliers": "1;0",
"ld15iqr": 5.5531277088448405,
"hd15iqr": 6.837524174712598,
"ops": 0.16436986481907398,
"total": 30.419201266020536,
"iterations": 1
}
},
{
"group": null,
"name": "test_should_run_pytorch_and_ssd",
"fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ssd",
"params": null,
"param": null,
"extra_info": {},
"options": {
"disable_gc": false,
"timer": "perf_counter",
"min_rounds": 5,
"max_time": 1.0,
"min_time": 5e-06,
"warmup": false
},
"stats": {
"min": 2.6702366108074784,
"max": 3.0154475858435035,
"mean": 2.844165700674057,
"stddev": 0.15508371222011108,
"rounds": 5,
"median": 2.842389179393649,
"iqr": 0.28922474291175604,
"q1": 2.700881325872615,
"q3": 2.990106068784371,
"iqr_outliers": 0,
"stddev_outliers": 2,
"outliers": "2;0",
"ld15iqr": 2.6702366108074784,
"hd15iqr": 3.0154475858435035,
"ops": 0.3515969550448497,
"total": 14.220828503370285,
"iterations": 1
}
},
{
"group": null,
"name": "test_should_run_pytorch_and_facenet",
"fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_facenet",
"params": null,
"param": null,
"extra_info": {},
"options": {
"disable_gc": false,
"timer": "perf_counter",
"min_rounds": 5,
"max_time": 1.0,
"min_time": 5e-06,
"warmup": false
},
"stats": {
"min": 0.3764120349660516,
"max": 0.4438629001379013,
"mean": 0.4140555374324322,
"stddev": 0.02878907926075933,
"rounds": 5,
"median": 0.42684925626963377,
"iqr": 0.047489251708611846,
"q1": 0.3874723019544035,
"q3": 0.43496155366301537,
"iqr_outliers": 0,
"stddev_outliers": 2,
"outliers": "2;0",
"ld15iqr": 0.3764120349660516,
"hd15iqr": 0.4438629001379013,
"ops": 2.415134950738789,
"total": 2.070277687162161,
"iterations": 1
}
},
{
"group": null,
"name": "test_should_run_pytorch_and_ocr",
"fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_ocr",
"params": null,
"param": null,
"extra_info": {},
"options": {
"disable_gc": false,
"timer": "perf_counter",
"min_rounds": 5,
"max_time": 1.0,
"min_time": 5e-06,
"warmup": false
},
"stats": {
"min": 2.5097263120114803,
"max": 2.5920280972495675,
"mean": 2.5351627355441453,
"stddev": 0.033354057821116334,
"rounds": 5,
"median": 2.530061839148402,
"iqr": 0.035450790310278535,
"q1": 2.51149294199422,
"q3": 2.5469437323044986,
"iqr_outliers": 0,
"stddev_outliers": 1,
"outliers": "1;0",
"ld15iqr": 2.5097263120114803,
"hd15iqr": 2.5920280972495675,
"ops": 0.3944519955186864,
"total": 12.675813677720726,
"iterations": 1
}
},
{
"group": null,
"name": "test_should_run_pytorch_and_resnet50",
"fullname": "test/integration_tests/test_benchmark_pytorch.py::test_should_run_pytorch_and_resnet50",
"params": null,
"param": null,
"extra_info": {},
"options": {
"disable_gc": false,
"timer": "perf_counter",
"min_rounds": 5,
"max_time": 1.0,
"min_time": 5e-06,
"warmup": false
},
"stats": {
"min": 1.4770564455538988,
"max": 1.5711254822090268,
"mean": 1.5154685793444513,
"stddev": 0.04252216822718916,
"rounds": 5,
"median": 1.4958880990743637,
"iqr": 0.07408787799067795,
"q1": 1.4814561281818897,
"q3": 1.5555440061725676,
"iqr_outliers": 0,
"stddev_outliers": 1,
"outliers": "1;0",
"ld15iqr": 1.4770564455538988,
"hd15iqr": 1.5711254822090268,
"ops": 0.6598619157333975,
"total": 7.577342896722257,
"iterations": 1
}
}
],
"datetime": "2022-09-23T19:03:10.673591",
"version": "3.4.1"
}
1 change: 1 addition & 0 deletions pytest.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ filterwarnings =

markers =
torchtest: marks tests that rely on torch (deselect with '-m "not torchtest"')
benchmark: marks tests that need to be benchmarked (deselect with '-m "not benchmark"')
1 change: 1 addition & 0 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ def read(path, encoding="utf-8"):
]

benchmark_libs = [
"pytest-benchmark"
]

doc_libs = [
Expand Down
Loading

0 comments on commit d593ae9

Please # to comment.