diff --git a/src/brevitas_examples/imagenet_classification/ptq/README.md b/src/brevitas_examples/imagenet_classification/ptq/README.md
index 2fa2baf4b..82e7829ed 100644
--- a/src/brevitas_examples/imagenet_classification/ptq/README.md
+++ b/src/brevitas_examples/imagenet_classification/ptq/README.md
@@ -5,13 +5,13 @@ This folder contains an example on how to use Brevitas PTQ flow to quantize torc
We provide two workflows:
- A benchmark suite that will tests several quantization configurations on few selected models;
-- An export script that, given a model name and a quantization configuration, evaluate its performance and allows to export it to ONNX format.
+- An evaluation script that, given a model name and a quantization configuration, evaluate its performance and allows to export it to ONNX format.
For both these flows, the following options are evaluated:
- Weights and Activations are quantized to 8 bit;
-- Scale factors can be either Floating Point (FP) or Power of Two (Po2);
+- Scale factors can be either float32 or power of two (po2);
- Weights' scale factors can be either per-tensor or per-channel;
-- Bias are quantized to int32 values for FP scale factors, int16 or int32 otherwise;
+- Bias are quantized to int32 values for float32 scale factors, int16 or int32 otherwise;
- Activations quantizer can be symmetric or asymmetric;
- Three different percentiles can be used for the activations' statistics computation (99.9, 99.99, 99.999).
@@ -34,7 +34,7 @@ where we map the torch compute layers and activations with their corresponding q
Starting from pretrained floating-point torchvision models, Brevitas offers the possibility to automatically obtain the corresponding quantized model leveraging torch.fx transformations.
For the selected subset of torchvision models, we test several possible combinations of the options described above.
-The second type of benchmarks will run pre-defined quantized MobileNet v1, starting from the pre-trained FP weights[4 ].
+The second type of benchmarks will run pre-defined quantized MobileNet v1, starting from the pre-trained floating point weights[4 ].
The pre-defined quantized model uses floating point scale factors, with a mix of per-tensor and per-channel strategies.
Weights and Activations are quantized at 8 bit.
@@ -44,11 +44,14 @@ For example, to run the script on the GPU 0:
```bash
brevitas_ptq_imagenet_benchmark --calibration-dir /path/to/imagenet/calibration/folder --validation-dir /path/to/imagenet/validation/folder --gpu 0
```
-The script requires to specify the calibration folder (`--calibration-dir`), from which the calibration samples will be taken (configurable with the `--calibration-samples` argument), and a validation folder (`--valid-dir`).
+The script requires to specify the calibration folder (`--calibration-dir`), from which the calibration samples will be taken (configurable with the `--calibration-samples` argument), and a validation folder (`--validation-dir`).
After launching the script, a `RESULT_TORCHVISION.md` markdown file will be generated with the results on the torchvision models,
and a `RESULTS_IMGCLSMOB.md` with the results on manually quantized models starting from floating point weights.
+In this folder it is possible to find a pre-computed `RESULT_TORCHVISION.md` file with all the ~300 combinations on three different torchvision models,
+as well as the results on the pre-defined quantized Mobilenet V1 (`RESULTS_IMGCLSMOB.md`).
+
## Evaluation flow
diff --git a/src/brevitas_examples/imagenet_classification/ptq/RESULTS_IMGCLSMOB.csv b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_IMGCLSMOB.csv
new file mode 100644
index 000000000..9ecbce141
--- /dev/null
+++ b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_IMGCLSMOB.csv
@@ -0,0 +1,2 @@
+Model,Bias correction,Top 1% floating point accuracy,Top 1% Quant accuracy,Floating point accuracy - Quant accuracy,Quant accuracy / floating point accuracy,Calibration size,Calibration batch size,Torch version,Brevitas version
+quant_mobilenet_v1,True,73.39,72.704,-0.686,0.991,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
diff --git a/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION.csv b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION.csv
new file mode 100644
index 000000000..b01cf192d
--- /dev/null
+++ b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION.csv
@@ -0,0 +1,325 @@
+Model,Target backend,Scale factor type,Activations and weights bit width,Bias bit width,Per-channel scale,Activation quantization type,Bias correction,Graph equalization iters,Merge Bias in graph equalization,Activation quantization percentile,Top 1% floating point accuracy,Top 1% quant accuracy,Floating point accuracy - quant accuracy,Quant accuracy / floating point accuracy,Calibration size,Calibration batch size,Torch version,Brevitas version
+resnet18,generic,float32,8,int32,False,asymmetric,True,0,False,99.9,69.758,62.562,-7.196,0.897,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,0,False,99.99,69.758,68.348,-1.41,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,0,False,99.999,69.758,69.404,-0.354,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,False,99.9,69.758,66.962,-2.796,0.96,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,False,99.99,69.758,68.916,-0.842,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,False,99.999,69.758,69.428,-0.33,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,True,99.9,69.758,66.326,-3.432,0.951,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,True,99.99,69.758,68.766,-0.992,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,True,99.999,69.758,69.382,-0.376,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,0,False,99.9,69.758,0.52,-69.238,0.007,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,0,False,99.99,69.758,57.116,-12.642,0.819,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,0,False,99.999,69.758,69.404,-0.354,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,False,99.9,69.758,37.13,-32.628,0.532,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,False,99.99,69.758,68.41,-1.348,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,False,99.999,69.758,69.428,-0.33,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,True,99.9,69.758,12.31,-57.448,0.176,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,True,99.99,69.758,66.88,-2.878,0.959,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,True,99.999,69.758,69.382,-0.376,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,0,False,99.9,69.758,62.794,-6.964,0.9,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,0,False,99.99,69.758,68.362,-1.396,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,0,False,99.999,69.758,69.342,-0.416,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,False,99.9,69.758,67.104,-2.654,0.962,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,False,99.99,69.758,69.004,-0.754,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,False,99.999,69.758,69.422,-0.336,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,True,99.9,69.758,66.6,-3.158,0.955,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,True,99.99,69.758,68.872,-0.886,0.987,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,True,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,0,False,99.9,69.758,0.54,-69.218,0.008,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,0,False,99.99,69.758,57.344,-12.414,0.822,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,0,False,99.999,69.758,69.342,-0.416,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,False,99.9,69.758,37.164,-32.594,0.533,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,False,99.99,69.758,68.4,-1.358,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,False,99.999,69.758,69.422,-0.336,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,True,99.9,69.758,12.476,-57.282,0.179,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,True,99.99,69.758,67.096,-2.662,0.962,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,True,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,0,False,99.9,69.758,68.408,-1.35,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,0,False,99.99,69.758,69.03,-0.728,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,0,False,99.999,69.758,69.332,-0.426,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,False,99.9,69.758,68.966,-0.792,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,False,99.99,69.758,69.038,-0.72,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,False,99.999,69.758,69.306,-0.452,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,True,99.9,69.758,68.114,-1.644,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,True,99.99,69.758,68.056,-1.702,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,20,True,99.999,69.758,68.206,-1.552,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,0,False,99.9,69.758,59.198,-10.56,0.849,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,0,False,99.99,69.758,67.972,-1.786,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,0,False,99.999,69.758,69.332,-0.426,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,False,99.9,69.758,65.858,-3.9,0.944,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,False,99.99,69.758,68.248,-1.51,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,False,99.999,69.758,69.306,-0.452,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,True,99.9,69.758,67.184,-2.574,0.963,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,True,99.99,69.758,68.008,-1.75,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,20,True,99.999,69.758,68.206,-1.552,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,0,False,99.9,69.758,68.78,-0.978,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,0,False,99.99,69.758,69.384,-0.374,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,0,False,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,False,99.9,69.758,69.302,-0.456,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,False,99.99,69.758,69.294,-0.464,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,False,99.999,69.758,69.638,-0.12,0.998,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,True,99.9,69.758,69.404,-0.354,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,True,99.99,69.758,69.432,-0.326,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,True,99.999,69.758,69.556,-0.202,0.997,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,0,False,99.9,69.758,59.486,-10.272,0.853,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,0,False,99.99,69.758,68.382,-1.376,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,0,False,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,False,99.9,69.758,66.224,-3.534,0.949,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,False,99.99,69.758,68.614,-1.144,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,False,99.999,69.758,69.638,-0.12,0.998,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,True,99.9,69.758,68.634,-1.124,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,True,99.99,69.758,69.328,-0.43,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,True,99.999,69.758,69.556,-0.202,0.997,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,0,False,99.9,69.758,68.414,-1.344,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,0,False,99.99,69.758,69.02,-0.738,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,False,99.9,69.758,68.954,-0.804,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,False,99.99,69.758,69.038,-0.72,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,False,99.999,69.758,69.244,-0.514,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,True,99.9,69.758,68.09,-1.668,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,True,99.99,69.758,68.088,-1.67,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,20,True,99.999,69.758,68.216,-1.542,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,0,False,99.9,69.758,59.19,-10.568,0.849,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,0,False,99.99,69.758,67.984,-1.774,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,False,99.9,69.758,65.872,-3.886,0.944,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,False,99.99,69.758,68.286,-1.472,0.979,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,False,99.999,69.758,69.244,-0.514,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,True,99.9,69.758,67.242,-2.516,0.964,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,True,99.99,69.758,67.972,-1.786,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,20,True,99.999,69.758,68.216,-1.542,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,0,False,99.9,69.758,67.962,-1.796,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,0,False,99.99,69.758,69.02,-0.738,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,False,99.9,69.758,68.712,-1.046,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,False,99.99,69.758,69.04,-0.718,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,False,99.999,69.758,69.244,-0.514,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,True,99.9,69.758,68.142,-1.616,0.977,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,True,99.99,69.758,68.088,-1.67,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,20,True,99.999,69.758,68.216,-1.542,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,0,False,99.9,69.758,58.832,-10.926,0.843,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,0,False,99.99,69.758,67.984,-1.774,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,False,99.9,69.758,65.568,-4.19,0.94,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,False,99.99,69.758,68.344,-1.414,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,False,99.999,69.758,69.244,-0.514,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,True,99.9,69.758,67.194,-2.564,0.963,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,True,99.99,69.758,67.972,-1.786,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,20,True,99.999,69.758,68.216,-1.542,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,0,False,99.9,71.898,61.99,-9.908,0.862,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,0,False,99.99,71.898,70.716,-1.182,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,0,False,99.999,71.898,70.614,-1.284,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,False,99.9,71.898,62.388,-9.51,0.868,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,False,99.99,71.898,71.006,-0.892,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,False,99.999,71.898,70.568,-1.33,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,True,99.9,71.898,62.582,-9.316,0.87,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,True,99.99,71.898,71.108,-0.79,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,True,99.999,71.898,70.692,-1.206,0.983,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,0,False,99.9,71.898,61.476,-10.422,0.855,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,0,False,99.99,71.898,70.722,-1.176,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,0,False,99.999,71.898,70.614,-1.284,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,False,99.9,71.898,62.784,-9.114,0.873,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,False,99.99,71.898,70.728,-1.17,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,False,99.999,71.898,70.568,-1.33,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,True,99.9,71.898,63.77,-8.128,0.887,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,True,99.99,71.898,71.008,-0.89,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,True,99.999,71.898,70.692,-1.206,0.983,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,0,False,99.9,71.898,62.41,-9.488,0.868,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,0,False,99.99,71.898,71.158,-0.74,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,0,False,99.999,71.898,70.74,-1.158,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,False,99.9,71.898,62.178,-9.72,0.865,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,False,99.99,71.898,71.076,-0.822,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,False,99.999,71.898,70.622,-1.276,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,True,99.9,71.898,62.28,-9.618,0.866,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,True,99.99,71.898,71.124,-0.774,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,20,True,99.999,71.898,70.628,-1.27,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,0,False,99.9,71.898,62.062,-9.836,0.863,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,0,False,99.99,71.898,71.052,-0.846,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,0,False,99.999,71.898,70.74,-1.158,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,False,99.9,71.898,62.588,-9.31,0.871,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,False,99.99,71.898,70.824,-1.074,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,False,99.999,71.898,70.622,-1.276,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,True,99.9,71.898,63.616,-8.282,0.885,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,True,99.99,71.898,70.97,-0.928,0.987,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,20,True,99.999,71.898,70.628,-1.27,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,0,False,99.9,71.898,70.36,-1.538,0.979,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,0,False,99.99,71.898,70.014,-1.884,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,False,99.9,71.898,70.514,-1.384,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,False,99.99,71.898,70.462,-1.436,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,False,99.999,71.898,69.116,-2.782,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,True,99.9,71.898,70.592,-1.306,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,True,99.99,71.898,70.818,-1.08,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,0,False,99.9,71.898,68.8,-3.098,0.957,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,0,False,99.99,71.898,69.994,-1.904,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,False,99.9,71.898,68.886,-3.012,0.958,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,False,99.99,71.898,70.054,-1.844,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,False,99.999,71.898,69.116,-2.782,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,True,99.9,71.898,69.188,-2.71,0.962,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,0,False,99.9,71.898,70.894,-1.004,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,0,False,99.99,71.898,70.584,-1.314,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,0,False,99.999,71.898,70.358,-1.54,0.979,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,False,99.9,71.898,70.666,-1.232,0.983,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,False,99.99,71.898,70.582,-1.316,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,False,99.999,71.898,69.238,-2.66,0.963,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,True,99.9,71.898,70.88,-1.018,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,True,99.99,71.898,70.78,-1.118,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,20,True,99.999,71.898,69.35,-2.548,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,0,False,99.9,71.898,69.184,-2.714,0.962,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,0,False,99.99,71.898,70.63,-1.268,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,0,False,99.999,71.898,70.358,-1.54,0.979,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,False,99.9,71.898,69.088,-2.81,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,False,99.99,71.898,70.338,-1.56,0.978,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,False,99.999,71.898,69.238,-2.66,0.963,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,True,99.9,71.898,69.456,-2.442,0.966,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,True,99.99,71.898,70.864,-1.034,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,True,99.999,71.898,69.35,-2.548,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,0,False,99.9,71.898,70.36,-1.538,0.979,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,0,False,99.99,71.898,70.014,-1.884,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,False,99.9,71.898,70.514,-1.384,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,False,99.99,71.898,70.462,-1.436,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,False,99.999,71.898,69.116,-2.782,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,True,99.9,71.898,70.592,-1.306,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,True,99.99,71.898,70.818,-1.08,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,0,False,99.9,71.898,68.8,-3.098,0.957,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,0,False,99.99,71.898,69.994,-1.904,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,False,99.9,71.898,68.886,-3.012,0.958,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,False,99.99,71.898,70.054,-1.844,0.974,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,False,99.999,71.898,69.116,-2.782,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,True,99.9,71.898,69.188,-2.71,0.962,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,0,False,99.9,71.898,70.274,-1.624,0.977,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,0,False,99.99,71.898,69.972,-1.926,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,False,99.9,71.898,36.652,-35.246,0.51,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,False,99.99,71.898,69.45,-2.448,0.966,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,False,99.999,71.898,68.758,-3.14,0.956,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,True,99.9,71.898,68.83,-3.068,0.957,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,True,99.99,71.898,70.75,-1.148,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,0,False,99.9,71.898,68.848,-3.05,0.958,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,0,False,99.99,71.898,69.902,-1.996,0.972,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,0,False,99.999,71.898,69.934,-1.964,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,False,99.9,71.898,67.35,-4.548,0.937,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,False,99.99,71.898,69.036,-2.862,0.96,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,False,99.999,71.898,68.758,-3.14,0.956,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,True,99.9,71.898,67.508,-4.39,0.939,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,True,99.999,71.898,69.064,-2.834,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,0,False,99.9,75.912,6.004,-69.908,0.079,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,0,False,99.99,75.912,8.016,-67.896,0.106,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,0,False,99.999,75.912,74.478,-1.434,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,False,99.9,75.912,5.45,-70.462,0.072,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,False,99.99,75.912,7.632,-68.28,0.101,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,False,99.999,75.912,72.858,-3.054,0.96,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,True,99.9,75.912,5.718,-70.194,0.075,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,True,99.99,75.912,15.114,-60.798,0.199,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,20,True,99.999,75.912,74.378,-1.534,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,0,False,99.9,75.912,6.44,-69.472,0.085,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,0,False,99.99,75.912,35.35,-40.562,0.466,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,0,False,99.999,75.912,74.478,-1.434,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,False,99.9,75.912,6.14,-69.772,0.081,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,False,99.99,75.912,66.022,-9.89,0.87,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,False,99.999,75.912,72.858,-3.054,0.96,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,True,99.9,75.912,6.84,-69.072,0.09,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,True,99.99,75.912,52.36,-23.552,0.69,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,20,True,99.999,75.912,74.378,-1.534,0.98,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,0,False,99.9,75.912,5.908,-70.004,0.078,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,0,False,99.99,75.912,7.876,-68.036,0.104,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,0,False,99.999,75.912,74.452,-1.46,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,False,99.9,75.912,5.318,-70.594,0.07,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,False,99.99,75.912,7.554,-68.358,0.1,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,False,99.999,75.912,72.976,-2.936,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,True,99.9,75.912,5.478,-70.434,0.072,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,True,99.99,75.912,15.052,-60.86,0.198,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,True,99.999,75.912,74.52,-1.392,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,0,False,99.9,75.912,6.442,-69.47,0.085,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,0,False,99.99,75.912,35.9,-40.012,0.473,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,0,False,99.999,75.912,74.452,-1.46,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,False,99.9,75.912,6.044,-69.868,0.08,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,False,99.99,75.912,66.242,-9.67,0.873,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,False,99.999,75.912,72.976,-2.936,0.961,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,True,99.9,75.912,6.668,-69.244,0.088,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,True,99.99,75.912,53.034,-22.878,0.699,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,True,99.999,75.912,74.52,-1.392,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,0,False,99.9,75.912,6.812,-69.1,0.09,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,0,False,99.99,75.912,9.302,-66.61,0.123,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,0,False,99.999,75.912,73.982,-1.93,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,False,99.9,75.912,7.996,-67.916,0.105,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,False,99.99,75.912,24.952,-50.96,0.329,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,False,99.999,75.912,73.996,-1.916,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,True,99.9,75.912,7.294,-68.618,0.096,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,True,99.99,75.912,10.1,-65.812,0.133,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,True,99.999,75.912,73.282,-2.63,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,0,False,99.9,75.912,8.808,-67.104,0.116,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,0,False,99.99,75.912,31.096,-44.816,0.41,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,0,False,99.999,75.912,73.982,-1.93,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,False,99.9,75.912,14.25,-61.662,0.188,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,False,99.99,75.912,64.394,-11.518,0.848,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,False,99.999,75.912,73.996,-1.916,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,True,99.9,75.912,9.426,-66.486,0.124,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,True,99.99,75.912,38.384,-37.528,0.506,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,True,99.999,75.912,73.282,-2.63,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,0,False,99.9,75.912,6.872,-69.04,0.091,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,0,False,99.99,75.912,9.538,-66.374,0.126,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,0,False,99.999,75.912,74.074,-1.838,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,False,99.9,75.912,7.928,-67.984,0.104,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,False,99.99,75.912,25.426,-50.486,0.335,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,False,99.999,75.912,74.038,-1.874,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,True,99.9,75.912,7.292,-68.62,0.096,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,True,99.99,75.912,10.19,-65.722,0.134,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,20,True,99.999,75.912,73.516,-2.396,0.968,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,0,False,99.9,75.912,8.81,-67.102,0.116,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,0,False,99.99,75.912,29.986,-45.926,0.395,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,0,False,99.999,75.912,74.074,-1.838,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,False,99.9,75.912,14.398,-61.514,0.19,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,False,99.99,75.912,64.928,-10.984,0.855,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,False,99.999,75.912,74.038,-1.874,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,True,99.9,75.912,9.606,-66.306,0.127,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,True,99.99,75.912,37.178,-38.734,0.49,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,20,True,99.999,75.912,73.516,-2.396,0.968,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,0,False,99.9,75.912,6.712,-69.2,0.088,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,0,False,99.99,75.912,9.356,-66.556,0.123,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,0,False,99.999,75.912,73.87,-2.042,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,False,99.9,75.912,7.996,-67.916,0.105,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,False,99.99,75.912,25.952,-49.96,0.342,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,True,99.9,75.912,7.232,-68.68,0.095,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,True,99.99,75.912,10.172,-65.74,0.134,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,True,99.999,75.912,73.218,-2.694,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,0,False,99.9,75.912,8.65,-67.262,0.114,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,0,False,99.99,75.912,32.398,-43.514,0.427,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,0,False,99.999,75.912,73.87,-2.042,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,False,99.9,75.912,15.148,-60.764,0.2,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,False,99.99,75.912,65.66,-10.252,0.865,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,True,99.9,75.912,9.396,-66.516,0.124,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,True,99.99,75.912,38.91,-37.002,0.513,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,True,99.999,75.912,73.218,-2.694,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,0,False,99.9,75.912,6.704,-69.208,0.088,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,0,False,99.99,75.912,9.348,-66.564,0.123,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,0,False,99.999,75.912,73.87,-2.042,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,False,99.9,75.912,8.044,-67.868,0.106,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,False,99.99,75.912,26.012,-49.9,0.343,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,True,99.9,75.912,7.258,-68.654,0.096,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,True,99.99,75.912,10.124,-65.788,0.133,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,True,99.999,75.912,73.218,-2.694,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,0,False,99.9,75.912,8.65,-67.262,0.114,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,0,False,99.99,75.912,32.398,-43.514,0.427,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,0,False,99.999,75.912,73.87,-2.042,0.973,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,False,99.9,75.912,15.148,-60.764,0.2,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,False,99.99,75.912,65.66,-10.252,0.865,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,True,99.9,75.912,9.396,-66.516,0.124,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,True,99.99,75.912,38.91,-37.002,0.513,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,True,99.999,75.912,73.218,-2.694,0.965,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
diff --git a/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION_BEST_CONFIGS.csv b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION_BEST_CONFIGS.csv
new file mode 100644
index 000000000..9b084852d
--- /dev/null
+++ b/src/brevitas_examples/imagenet_classification/ptq/RESULTS_TORCHVISION_BEST_CONFIGS.csv
@@ -0,0 +1,37 @@
+Model,Target backend,Scale factor type,Activations and weights bit width,Bias bit width,Per-channel scale,Activation quantization type,Bias correction,Graph equalization iters,Merge Bias in graph equalization,Activation quantization percentile,Top 1% floating point accuracy,Top 1% quant accuracy,Floating point accuracy - quant accuracy,Quant accuracy / floating point accuracy,Calibration size,Calibration batch size,Torch version,Brevitas version
+mobilenet_v2,flexml,po2,8,int16,False,asymmetric,True,20,True,99.99,71.898,70.75,-1.148,0.984,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,asymmetric,True,20,True,99.99,71.898,70.818,-1.08,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,asymmetric,True,20,True,99.99,71.898,70.818,-1.08,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int32,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,flexml,po2,8,int16,False,symmetric,True,20,True,99.99,71.898,70.84,-1.058,0.985,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,symmetric,True,20,True,99.99,71.898,70.864,-1.034,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,po2,8,int32,True,asymmetric,True,0,False,99.9,71.898,70.894,-1.004,0.986,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,symmetric,True,20,True,99.99,71.898,71.008,-0.89,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,symmetric,True,0,False,99.99,71.898,71.052,-0.846,0.988,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,False,asymmetric,True,20,True,99.99,71.898,71.108,-0.79,0.989,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+mobilenet_v2,generic,float32,8,int32,True,asymmetric,True,0,False,99.99,71.898,71.158,-0.74,0.99,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,asymmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int32,False,symmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,asymmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,flexml,po2,8,int16,False,symmetric,True,0,False,99.999,69.758,69.29,-0.468,0.993,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,asymmetric,True,0,False,99.999,69.758,69.332,-0.426,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,False,symmetric,True,0,False,99.999,69.758,69.332,-0.426,0.994,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,asymmetric,True,20,False,99.999,69.758,69.428,-0.33,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,False,symmetric,True,20,False,99.999,69.758,69.428,-0.33,0.995,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,asymmetric,True,20,True,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,float32,8,int32,True,symmetric,True,20,True,99.999,69.758,69.484,-0.274,0.996,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,asymmetric,True,20,False,99.999,69.758,69.638,-0.12,0.998,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+resnet18,generic,po2,8,int32,True,symmetric,True,20,False,99.999,69.758,69.638,-0.12,0.998,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,asymmetric,True,20,False,99.999,75.912,73.996,-1.916,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,False,symmetric,True,20,False,99.999,75.912,73.996,-1.916,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,asymmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int32,False,symmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,asymmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,flexml,po2,8,int16,False,symmetric,True,20,False,99.999,75.912,74.04,-1.872,0.975,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,asymmetric,True,0,False,99.999,75.912,74.074,-1.838,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,po2,8,int32,True,symmetric,True,0,False,99.999,75.912,74.074,-1.838,0.976,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,asymmetric,True,0,False,99.999,75.912,74.478,-1.434,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,False,symmetric,True,0,False,99.999,75.912,74.478,-1.434,0.981,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,asymmetric,True,20,True,99.999,75.912,74.52,-1.392,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
+vit_b_32,generic,float32,8,int32,True,symmetric,True,20,True,99.999,75.912,74.52,-1.392,0.982,1000,64,1.13.1,0.8.1.dev98+ge2bdfc04.d20230412
diff --git a/src/brevitas_examples/imagenet_classification/ptq/ptq_benchmark.py b/src/brevitas_examples/imagenet_classification/ptq/ptq_benchmark.py
index 63e11c364..9b7f1b63b 100644
--- a/src/brevitas_examples/imagenet_classification/ptq/ptq_benchmark.py
+++ b/src/brevitas_examples/imagenet_classification/ptq/ptq_benchmark.py
@@ -78,28 +78,29 @@ def main():
'Per-channel scale',
'Activation quantization type',
'Bias correction',
- 'Graph equalization eters',
+ 'Graph equalization iters',
'Merge Bias in graph equalization',
'Activation quantization percentile',
- 'Top 1% FP accuracy',
+ 'Top 1% floating point accuracy',
'Top 1% quant accuracy',
- 'FP accuracy - quant accuracy',
- 'Quant accuracy / FP accuracy',
+ 'Floating point accuracy - quant accuracy',
+ 'Quant accuracy / floating point accuracy',
'Calibration size',
'Calibration batch size',
'Torch version',
'Brevitas version'])
+ torchvision_best_configs_df = torchvision_df.copy()
- ptq_torchvision_models(torchvision_df, args)
+ ptq_torchvision_models(torchvision_df, torchvision_best_configs_df, args)
quant_model_df = pd.DataFrame(
columns=[
'Model',
'Bias correction',
- 'Top 1% FP accuracy',
- 'Top 1% Quant accuracy',
- 'FP accuracy - Quant accuracy',
- 'Quant accuracy / FP accuracy',
+ 'Top 1% floating point accuracy',
+ 'Top 1% quant accuracy',
+ 'Floating point accuracy - quant accuracy',
+ 'Quant accuracy / floating point accuracy',
'Calibration size',
'Calibration batch size',
'Torch version',
@@ -108,7 +109,7 @@ def main():
ptq_quant_models(quant_model_df, args)
-def ptq_torchvision_models(df, args):
+def ptq_torchvision_models(df, best_config_df, args):
options = [
TORCHVISION_TOP1_MAP.keys(),
@@ -126,7 +127,7 @@ def ptq_torchvision_models(df, args):
combinations = list(product(*options))
k = 0
-
+ best_results_map = {model_name: dict() for model_name in TORCHVISION_TOP1_MAP.keys()}
for (model_name,
target_backend,
scale_factor_type,
@@ -257,8 +258,26 @@ def ptq_torchvision_models(df, args):
args.batch_size_calibration,
torch_version,
brevitas_version]
+
+ hardware_config = '_'.join(
+ str(el) for el in [
+ args.target_backend,
+ args.scale_factor_type,
+ args.bit_width,
+ args.bias_bit_width,
+ args.scaling_per_output_channel,
+ args.act_quant_type])
+ if hardware_config not in best_results_map[args.model_name] or (
+ hardware_config in best_results_map[args.model_name] and
+ top1 > best_results_map[args.model_name][hardware_config]):
+ best_results_map[args.model_name][hardware_config] = top1
+ best_config_df.at[len(best_config_df.index), :] = df.iloc[k]
+
+ best_config_df.sort_values(by=['Model', 'Top 1% quant accuracy'])
+
+ df.to_csv('RESULTS_TORCHVISION.csv', index=False, mode='w')
+ best_config_df.to_csv('RESULTS_TORCHVISION_BEST_CONFIGS.csv', index=False, mode='w')
k += 1
- df.to_markdown('RESULTS_TORCHVISION.md', index=False, mode='w')
def ptq_quant_models(df, args):
@@ -327,7 +346,7 @@ def ptq_quant_models(df, args):
torch_version,
brevitas_version]
k += 1
- df.to_markdown('RESULTS_IMGCLSMOB.md', index=False, mode='a')
+ df.to_csv('RESULTS_IMGCLSMOB.csv', index=False, mode='w')
if __name__ == '__main__':