feat(benchpress): more smoothness metrics

Benchpress now prints out the best and worst frame time in addition to the percentage of frames that hit the target of 60fps.

It also renames 'meanFrameTime' to 'frameTime.mean'. That way, all frameTime metrics start with a common suffix and will be grouped together in the console reporter.

part of #821

Author: goderbauer

modules/benchpress/src/metric/perflog_metric.ts

@@ -62,9 +62,19 @@ export class PerflogMetric extends Metric {
       }
     }
     if (this._captureFrames) {
-      res['meanFrameTime'] = this._perfLogFeatures.frameCapture ?
-                                 'mean frame time in ms (target: 16.6ms for 60fps)' :
-                                 'WARNING: Metric requested, but not supported by driver';
+      if (!this._perfLogFeatures.frameCapture) {
+        var warningMsg = 'WARNING: Metric requested, but not supported by driver';
+        // using dot syntax for metric name to keep them grouped together in console reporter
+        res['frameTime.mean'] = warningMsg;
+        res['frameTime.worst'] = warningMsg;
+        res['frameTime.best'] = warningMsg;
+        res['frameTime.smooth'] = warningMsg;
+      } else {
+        res['frameTime.mean'] = 'mean frame time in ms (target: 16.6ms for 60fps)';
+        res['frameTime.worst'] = 'worst frame time in ms';
+        res['frameTime.best'] = 'best frame time in ms';
+        res['frameTime.smooth'] = 'percentage of frames that hit 60fps';
+      }
     }
     StringMapWrapper.forEach(this._microMetrics,
                              (desc, name) => { StringMapWrapper.set(res, name, desc); });
@@ -172,7 +182,10 @@ export class PerflogMetric extends Metric {
       result['renderTime'] = 0;
     }
     if (this._captureFrames) {
-      result['meanFrameTime'] = 0;
+      result['frameTime.mean'] = 0;
+      result['frameTime.best'] = 0;
+      result['frameTime.worst'] = 0;
+      result['frameTime.smooth'] = 0;
     }
     StringMapWrapper.forEach(this._microMetrics, (desc, name) => { result[name] = 0; });
 
@@ -288,22 +301,35 @@ export class PerflogMetric extends Metric {
           'frame capture requested in benchpress, but no start event was found');
     }
     if (frameTimes.length > 0) {
-      result['meanFrameTime'] =
-          ListWrapper.reduce(frameTimes, (a, b) => a + b, 0) / frameTimes.length;
+      this._addFrameMetrics(result, frameTimes);
     }
     result['pureScriptTime'] = result['scriptTime'] - gcTimeInScript - renderTimeInScript;
     return result;
   }
 
+  _addFrameMetrics(result: StringMap<string, any>, frameTimes: any[]) {
+    result['frameTime.mean'] =
+        ListWrapper.reduce(frameTimes, (a, b) => a + b, 0) / frameTimes.length;
+    var firstFrame = ListWrapper.get(frameTimes, 0);
+    result['frameTime.worst'] = ListWrapper.reduce(frameTimes, (a, b) => a > b ? a : b, firstFrame);
+    result['frameTime.best'] = ListWrapper.reduce(frameTimes, (a, b) => a < b ? a : b, firstFrame);
+    result['frameTime.smooth'] =
+        ListWrapper.filter(frameTimes, (a) => a < _FRAME_TIME_SMOOTH_THRESHOLD).length /
+        frameTimes.length;
+  }
+
   _markName(index) { return `${_MARK_NAME_PREFIX}${index}`; }
 }
+
 var _MICRO_ITERATIONS_REGEX = RegExpWrapper.create('(.+)\\*(\\d+)$');
 
 var _MAX_RETRY_COUNT = 20;
 var _MARK_NAME_PREFIX = 'benchpress';
 var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout');
 
 var _MARK_NAME_FRAME_CAPUTRE = 'frameCapture';
+// using 17ms as a somewhat looser threshold, instead of 16.6666ms
+var _FRAME_TIME_SMOOTH_THRESHOLD = 17;
 
 var _BINDINGS = [
   bind(PerflogMetric)

modules/benchpress/test/metric/perflog_metric_spec.ts

@@ -106,14 +106,20 @@ export function main() {
       var description =
           createMetric([[]], null, new PerfLogFeatures({frameCapture: true}), null, true)
               .describe();
-      expect(description['meanFrameTime']).not.toContain('WARNING');
+      expect(description['frameTime.mean']).not.toContain('WARNING');
+      expect(description['frameTime.best']).not.toContain('WARNING');
+      expect(description['frameTime.worst']).not.toContain('WARNING');
+      expect(description['frameTime.smooth']).not.toContain('WARNING');
     });
 
     it('should describe itself if frame capture is requested and not available', () => {
       var description =
           createMetric([[]], null, new PerfLogFeatures({frameCapture: false}), null, true)
               .describe();
-      expect(description['meanFrameTime']).toContain('WARNING');
+      expect(description['frameTime.mean']).toContain('WARNING');
+      expect(description['frameTime.best']).toContain('WARNING');
+      expect(description['frameTime.worst']).toContain('WARNING');
+      expect(description['frameTime.smooth']).toContain('WARNING');
     });
 
     describe('beginMeasure', () => {
@@ -336,7 +342,7 @@ export function main() {
              ],
                        null, true)
                  .then((data) => {
-                   expect(data['meanFrameTime']).toBe(((3 - 1) + (4 - 3)) / 2);
+                   expect(data['frameTime.mean']).toBe(((3 - 1) + (4 - 3)) / 2);
                    async.done();
                  });
            }));
@@ -398,6 +404,58 @@ export function main() {
              });
            }));
 
+        it('should calculate best and worst frame time', inject([AsyncTestCompleter], (async) => {
+             aggregate([
+               eventFactory.markStart('frameCapture', 0),
+               eventFactory.instant('frame', 1),
+               eventFactory.instant('frame', 9),
+               eventFactory.instant('frame', 15),
+               eventFactory.instant('frame', 18),
+               eventFactory.instant('frame', 28),
+               eventFactory.instant('frame', 32),
+               eventFactory.markEnd('frameCapture', 10)
+             ],
+                       null, true)
+                 .then((data) => {
+                   expect(data['frameTime.worst']).toBe(10);
+                   expect(data['frameTime.best']).toBe(3);
+                   async.done();
+                 });
+           }));
+
+        it('should calculate percentage of smoothness to be good',
+           inject([AsyncTestCompleter], (async) => {
+             aggregate([
+               eventFactory.markStart('frameCapture', 0),
+               eventFactory.instant('frame', 1),
+               eventFactory.instant('frame', 2),
+               eventFactory.instant('frame', 3),
+               eventFactory.markEnd('frameCapture', 4)
+             ],
+                       null, true)
+                 .then((data) => {
+                   expect(data['frameTime.smooth']).toBe(1.0);
+                   async.done();
+                 });
+           }));
+
+        it('should calculate percentage of smoothness to be bad',
+           inject([AsyncTestCompleter], (async) => {
+             aggregate([
+               eventFactory.markStart('frameCapture', 0),
+               eventFactory.instant('frame', 1),
+               eventFactory.instant('frame', 2),
+               eventFactory.instant('frame', 22),
+               eventFactory.instant('frame', 23),
+               eventFactory.instant('frame', 24),
+               eventFactory.markEnd('frameCapture', 4)
+             ],
+                       null, true)
+                 .then((data) => {
+                   expect(data['frameTime.smooth']).toBe(0.75);
+                   async.done();
+                 });
+           }));
 
       });