Benchmarks: combined view of both JavaSCript and Wasm

Author: vouillon

benchmarks/Makefile

@@ -24,9 +24,9 @@ microbenchmarks:
 	make _noprecomp
 	$(RUN) ./report-wasm-cb.config ./report-jsoo-cb.config
 	$(REPORT) -format current-bench -config report-wasm-cb.config \
-	  -ylabel "Wasm_of_ocaml"
+	  -ylabel "Wasm_of_ocaml" | sh utils/aggregate.sh wasm
 	$(REPORT) -format current-bench -config report-jsoo-cb.config \
-	  -ylabel "Js_of_ocaml"
+	  -ylabel "Js_of_ocaml" | sh utils/aggregate.sh js
 
 graphsnopr: _noprecomp $(GRAPHSNOPR)
 

benchmarks/benchmark-camlboy/Makefile

@@ -8,10 +8,11 @@ SCRIPT=../../CAMLBOY/_build/default/bin/web/bench_node.bc
 ROM=../../CAMLBOY/resource/games/tobu.gb
 
 bench:
-	$(MAKE) perform COMPILER=Js_of_ocaml SUFFIX=.js
-	$(MAKE) perform COMPILER=Wasm_of_ocaml SUFFIX=.wasm.js
+	$(MAKE) perform COMPILER=Js_of_ocaml SUFFIX=.js KIND=js
+	$(MAKE) perform COMPILER=Wasm_of_ocaml SUFFIX=.wasm.js KIND=wasm
 
 perform:
 	node $(SCRIPT)$(SUFFIX) $(ROM) | \
 	tee /dev/stderr | \
-	ocaml output_results.ml $(COMPILER)
+	ocaml output_results.ml $(COMPILER) | \
+	sh ../utils/aggregate.sh $(KIND)

benchmarks/benchmark-fiat-crypto/Makefile

@@ -5,12 +5,13 @@ export NAME=Fiat-Crypto
 SHELL=/bin/bash -o pipefail
 
 bench:
-	$(MAKE) perform COMPILER=js_of_ocaml EXTRA_ARGS="--enable effects"
-	$(MAKE) perform COMPILER=wasm_of_ocaml EXTRA_ARGS=""
+	$(MAKE) perform COMPILER=js_of_ocaml EXTRA_ARGS="--enable effects" KIND=js
+	$(MAKE) perform COMPILER=wasm_of_ocaml EXTRA_ARGS="" KIND=wasm
 
 perform: bedrock2_fiat_crypto.byte
 	/usr/bin/time -f "%E %R" $(COMPILER) --debug times --source-map $(EXTRA_ARGS) $< 2>&1 | \
-	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME)
+	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
+	sh ../utils/aggregate.sh $(KIND)
 
 bedrock2_fiat_crypto.byte: bedrock2_fiat_crypto.ml
 	ocamlfind ocamlc -w -20 -g -linkpkg -package js_of_ocaml -g $< -o $@

benchmarks/benchmark-ocamlc/Makefile

@@ -6,13 +6,16 @@ SHELL=/bin/bash -o pipefail
 
 bench:
 	cp -r ../sources/ml .
-	$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=ocamlc.js
-	$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=ocamlc.wasm.js
+	$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=ocamlc.js KIND=js
+	$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=ocamlc.wasm.js KIND=wasm
 
 ARGS=ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml
 
 perform:
 	/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty `which ocamlc.byte` -o $(SCRIPT) 2>&1 | \
-	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME)
+	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
+	sh ../utils/aggregate.sh $(KIND)
 	/usr/bin/time -f '{"compiler": "$(COMPILER)", "time":"%E"}' node $(SCRIPT) -c $(ARGS) 2>&1 | \
-	sh ../utils/format_metrics.sh exec
+	sh ../utils/format_metrics.sh exec | \
+	sh ../utils/aggregate.sh $(KIND)
+

benchmarks/benchmark-partial-render-table/Makefile

@@ -5,15 +5,17 @@ export NAME=Partial Render Table
 SHELL=/bin/bash -o pipefail
 
 bench:
-	$(MAKE) perform COMPILER=Js_of_ocaml SCRIPT=main.bc.js
-	$(MAKE) perform COMPILER=Wasm_of_ocaml SCRIPT=main.bc.wasm.js
+	$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=main.bc.js KIND=js
+	$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=main.bc.wasm.js KIND=wasm
 
 BYTE_FILE=../../janestreet/_build/default/lib/bonsai_web_components/partial_render_table/bench/bin/main.bc-for-jsoo
 
 perform:
 	/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty $(BYTE_FILE) -o out.js 2>&1 | \
 	tee /dev/stderr | \
-	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) "$(NAME)"
+	ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) "$(NAME)" | \
+	sh ../utils/aggregate.sh $(KIND)
 	node $(SCRIPT) | \
 	tee /dev/stderr | \
-	ocaml -I +str str.cma summarize_results.ml $(COMPILER)
+	ocaml -I +str str.cma summarize_results.ml $(COMPILER) | \
+	sh ../utils/aggregate.sh $(KIND)

benchmarks/run.ml

@@ -34,13 +34,13 @@ module Param = struct
     { warm_up_time = 1.0
     ; min_measures = 10
     ; max_confidence = 0.03
-    ; max_duration = 20.
+    ; max_duration = 120.
     ; verbose = false
     }
 
-  let fast x = { x with min_measures = 5; max_confidence = 0.15 }
+  let fast x = { x with min_measures = 5; max_confidence = 0.15; max_duration = 20. }
 
-  let ffast x = { x with min_measures = 2; max_confidence = 42. }
+  let ffast x = { x with min_measures = 2; max_confidence = 42.; max_duration = 20. }
 
   let verbose x = { x with verbose = true }
 end

benchmarks/utils/aggregate.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+TEMP=$(mktemp)
+
+tee $TEMP | \
+jq '.name |= "Js vs Wasm"' | \
+jq '.results[] |= select(.name | test("Code size") | not)' |\
+jq '.results[].metrics[].name |= if test("/") then .+"-'$1'" else .+"/'$1'" end'
+cat $TEMP
+rm $TEMP