Add continuous performance benchmarking (#44)

* Script to generate JSON output for performance benchmarks
* Makefile target to generate performance data
* GA action to report performance benchmarks on GH pages
* Add `cli` build dependency

.github/workflows/benchmarks.yml

@@ -0,0 +1,36 @@
+name: benchmarks
+
+on: [push, pull_request]
+
+jobs:
+  benchmark:
+    name: Report benchmarks for Qi forms
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Install Racket
+        uses: Bogdanp/[email protected]
+        with:
+          architecture: 'x64'
+          distribution: 'full'
+          variant: 'CS'
+          version: 'stable'
+      - name: Install Package and its Dependencies
+        run: make install
+      - name: Run benchmark
+        run: make report-benchmarks | tee benchmarks.txt
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Qi Forms Benchmarks
+          tool: 'customSmallerIsBetter'
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+          output-file-path: benchmarks.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true

Makefile

@@ -39,6 +39,7 @@ help:
 	@echo "profile-competitive - Run competitive benchmarks"
 	@echo "profile-forms - Run benchmarks for individual Qi forms"
 	@echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)"
+	@echo "report-benchmarks - Run benchmarks for Qi forms and produce results for use in CI"
 
 # Primarily for use by CI.
 # Installs dependencies as well as linking this as a package.
@@ -165,4 +166,7 @@ profile-competitive:
 
 profile: profile-competitive profile-forms
 
-.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile
+report-benchmarks:
+	@racket profile/report.rkt
+
+.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile report-benchmarks

profile/forms.rkt

@@ -934,7 +934,9 @@ for the forms are run.
    (prefix-in apply: (submod ".." apply))
    (prefix-in clos: (submod ".." clos)))
 
-  (require relation
+  (require racket/match
+           racket/format
+           relation
            qi
            (only-in "util.rkt"
                     only-if
@@ -1020,7 +1022,8 @@ for the forms are run.
                    (only-if null?
                      (gen (hash-keys env)))
                    (sort <))])
-      (for/call ([f fs])
-        (hash-ref env f))))
+      (for ([f fs])
+        (match-let ([(list name ms) ((hash-ref env f))])
+          (displayln (~a name ": " ms " ms"))))))
 
   (run main))

profile/report.rkt

@@ -0,0 +1,151 @@
+#lang cli
+
+(require
+ (prefix-in one-of?: (submod "forms.rkt" one-of?))
+ (prefix-in and: (submod "forms.rkt" and))
+ (prefix-in or: (submod "forms.rkt" or))
+ (prefix-in not: (submod "forms.rkt" not))
+ (prefix-in and%: (submod "forms.rkt" and%))
+ (prefix-in or%: (submod "forms.rkt" or%))
+ (prefix-in group: (submod "forms.rkt" group))
+ (prefix-in count: (submod "forms.rkt" count))
+ (prefix-in relay: (submod "forms.rkt" relay))
+ (prefix-in relay*: (submod "forms.rkt" relay*))
+ (prefix-in amp: (submod "forms.rkt" amp))
+ (prefix-in ground: (submod "forms.rkt" ground))
+ (prefix-in thread: (submod "forms.rkt" thread))
+ (prefix-in thread-right: (submod "forms.rkt" thread-right))
+ (prefix-in crossover: (submod "forms.rkt" crossover))
+ (prefix-in all: (submod "forms.rkt" all))
+ (prefix-in any: (submod "forms.rkt" any))
+ (prefix-in none: (submod "forms.rkt" none))
+ (prefix-in all?: (submod "forms.rkt" all?))
+ (prefix-in any?: (submod "forms.rkt" any?))
+ (prefix-in none?: (submod "forms.rkt" none?))
+ (prefix-in collect: (submod "forms.rkt" collect))
+ (prefix-in sep: (submod "forms.rkt" sep))
+ (prefix-in gen: (submod "forms.rkt" gen))
+ (prefix-in esc: (submod "forms.rkt" esc))
+ (prefix-in AND: (submod "forms.rkt" AND))
+ (prefix-in OR: (submod "forms.rkt" OR))
+ (prefix-in NOT: (submod "forms.rkt" NOT))
+ (prefix-in NAND: (submod "forms.rkt" NAND))
+ (prefix-in NOR: (submod "forms.rkt" NOR))
+ (prefix-in XOR: (submod "forms.rkt" XOR))
+ (prefix-in XNOR: (submod "forms.rkt" XNOR))
+ (prefix-in tee: (submod "forms.rkt" tee))
+ (prefix-in try: (submod "forms.rkt" try))
+ (prefix-in currying: (submod "forms.rkt" currying))
+ (prefix-in template: (submod "forms.rkt" template))
+ (prefix-in catchall-template: (submod "forms.rkt" catchall-template))
+ (prefix-in if: (submod "forms.rkt" if))
+ (prefix-in when: (submod "forms.rkt" when))
+ (prefix-in unless: (submod "forms.rkt" unless))
+ (prefix-in switch: (submod "forms.rkt" switch))
+ (prefix-in sieve: (submod "forms.rkt" sieve))
+ (prefix-in gate: (submod "forms.rkt" gate))
+ (prefix-in input-aliases: (submod "forms.rkt" input-aliases))
+ (prefix-in fanout: (submod "forms.rkt" fanout))
+ (prefix-in inverter: (submod "forms.rkt" inverter))
+ (prefix-in feedback: (submod "forms.rkt" feedback))
+ (prefix-in select: (submod "forms.rkt" select))
+ (prefix-in block: (submod "forms.rkt" block))
+ (prefix-in bundle: (submod "forms.rkt" bundle))
+ (prefix-in effect: (submod "forms.rkt" effect))
+ (prefix-in live?: (submod "forms.rkt" live?))
+ (prefix-in rectify: (submod "forms.rkt" rectify))
+ (prefix-in pass: (submod "forms.rkt" pass))
+ (prefix-in foldl: (submod "forms.rkt" foldl))
+ (prefix-in foldr: (submod "forms.rkt" foldr))
+ (prefix-in loop: (submod "forms.rkt" loop))
+ (prefix-in loop2: (submod "forms.rkt" loop2))
+ (prefix-in apply: (submod "forms.rkt" apply))
+ (prefix-in clos: (submod "forms.rkt" clos)))
+
+(require racket/match
+         racket/format
+         relation
+         qi
+         json
+         (only-in "util.rkt"
+                  only-if
+                  for/call))
+
+;; It would be great if we could get the value of a variable
+;; by using its (string) name, but (eval (string->symbol name))
+;; doesn't find it. So instead, we reify the "lexical environment"
+;; here manually, so that the values can be looked up at runtime
+;; based on the string names (note that the value is always the key
+;; + ":" + "run")
+(define env
+  (hash
+   "one-of?" one-of?:run
+   "and" and:run
+   "or" or:run
+   "not" not:run
+   "and%" and%:run
+   "or%" or%:run
+   "group" group:run
+   "count" count:run
+   "relay" relay:run
+   "relay*" relay*:run
+   "amp" amp:run
+   "ground" ground:run
+   "thread" thread:run
+   "thread-right" thread-right:run
+   "crossover" crossover:run
+   "all" all:run
+   "any" any:run
+   "none" none:run
+   "all?" all?:run
+   "any?" any?:run
+   "none?" none?:run
+   "collect" collect:run
+   "sep" sep:run
+   "gen" gen:run
+   "esc" esc:run
+   "AND" AND:run
+   "OR" OR:run
+   "NOT" NOT:run
+   "NAND" NAND:run
+   "NOR" NOR:run
+   "XOR" XOR:run
+   "XNOR" XNOR:run
+   "tee" tee:run
+   "try" try:run
+   "currying" currying:run
+   "template" template:run
+   "catchall-template" catchall-template:run
+   "if" if:run
+   "when" when:run
+   "unless" unless:run
+   "switch" switch:run
+   "sieve" sieve:run
+   "gate" gate:run
+   "input-aliases" input-aliases:run
+   "fanout" fanout:run
+   "inverter" inverter:run
+   "feedback" feedback:run
+   "select" select:run
+   "block" block:run
+   "bundle" bundle:run
+   "effect" effect:run
+   "live?" live?:run
+   "rectify" rectify:run
+   "pass" pass:run
+   "foldl" foldl:run
+   "foldr" foldr:run
+   "loop" loop:run
+   "loop2" loop2:run
+   "apply" apply:run
+   "clos" clos:run))
+
+(program (main)
+  (let* ([forms (hash-keys env)]
+         [fs (~>> (forms)
+                  (sort <))])
+    (write-json (for/list ([f fs])
+                  (match-let ([(list name ms) ((hash-ref env f))])
+                    (hash 'name name 'unit "ms" 'value ms))))))
+
+(run main)

profile/util.rkt

@@ -76,7 +76,7 @@
                 (symbol->string
                  (syntax->datum #'f-name)))
   (let ([ms (measure runner f-name n-times)])
-    (displayln (~a name ": " ms " ms"))))
+    (list name ms)))
 
 ;; Run many benchmarking functions (typically exercising a single form)
 ;; a specified number of times and report the time taken using the
@@ -89,7 +89,7 @@
       (let ([ms (measure r f n)])
         (set! results (cons ms results))))
     (let ([summarized-result (apply f-summary results)])
-      (displayln (~a name ": " summarized-result " ms")))))
+      (list name summarized-result))))
 
 ;; Run different implementations of the same benchmark (e.g. a Racket vs a Qi
 ;; implementation) a specified number of times, and report the time taken

qi/info.rkt

@@ -8,7 +8,7 @@
                "qi-test"
                "qi-probe"
                "Qi-Quickscripts"))
-(define build-deps '())
+(define build-deps '("cli"))
 (define implies '("qi-lib"
                   "qi-doc"
                   "qi-test"