From eef5d75b7e600cac190458f9b74b2cefcdedf2c7 Mon Sep 17 00:00:00 2001 From: Siddhartha Kasivajhula Date: Fri, 24 Jun 2022 16:34:07 -0700 Subject: [PATCH] Add continuous performance benchmarking (#44) * Script to generate JSON output for performance benchmarks * Makefile target to generate performance data * GA action to report performance benchmarks on GH pages * Add `cli` build dependency --- .github/workflows/benchmarks.yml | 36 ++++++++ Makefile | 6 +- profile/forms.rkt | 9 +- profile/report.rkt | 151 +++++++++++++++++++++++++++++++ profile/util.rkt | 4 +- qi/info.rkt | 2 +- 6 files changed, 201 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 profile/report.rkt diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..332fe82b --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,36 @@ +name: benchmarks + +on: [push, pull_request] + +jobs: + benchmark: + name: Report benchmarks for Qi forms + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + - name: Install Racket + uses: Bogdanp/setup-racket@v1.8.1 + with: + architecture: 'x64' + distribution: 'full' + variant: 'CS' + version: 'stable' + - name: Install Package and its Dependencies + run: make install + - name: Run benchmark + run: make report-benchmarks | tee benchmarks.txt + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Qi Forms Benchmarks + tool: 'customSmallerIsBetter' + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + output-file-path: benchmarks.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true diff --git a/Makefile b/Makefile index 414b4c35..36e15d7b 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ help: @echo "profile-competitive - Run competitive benchmarks" @echo "profile-forms - Run benchmarks for individual Qi forms" @echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)" + @echo "report-benchmarks - Run benchmarks for Qi forms and produce results for use in CI" # Primarily for use by CI. # Installs dependencies as well as linking this as a package. @@ -165,4 +166,7 @@ profile-competitive: profile: profile-competitive profile-forms -.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile +report-benchmarks: + @racket profile/report.rkt + +.PHONY: help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile report-benchmarks diff --git a/profile/forms.rkt b/profile/forms.rkt index b4e67aee..e0f0d57a 100644 --- a/profile/forms.rkt +++ b/profile/forms.rkt @@ -934,7 +934,9 @@ for the forms are run. (prefix-in apply: (submod ".." apply)) (prefix-in clos: (submod ".." clos))) - (require relation + (require racket/match + racket/format + relation qi (only-in "util.rkt" only-if @@ -1020,7 +1022,8 @@ for the forms are run. (only-if null? (gen (hash-keys env))) (sort <))]) - (for/call ([f fs]) - (hash-ref env f)))) + (for ([f fs]) + (match-let ([(list name ms) ((hash-ref env f))]) + (displayln (~a name ": " ms " ms")))))) (run main)) diff --git a/profile/report.rkt b/profile/report.rkt new file mode 100644 index 00000000..9d7f7343 --- /dev/null +++ b/profile/report.rkt @@ -0,0 +1,151 @@ +#lang cli + +(require + (prefix-in one-of?: (submod "forms.rkt" one-of?)) + (prefix-in and: (submod "forms.rkt" and)) + (prefix-in or: (submod "forms.rkt" or)) + (prefix-in not: (submod "forms.rkt" not)) + (prefix-in and%: (submod "forms.rkt" and%)) + (prefix-in or%: (submod "forms.rkt" or%)) + (prefix-in group: (submod "forms.rkt" group)) + (prefix-in count: (submod "forms.rkt" count)) + (prefix-in relay: (submod "forms.rkt" relay)) + (prefix-in relay*: (submod "forms.rkt" relay*)) + (prefix-in amp: (submod "forms.rkt" amp)) + (prefix-in ground: (submod "forms.rkt" ground)) + (prefix-in thread: (submod "forms.rkt" thread)) + (prefix-in thread-right: (submod "forms.rkt" thread-right)) + (prefix-in crossover: (submod "forms.rkt" crossover)) + (prefix-in all: (submod "forms.rkt" all)) + (prefix-in any: (submod "forms.rkt" any)) + (prefix-in none: (submod "forms.rkt" none)) + (prefix-in all?: (submod "forms.rkt" all?)) + (prefix-in any?: (submod "forms.rkt" any?)) + (prefix-in none?: (submod "forms.rkt" none?)) + (prefix-in collect: (submod "forms.rkt" collect)) + (prefix-in sep: (submod "forms.rkt" sep)) + (prefix-in gen: (submod "forms.rkt" gen)) + (prefix-in esc: (submod "forms.rkt" esc)) + (prefix-in AND: (submod "forms.rkt" AND)) + (prefix-in OR: (submod "forms.rkt" OR)) + (prefix-in NOT: (submod "forms.rkt" NOT)) + (prefix-in NAND: (submod "forms.rkt" NAND)) + (prefix-in NOR: (submod "forms.rkt" NOR)) + (prefix-in XOR: (submod "forms.rkt" XOR)) + (prefix-in XNOR: (submod "forms.rkt" XNOR)) + (prefix-in tee: (submod "forms.rkt" tee)) + (prefix-in try: (submod "forms.rkt" try)) + (prefix-in currying: (submod "forms.rkt" currying)) + (prefix-in template: (submod "forms.rkt" template)) + (prefix-in catchall-template: (submod "forms.rkt" catchall-template)) + (prefix-in if: (submod "forms.rkt" if)) + (prefix-in when: (submod "forms.rkt" when)) + (prefix-in unless: (submod "forms.rkt" unless)) + (prefix-in switch: (submod "forms.rkt" switch)) + (prefix-in sieve: (submod "forms.rkt" sieve)) + (prefix-in gate: (submod "forms.rkt" gate)) + (prefix-in input-aliases: (submod "forms.rkt" input-aliases)) + (prefix-in fanout: (submod "forms.rkt" fanout)) + (prefix-in inverter: (submod "forms.rkt" inverter)) + (prefix-in feedback: (submod "forms.rkt" feedback)) + (prefix-in select: (submod "forms.rkt" select)) + (prefix-in block: (submod "forms.rkt" block)) + (prefix-in bundle: (submod "forms.rkt" bundle)) + (prefix-in effect: (submod "forms.rkt" effect)) + (prefix-in live?: (submod "forms.rkt" live?)) + (prefix-in rectify: (submod "forms.rkt" rectify)) + (prefix-in pass: (submod "forms.rkt" pass)) + (prefix-in foldl: (submod "forms.rkt" foldl)) + (prefix-in foldr: (submod "forms.rkt" foldr)) + (prefix-in loop: (submod "forms.rkt" loop)) + (prefix-in loop2: (submod "forms.rkt" loop2)) + (prefix-in apply: (submod "forms.rkt" apply)) + (prefix-in clos: (submod "forms.rkt" clos))) + +(require racket/match + racket/format + relation + qi + json + (only-in "util.rkt" + only-if + for/call)) + +;; It would be great if we could get the value of a variable +;; by using its (string) name, but (eval (string->symbol name)) +;; doesn't find it. So instead, we reify the "lexical environment" +;; here manually, so that the values can be looked up at runtime +;; based on the string names (note that the value is always the key +;; + ":" + "run") +(define env + (hash + "one-of?" one-of?:run + "and" and:run + "or" or:run + "not" not:run + "and%" and%:run + "or%" or%:run + "group" group:run + "count" count:run + "relay" relay:run + "relay*" relay*:run + "amp" amp:run + "ground" ground:run + "thread" thread:run + "thread-right" thread-right:run + "crossover" crossover:run + "all" all:run + "any" any:run + "none" none:run + "all?" all?:run + "any?" any?:run + "none?" none?:run + "collect" collect:run + "sep" sep:run + "gen" gen:run + "esc" esc:run + "AND" AND:run + "OR" OR:run + "NOT" NOT:run + "NAND" NAND:run + "NOR" NOR:run + "XOR" XOR:run + "XNOR" XNOR:run + "tee" tee:run + "try" try:run + "currying" currying:run + "template" template:run + "catchall-template" catchall-template:run + "if" if:run + "when" when:run + "unless" unless:run + "switch" switch:run + "sieve" sieve:run + "gate" gate:run + "input-aliases" input-aliases:run + "fanout" fanout:run + "inverter" inverter:run + "feedback" feedback:run + "select" select:run + "block" block:run + "bundle" bundle:run + "effect" effect:run + "live?" live?:run + "rectify" rectify:run + "pass" pass:run + "foldl" foldl:run + "foldr" foldr:run + "loop" loop:run + "loop2" loop2:run + "apply" apply:run + "clos" clos:run)) + +(program (main) + (let* ([forms (hash-keys env)] + [fs (~>> (forms) + (sort <))]) + (write-json (for/list ([f fs]) + (match-let ([(list name ms) ((hash-ref env f))]) + (hash 'name name 'unit "ms" 'value ms)))))) + +(run main) diff --git a/profile/util.rkt b/profile/util.rkt index 8ff8bd2c..cb01018a 100644 --- a/profile/util.rkt +++ b/profile/util.rkt @@ -76,7 +76,7 @@ (symbol->string (syntax->datum #'f-name))) (let ([ms (measure runner f-name n-times)]) - (displayln (~a name ": " ms " ms")))) + (list name ms))) ;; Run many benchmarking functions (typically exercising a single form) ;; a specified number of times and report the time taken using the @@ -89,7 +89,7 @@ (let ([ms (measure r f n)]) (set! results (cons ms results)))) (let ([summarized-result (apply f-summary results)]) - (displayln (~a name ": " summarized-result " ms"))))) + (list name summarized-result)))) ;; Run different implementations of the same benchmark (e.g. a Racket vs a Qi ;; implementation) a specified number of times, and report the time taken diff --git a/qi/info.rkt b/qi/info.rkt index ddc24574..c91cdb60 100644 --- a/qi/info.rkt +++ b/qi/info.rkt @@ -8,7 +8,7 @@ "qi-test" "qi-probe" "Qi-Quickscripts")) -(define build-deps '()) +(define build-deps '("cli")) (define implies '("qi-lib" "qi-doc" "qi-test"