diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 374cd708..a5589904 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -92,6 +92,38 @@ jobs: name: results-vs path: benchmarks/results/* + benchmark-ws: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - uses: pyo3/maturin-action@v1 + with: + command: build + args: --release --interpreter python3.11 + target: x64 + manylinux: auto + container: off + - run: | + export _whl=$(ls target/wheels/granian-*.whl) + pip install $_whl + - name: deps + run: | + pip install -r benchmarks/envs/asgi.txt + pip install websockets + - name: benchmark + working-directory: ./benchmarks + run: | + python benchmarks.py vs_ws + - name: upload results + uses: actions/upload-artifact@v4 + with: + name: results-ws + path: benchmarks/results/* + benchmark-pyver: runs-on: ubuntu-latest needs: [toolchain] @@ -146,7 +178,7 @@ jobs: results: runs-on: ubuntu-latest - needs: [benchmark-base, benchmark-vs, benchmark-pyver] + needs: [benchmark-base, benchmark-vs, benchmark-ws, benchmark-pyver] steps: - uses: actions/checkout@v4 @@ -163,6 +195,12 @@ jobs: path: benchmarks/results - run: | mv benchmarks/results/data.json benchmarks/results/vs.json + - uses: actions/download-artifact@v4 + with: + name: results-ws + path: benchmarks/results + - run: | + mv benchmarks/results/data.json benchmarks/results/ws.json - uses: actions/download-artifact@v4 with: name: results-pyver @@ -171,7 +209,7 @@ jobs: working-directory: ./benchmarks run: | noir -c data:results/base.json -v 'benv=GHA Linux x86_64' templates/main.md > README.md - noir -c data:results/vs.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md + noir -c data:results/vs.json -c wsdata:results/ws.json -v 'benv=GHA Linux x86_64' templates/vs.md > vs.md noir \ -c data310:results/py310.json \ -c data311:results/py311.json \ diff --git a/benchmarks/README.md b/benchmarks/README.md index 773bf395..524ed6c0 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,10 +2,10 @@ -Run at: Tue 03 Sep 2024, 21:44 +Run at: Mon 28 Oct 2024, 02:09 Environment: GHA Linux x86_64 (CPUs: 4) Python version: 3.11 -Granian version: 1.6.0 +Granian version: 1.6.2 ## RSGI response types @@ -14,10 +14,10 @@ The "small" response is 4 bytes, the "big" one is 80kbytes. | Type | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| bytes small (c64) | 413639 | 41365 | 1.544ms | 4.612ms | -| str small (c128) | 433016 | 43346 | 2.944ms | 19.673ms | -| bytes big (c64) | 281576 | 28162 | 2.267ms | 5.781ms | -| str big (c64) | 288430 | 28851 | 2.211ms | 6.574ms | +| bytes small (c256) | 392764 | 39392 | 6.47ms | 61.121ms | +| str small (c512) | 424544 | 42723 | 11.921ms | 130.529ms | +| bytes big (c64) | 288745 | 28874 | 2.212ms | 6.843ms | +| str big (c64) | 276754 | 27681 | 2.305ms | 5.861ms | ## Interfaces @@ -29,15 +29,15 @@ The "echo" request is a 4bytes POST request responding with the same body. | Request | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| RSGI bytes (c128) | 423456 | 42379 | 3.009ms | 27.917ms | -| RSGI str (c128) | 428954 | 42942 | 2.972ms | 24.262ms | -| RSGI echo (c256) | 377133 | 37812 | 6.735ms | 74.44ms | -| ASGI bytes (c64) | 436301 | 43632 | 1.463ms | 4.525ms | -| ASGI str (c64) | 422612 | 42269 | 1.51ms | 4.182ms | -| ASGI echo (c512) | 217386 | 21864 | 23.309ms | 121.784ms | -| WSGI bytes (c128) | 367719 | 36827 | 3.463ms | 18.241ms | -| WSGI str (c64) | 373343 | 37340 | 1.71ms | 4.425ms | -| WSGI echo (c64) | 342874 | 34293 | 1.861ms | 4.515ms | +| RSGI bytes (c64) | 397815 | 39792 | 1.602ms | 4.506ms | +| RSGI str (c64) | 414448 | 41451 | 1.541ms | 4.152ms | +| RSGI echo (c256) | 366975 | 36780 | 6.93ms | 59.373ms | +| ASGI bytes (c512) | 394155 | 39596 | 12.855ms | 143.895ms | +| ASGI str (c64) | 402564 | 40258 | 1.585ms | 3.888ms | +| ASGI echo (c128) | 217107 | 21742 | 5.864ms | 31.625ms | +| WSGI bytes (c128) | 375703 | 37613 | 3.386ms | 39.83ms | +| WSGI str (c256) | 371867 | 37268 | 6.837ms | 58.755ms | +| WSGI echo (c128) | 352872 | 35325 | 3.609ms | 33.688ms | ## HTTP/2 @@ -46,10 +46,10 @@ Comparison between Granian HTTP versions on RSGI using 4bytes plain text respons | Request | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| HTTP/1 [GET] (c128) | 409169 | 40964 | 3.11ms | 35.937ms | -| HTTP/1 [POST] (c256) | 367186 | 36816 | 6.927ms | 63.568ms | -| HTTP/2 [GET] (c64) | 429019 | 42908 | 1.489ms | 6.358ms | -| HTTP/2 [POST] (c64) | 279182 | 27921 | 2.287ms | 6.201ms | +| HTTP/1 [GET] (c64) | 402839 | 40293 | 1.584ms | 3.983ms | +| HTTP/1 [POST] (c64) | 361471 | 36152 | 1.766ms | 5.997ms | +| HTTP/2 [GET] (c128) | 411331 | 41181 | 3.097ms | 35.375ms | +| HTTP/2 [POST] (c256) | 285962 | 28684 | 8.881ms | 102.249ms | ## File responses @@ -59,15 +59,16 @@ WSGI is not part of the benchmark since the protocol doesn't implement anything | Request | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| RSGI (c64) | 340095 | 34014 | 1.877ms | 3.9ms | -| ASGI (c256) | 168845 | 16957 | 14.998ms | 81.803ms | -| ASGI pathsend (c64) | 299960 | 29998 | 2.129ms | 4.663ms | +| RSGI (c64) | 336910 | 33698 | 1.894ms | 9.23ms | +| ASGI (c64) | 167204 | 16723 | 3.818ms | 5.76ms | +| ASGI pathsend (c128) | 290053 | 29038 | 4.392ms | 28.584ms | ### Other benchmarks - [Versus 3rd party servers](./vs.md) - [Concurrency](./concurrency.md) +- [Python versions](./pyver.md) ### 3rd party benchmarks diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index d3c6fab9..e4987613 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -11,37 +11,38 @@ CPU = multiprocessing.cpu_count() WRK_CONCURRENCIES = [64, 128, 256, 512] +WS_CONCURRENCIES = [(8, 20_000), (16, 10_000), (32, 5000), (64, 2500)] APPS = { 'asgi': ( 'granian --interface asgi --log-level warning --backlog 2048 ' - '--no-ws --http {http} ' + '{wsmode}--http {http} ' '--workers {procs} --threads {threads}{bthreads} ' - '--threading-mode {thmode} app.asgi:app' + '--threading-mode {thmode} {app}.asgi:app' ), 'rsgi': ( 'granian --interface rsgi --log-level warning --backlog 2048 ' - '--no-ws --http {http} ' + '{wsmode}--http {http} ' '--workers {procs} --threads {threads}{bthreads} ' - '--threading-mode {thmode} app.rsgi:app' + '--threading-mode {thmode} {app}.rsgi:app' ), 'wsgi': ( 'granian --interface wsgi --log-level warning --backlog 2048 ' - '--no-ws --http {http} ' + '{wsmode}--http {http} ' '--workers {procs} --threads {threads}{bthreads} ' '--threading-mode {thmode} app.wsgi:app' ), 'uvicorn_h11': ( - 'uvicorn --interface asgi3 --no-access-log --log-level warning --http h11 --workers {procs} app.asgi:app' + 'uvicorn --interface asgi3 --no-access-log --log-level warning --http h11 --workers {procs} {app}.asgi:app' ), 'uvicorn_httptools': ( 'uvicorn --interface asgi3 ' '--no-access-log --log-level warning ' - '--http httptools --workers {procs} app.asgi:app' + '--http httptools --workers {procs} {app}.asgi:app' ), 'hypercorn': ( 'hypercorn -b localhost:8000 -k uvloop --log-level warning --backlog 2048 ' - '--workers {procs} asgi:app.asgi:async_app' + '--workers {procs} asgi:{app}.asgi:async_app' ), 'gunicorn_gthread': 'gunicorn --workers {procs} -k gthread app.wsgi:app', 'gunicorn_gevent': 'gunicorn --workers {procs} -k gevent app.wsgi:app', @@ -54,18 +55,21 @@ @contextmanager -def app(name, procs=None, threads=None, bthreads=None, thmode=None, http='1'): +def app(name, procs=None, threads=None, bthreads=None, thmode=None, http='1', ws=False, app_path='app'): procs = procs or 1 threads = threads or 1 bthreads = f' --blocking-threads {bthreads}' if bthreads else '' thmode = thmode or 'workers' + wsmode = '--no-ws ' if not ws else '' exc_prefix = os.environ.get('BENCHMARK_EXC_PREFIX') proc_cmd = APPS[name].format( + app=app_path, procs=procs, threads=threads, bthreads=bthreads, thmode=thmode, http=http, + wsmode=wsmode, ) if exc_prefix: proc_cmd = f'{exc_prefix}/{proc_cmd}' @@ -113,6 +117,44 @@ def wrk(duration, concurrency, endpoint, post=False, h2=False): } +def wsb(concurrency, msgs): + exc_prefix = os.environ.get('BENCHMARK_EXC_PREFIX') + cmd_parts = [ + f'{exc_prefix}/python' if exc_prefix else 'python', + os.path.join(os.path.dirname(__file__), 'ws/benchmark.py'), + ] + env = dict(os.environ) + try: + proc = subprocess.run( # noqa: S602 + ' '.join(cmd_parts), + shell=True, + check=True, + capture_output=True, + env={ + 'BENCHMARK_CONCURRENCY': str(concurrency), + 'BENCHMARK_MSGNO': str(msgs), + **env + } + ) + return json.loads(proc.stdout.decode('utf8')) + except Exception as e: + print(f'WARN: got exception {e} while loading wsbench data') + return { + 'timings': { + 'recv': {'avg': 0, 'max': 0, 'min': 0}, + 'send': {'avg': 0, 'max': 0, 'min': 0}, + 'sum': {'avg': 0, 'max': 0, 'min': 0}, + 'all': {'avg': 0, 'max': 0, 'min': 0}, + }, + 'throughput': { + 'recv': 0, + 'send': 0, + 'all': 0, + 'sum': 0, + }, + } + + def benchmark(endpoint, post=False, h2=False, concurrencies=None): concurrencies = concurrencies or WRK_CONCURRENCIES results = {} @@ -131,6 +173,17 @@ def benchmark(endpoint, post=False, h2=False, concurrencies=None): return results +def benchmark_ws(concurrencies=None): + concurrencies = concurrencies or WS_CONCURRENCIES + results = {} + # bench + for concurrency, msgs in concurrencies: + res = wsb(concurrency, msgs) + results[concurrency] = res + time.sleep(2) + return results + + def concurrencies(): nperm = sorted({1, 2, 4, round(CPU / 2), CPU}) results = {'wsgi': {}} @@ -264,6 +317,21 @@ def vs_io(): return results +def vs_ws(): + results = {} + for fw in [ + 'granian_rsgi', + 'granian_asgi', + 'uvicorn_h11', + 'hypercorn', + ]: + fw_app = fw.split('_')[1] if fw.startswith('granian') else fw + title = ' '.join(item.title() for item in fw.split('_')) + with app(fw_app, ws=True, app_path='ws.app'): + results[title] = benchmark_ws() + return results + + def _granian_version(): import granian @@ -282,6 +350,7 @@ def run(): 'vs_http2': vs_http2, 'vs_files': vs_files, 'vs_io': vs_io, + 'vs_ws': vs_ws, } inp_benchmarks = sys.argv[1:] or ['base'] if 'base' in inp_benchmarks: diff --git a/benchmarks/pyver.md b/benchmarks/pyver.md new file mode 100644 index 00000000..9cbb565a --- /dev/null +++ b/benchmarks/pyver.md @@ -0,0 +1,53 @@ +# Granian benchmarks + + + +## Python versions + +Run at: Mon 28 Oct 2024, 02:17 +Environment: GHA Linux x86_64 (CPUs: 4) +Granian version: 1.6.2 + +Comparison between different Python versions of Granian application protocols using 4bytes plain text response. +Bytes and string response are reported for every protocol just to report the difference with RSGI protocol. +ASGI and WSGI responses are always returned as bytes by the application. +The "echo" request is a 4bytes POST request responding with the same body. + +| Python version | Request | Total requests | RPS | avg latency | max latency | +| --- | --- | --- | --- | --- | --- | +| 3.10 | RSGI bytes (c64) | 445347 | 44540 | 1.433ms | 4.16ms | +| 3.10 | RSGI str (c64) | 407154 | 40713 | 1.569ms | 4.146ms | +| 3.10 | RSGI echo (c64) | 409490 | 40952 | 1.559ms | 3.98ms | +| 3.10 | ASGI bytes (c64) | 404741 | 40475 | 1.577ms | 3.853ms | +| 3.10 | ASGI str (c64) | 443742 | 44379 | 1.439ms | 3.743ms | +| 3.10 | ASGI echo (c64) | 288973 | 28904 | 2.208ms | 4.757ms | +| 3.10 | WSGI bytes (c128) | 559148 | 55978 | 2.28ms | 24.538ms | +| 3.10 | WSGI str (c256) | 567857 | 56935 | 4.476ms | 66.155ms | +| 3.10 | WSGI echo (c512) | 501010 | 50392 | 10.098ms | 117.374ms | +| 3.11 | RSGI bytes (c64) | 408906 | 40897 | 1.56ms | 3.754ms | +| 3.11 | RSGI str (c64) | 444267 | 44427 | 1.437ms | 4.131ms | +| 3.11 | RSGI echo (c256) | 375214 | 37628 | 6.77ms | 69.911ms | +| 3.11 | ASGI bytes (c128) | 394648 | 39496 | 3.232ms | 19.781ms | +| 3.11 | ASGI str (c512) | 416410 | 41848 | 12.182ms | 104.538ms | +| 3.11 | ASGI echo (c64) | 220900 | 22093 | 2.888ms | 5.823ms | +| 3.11 | WSGI bytes (c128) | 381549 | 38197 | 3.339ms | 24.674ms | +| 3.11 | WSGI str (c128) | 377113 | 37754 | 3.377ms | 32.506ms | +| 3.11 | WSGI echo (c256) | 342484 | 34363 | 7.42ms | 66.93ms | +| 3.12 | RSGI bytes (c512) | 423560 | 42590 | 11.958ms | 141.381ms | +| 3.12 | RSGI str (c512) | 427772 | 43035 | 11.842ms | 114.818ms | +| 3.12 | RSGI echo (c512) | 361753 | 36365 | 13.995ms | 169.242ms | +| 3.12 | ASGI bytes (c128) | 419544 | 42014 | 3.032ms | 36.038ms | +| 3.12 | ASGI str (c64) | 415682 | 41567 | 1.536ms | 3.97ms | +| 3.12 | ASGI echo (c64) | 212445 | 21248 | 3.004ms | 6.795ms | +| 3.12 | WSGI bytes (c256) | 356792 | 35798 | 7.121ms | 64.292ms | +| 3.12 | WSGI str (c128) | 362003 | 36244 | 3.52ms | 20.129ms | +| 3.12 | WSGI echo (c512) | 322632 | 32432 | 15.676ms | 144.224ms | +| 3.13 | RSGI bytes (c512) | 400784 | 40264 | 12.635ms | 175.762ms | +| 3.13 | RSGI str (c64) | 407448 | 40745 | 1.568ms | 4.455ms | +| 3.13 | RSGI echo (c128) | 384481 | 38485 | 3.315ms | 23.927ms | +| 3.13 | ASGI bytes (c512) | 414131 | 41618 | 12.238ms | 147.581ms | +| 3.13 | ASGI str (c64) | 449520 | 44963 | 1.419ms | 3.919ms | +| 3.13 | ASGI echo (c128) | 225465 | 22567 | 5.648ms | 36.339ms | +| 3.13 | WSGI bytes (c128) | 371324 | 37194 | 3.429ms | 18.313ms | +| 3.13 | WSGI str (c64) | 369008 | 36910 | 1.729ms | 5.06ms | +| 3.13 | WSGI echo (c128) | 363711 | 36408 | 3.505ms | 20.864ms | diff --git a/benchmarks/templates/_helpers.tpl b/benchmarks/templates/_helpers.tpl index 661b1e9d..5ef2f5be 100644 --- a/benchmarks/templates/_helpers.tpl +++ b/benchmarks/templates/_helpers.tpl @@ -1,6 +1,6 @@ {{ - def get_max_concurrency_run(data): - concurrency_values = {data[ckey]["requests"]["rps"]: ckey for ckey in data.keys()} + def get_max_concurrency_run(data, k1="requests", k2="rps"): + concurrency_values = {data[ckey][k1][k2]: ckey for ckey in data.keys()} maxc = concurrency_values[max(concurrency_values.keys())] return maxc, data[maxc] diff --git a/benchmarks/templates/_vs_ws_table.tpl b/benchmarks/templates/_vs_ws_table.tpl new file mode 100644 index 00000000..01718956 --- /dev/null +++ b/benchmarks/templates/_vs_ws_table.tpl @@ -0,0 +1,8 @@ +| Server | Send throughput | Receive throughput | Combined throughput | +| --- | --- | --- | --- | +{{ for key, runs in _data.items(): }} +{{ max_c, run = get_max_concurrency_run(runs, "throughput", "sum") }} +{{ if run["throughput"]["sum"]: }} +| {{ =key }} (c{{ =max_c }}) | {{ =round(run["throughput"]["send"]) }} | {{ =round(run["throughput"]["recv"]) }} | {{ =round(run["throughput"]["sum"]) }} | +{{ pass }} +{{ pass }} diff --git a/benchmarks/templates/vs.md b/benchmarks/templates/vs.md index 11e9f2e7..7584fcc3 100644 --- a/benchmarks/templates/vs.md +++ b/benchmarks/templates/vs.md @@ -35,3 +35,12 @@ Plain text 4 bytes response comparison simulating *long* I/O waits (10ms and 100 {{ _data = data.results["vs_io"] }} {{ include './_vs_table.tpl' }} + +{{ if wsdata := globals().get("wsdata"): }} +### Websockets + +Websocket broadcasting comparison with concurrent clients sending a predefined amount of messages and receiving those messages from all the connected clients. The benchmark takes the time required for the test to run and compute the relevant throughput (in messages per second). + +{{ _data = wsdata.results["vs_ws"] }} +{{ include './_vs_ws_table.tpl' }} +{{ pass }} diff --git a/benchmarks/vs.md b/benchmarks/vs.md index f00ca5c7..ff753afc 100644 --- a/benchmarks/vs.md +++ b/benchmarks/vs.md @@ -4,57 +4,57 @@ ## VS 3rd party comparison -Run at: Tue 03 Sep 2024, 21:44 +Run at: Mon 28 Oct 2024, 02:10 Environment: GHA Linux x86_64 (CPUs: 4) Python version: 3.11 -Granian version: 1.6.0 +Granian version: 1.6.2 ### ASGI | Server | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| Granian Asgi [GET] (c128) | 453458 | 45416 | 2.806ms | 31.751ms | -| Granian Asgi [POST] (c64) | 219795 | 21983 | 2.904ms | 6.354ms | -| Uvicorn H11 [GET] (c64) | 80046 | 8006 | 7.972ms | 18.857ms | -| Uvicorn H11 [POST] (c64) | 71376 | 7139 | 8.942ms | 24.293ms | -| Uvicorn Httptools [GET] (c128) | 371460 | 37210 | 3.424ms | 32.85ms | -| Uvicorn Httptools [POST] (c128) | 342512 | 34301 | 3.718ms | 24.748ms | -| Hypercorn [GET] (c128) | 49725 | 4978 | 25.595ms | 31.829ms | -| Hypercorn [POST] (c128) | 45388 | 4547 | 28.022ms | 31.55ms | +| Granian Asgi [GET] (c64) | 412041 | 41213 | 1.549ms | 4.255ms | +| Granian Asgi [POST] (c64) | 221079 | 22111 | 2.888ms | 5.488ms | +| Uvicorn H11 [GET] (c64) | 75123 | 7513 | 8.5ms | 22.009ms | +| Uvicorn H11 [POST] (c64) | 66085 | 6609 | 9.657ms | 26.091ms | +| Uvicorn Httptools [GET] (c128) | 339905 | 34045 | 3.745ms | 23.819ms | +| Uvicorn Httptools [POST] (c128) | 314202 | 31455 | 4.055ms | 23.736ms | +| Hypercorn [GET] (c128) | 48884 | 4893 | 26.054ms | 42.096ms | +| Hypercorn [POST] (c128) | 42517 | 4257 | 29.922ms | 54.175ms | ### WSGI | Server | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| Granian Wsgi [GET] (c256) | 380226 | 38107 | 6.691ms | 57.374ms | -| Granian Wsgi [POST] (c128) | 332496 | 33295 | 3.829ms | 33.119ms | -| Gunicorn Gthread [GET] (c64) | 36866 | 3687 | 17.304ms | 19.372ms | -| Gunicorn Gthread [POST] (c64) | 35502 | 3551 | 17.962ms | 20.207ms | -| Gunicorn Gevent [GET] (c64) | 63589 | 6361 | 6.593ms | 7589.315ms | -| Gunicorn Gevent [POST] (c256) | 60097 | 6033 | 7.233ms | 9884.01ms | -| Uwsgi [GET] (c256) | 72701 | 7286 | 33.627ms | 4735.69ms | -| Uwsgi [POST] (c256) | 72121 | 7227 | 34.111ms | 3470.737ms | +| Granian Wsgi [GET] (c256) | 381571 | 38324 | 6.644ms | 70.442ms | +| Granian Wsgi [POST] (c128) | 347832 | 34819 | 3.662ms | 34.109ms | +| Gunicorn Gthread [GET] (c64) | 36680 | 3669 | 17.386ms | 19.742ms | +| Gunicorn Gthread [POST] (c64) | 35078 | 3508 | 18.191ms | 20.25ms | +| Gunicorn Gevent [GET] (c128) | 62624 | 6270 | 7.909ms | 9876.337ms | +| Gunicorn Gevent [POST] (c128) | 57223 | 5729 | 12.683ms | 9777.452ms | +| Uwsgi [GET] (c64) | 71314 | 7131 | 8.955ms | 16.786ms | +| Uwsgi [POST] (c64) | 71214 | 7122 | 8.964ms | 13.792ms | ### HTTP/2 | Server | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| Granian Asgi [GET] (c64) | 373977 | 37408 | 1.706ms | 6.236ms | -| Granian Asgi [POST] (c256) | 197345 | 19804 | 12.862ms | 92.805ms | -| Hypercorn [GET] (c64) | 31630 | 3164 | 20.144ms | 48.95ms | -| Hypercorn [POST] (c64) | 28163 | 2817 | 22.641ms | 69.083ms | +| Granian Asgi [GET] (c128) | 346031 | 34673 | 3.673ms | 44.303ms | +| Granian Asgi [POST] (c64) | 196285 | 19633 | 3.25ms | 10.419ms | +| Hypercorn [GET] (c64) | 30507 | 3051 | 20.914ms | 50.49ms | +| Hypercorn [POST] (c64) | 27155 | 2716 | 23.491ms | 56.879ms | ### ASGI file responses | Server | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| Granian (pathsend) (c64) | 301264 | 30123 | 2.12ms | 5.324ms | -| Uvicorn H11 (c128) | 78809 | 7893 | 16.152ms | 27.46ms | -| Uvicorn Httptools (c128) | 207081 | 20727 | 6.152ms | 27.753ms | -| Hypercorn (c128) | 49403 | 4945 | 25.77ms | 32.498ms | +| Granian (pathsend) (c64) | 296068 | 29608 | 2.157ms | 4.387ms | +| Uvicorn H11 (c128) | 74931 | 7505 | 16.979ms | 43.618ms | +| Uvicorn Httptools (c64) | 190766 | 19080 | 3.347ms | 7.536ms | +| Hypercorn (c128) | 47773 | 4782 | 26.646ms | 44.924ms | ### Long I/O @@ -63,18 +63,29 @@ Plain text 4 bytes response comparison simulating *long* I/O waits (10ms and 100 | Server | Total requests | RPS | avg latency | max latency | | --- | --- | --- | --- | --- | -| Granian Rsgi 10ms (c512) | 400686 | 40280 | 12.66ms | 105.734ms | -| Granian Rsgi 100ms (c512) | 50049 | 5033 | 100.68ms | 157.381ms | -| Granian Asgi 10ms (c512) | 400154 | 40203 | 12.673ms | 113.876ms | -| Granian Asgi 100ms (c512) | 50070 | 5037 | 100.641ms | 171.637ms | -| Granian Wsgi 10ms (c128) | 112522 | 11265 | 11.315ms | 31.271ms | -| Granian Wsgi 100ms (c512) | 50253 | 5054 | 100.295ms | 150.603ms | -| Uvicorn Httptools 10ms (c512) | 246558 | 24794 | 20.545ms | 104.779ms | -| Uvicorn Httptools 100ms (c512) | 50019 | 5028 | 100.811ms | 174.095ms | -| Hypercorn 10ms (c128) | 49761 | 4982 | 25.581ms | 42.662ms | -| Hypercorn 100ms (c128) | 49443 | 4949 | 25.719ms | 43.894ms | -| Gunicorn Gevent 10ms (c64) | 57483 | 5750 | 11.088ms | 25.63ms | -| Gunicorn Gevent 100ms (c512) | 48377 | 4862 | 104.317ms | 171.487ms | -| Uwsgi 10ms (c256) | 72900 | 7311 | 34.199ms | 6609.152ms | -| Uwsgi 100ms (c512) | 72643 | 7304 | 61.63ms | 7512.077ms | +| Granian Rsgi 10ms (c512) | 398741 | 40076 | 12.704ms | 122.917ms | +| Granian Rsgi 100ms (c512) | 50074 | 5035 | 100.659ms | 161.13ms | +| Granian Asgi 10ms (c512) | 368346 | 37047 | 13.753ms | 126.133ms | +| Granian Asgi 100ms (c512) | 50056 | 5031 | 100.724ms | 151.241ms | +| Granian Wsgi 10ms (c128) | 109932 | 11005 | 11.578ms | 34.242ms | +| Granian Wsgi 100ms (c512) | 50218 | 5050 | 100.365ms | 162.437ms | +| Uvicorn Httptools 10ms (c256) | 227853 | 22854 | 11.14ms | 67.594ms | +| Uvicorn Httptools 100ms (c512) | 50074 | 5028 | 100.869ms | 181.026ms | +| Hypercorn 10ms (c128) | 48782 | 4884 | 26.081ms | 46.542ms | +| Hypercorn 100ms (c128) | 49325 | 4938 | 25.812ms | 46.076ms | +| Gunicorn Gevent 10ms (c128) | 55121 | 5519 | 23.096ms | 60.147ms | +| Gunicorn Gevent 100ms (c512) | 47896 | 4812 | 105.287ms | 191.957ms | +| Uwsgi 10ms (c256) | 71141 | 7136 | 34.26ms | 3345.351ms | +| Uwsgi 100ms (c256) | 70790 | 7102 | 34.882ms | 3071.967ms | + + +### Websockets + +Websocket broadcasting comparison with concurrent clients sending a predefined amount of messages and receiving those messages from all the connected clients. The benchmark takes the time required for the test to run and compute the relevant throughput (in messages per second). + +| Server | Send throughput | Receive throughput | Combined throughput | +| --- | --- | --- | --- | +| Granian Rsgi (c16) | 962877 | 76296 | 81065 | +| Granian Asgi (c16) | 941480 | 73632 | 78234 | +| Uvicorn H11 (c8) | 456731 | 80133 | 90149 | diff --git a/benchmarks/ws/app/asgi.py b/benchmarks/ws/app/asgi.py new file mode 100644 index 00000000..1285e684 --- /dev/null +++ b/benchmarks/ws/app/asgi.py @@ -0,0 +1,23 @@ +clients = set() + + +async def broadcast(message): + for ws in list(clients): + await ws({'type': 'websocket.send', 'bytes': message, 'text': None}) + + +async def app(scope, receive, send): + try: + await send({'type': 'websocket.accept'}) + clients.add(send) + + while True: + msg = await receive() + if msg['type'] == 'websocket.connect': + continue + if msg['type'] == 'websocket.disconnect': + break + await broadcast(msg['bytes']) + + finally: + clients.remove(send) diff --git a/benchmarks/ws/app/rsgi.py b/benchmarks/ws/app/rsgi.py new file mode 100644 index 00000000..4a7a62d1 --- /dev/null +++ b/benchmarks/ws/app/rsgi.py @@ -0,0 +1,25 @@ +from granian.rsgi import WebsocketMessageType + + +clients = set() + + +async def broadcast(message): + for ws in list(clients): + await ws.send_bytes(message) + + +async def app(scope, protocol): + trx = await protocol.accept() + clients.add(trx) + + try: + while True: + message = await trx.receive() + if message.kind == WebsocketMessageType.close: + break + await broadcast(message.data) + + finally: + clients.remove(trx) + protocol.close() diff --git a/benchmarks/ws/benchmark.py b/benchmarks/ws/benchmark.py new file mode 100644 index 00000000..bb7aa5fa --- /dev/null +++ b/benchmarks/ws/benchmark.py @@ -0,0 +1,133 @@ +import asyncio +import json +import os +import time + +import websockets + + +CONCURRENCY = int(os.environ.get('BENCHMARK_CONCURRENCY', '16')) +MSG_NO = int(os.environ.get('BENCHMARK_MSGNO', '2500')) +RUNS = int(os.environ.get('BENCHMARK_RUNS', '1')) +MEASUREMENTS = [] + +ready = asyncio.Event() + + +def _client_redy(target): + acks = {'all': 0} + + def inner(): + acks['all'] += 1 + if acks['all'] == target: + ready.set() + + return inner + + +async def _client_recv(ws, to_recv): + t_start = time.time() + recv = 0 + while recv < to_recv: + await ws.recv() + recv += 1 + return (recv, time.time() - t_start) + + +async def _client_send(ws, messages): + t_start = time.time() + sent = 0 + for message in messages: + await ws.send(message) + sent += 1 + return (sent, time.time() - t_start) + + +async def client(idx, messages, ready_signal): + # print(f'Starting client {idx}..') + async with websockets.connect('ws://127.0.0.1:8000') as ws: + ready_signal() + await ready.wait() + # print(f'Client {idx} ready') + await asyncio.sleep(3) + + _task_recv, _task_send = ( + asyncio.create_task(_client_recv(ws, len(messages) * CONCURRENCY)), + asyncio.create_task(_client_send(ws, messages)) + ) + t_start = time.time() + recv_data, send_data = await asyncio.gather(_task_recv, _task_send) + t_end = time.time() - t_start + # print(f'Client {idx} terminated') + return (recv_data, send_data, t_end) + + +async def benchmark(run): + # print(f'Starting benchmark run {run}..') + messages = [f'msg{str(idx).zfill(len(str(MSG_NO)))}'.encode('utf8') for idx in range(MSG_NO)] + client_ready = _client_redy(CONCURRENCY) + tasks = [] + for idx in range(CONCURRENCY): + tasks.append(asyncio.create_task(client(idx, messages, client_ready))) + res = await asyncio.gather(*tasks) + MEASUREMENTS.append(res) + ready.clear() + # print(f'Completed benchmark run {run}') + + +def build_results(data): + rv = [] + for run in data: + tot_timings = [] + recv_timings = [] + send_timings = [] + sum_timings = [] + tot_throughput = [] + recv_throughput = [] + send_throughput = [] + sum_throughput = [] + for recv_data, send_data, client_time in run: + recv_timings.append(recv_data[1]) + send_timings.append(send_data[1]) + tot_timings.append(client_time) + sum_timings.append(max(recv_data[1], send_data[1])) + recv_throughput.append(MSG_NO * CONCURRENCY / recv_data[1]) + send_throughput.append(MSG_NO / send_data[1]) + tot_throughput.append(MSG_NO * (CONCURRENCY + 1) / client_time) + sum_throughput.append(MSG_NO * (CONCURRENCY + 1) / sum_timings[-1]) + + recv_avg, recv_max, recv_min = sum(recv_timings) / len(recv_timings), max(recv_timings), min(recv_timings) + send_avg, send_max, send_min = sum(send_timings) / len(send_timings), max(send_timings), min(send_timings) + tot_avg, tot_max, tot_min = sum(tot_timings) / len(tot_timings), max(tot_timings), min(tot_timings) + sum_avg, sum_max, sum_min = sum(sum_timings) / len(sum_timings), max(sum_timings), min(sum_timings) + th_recv, th_send, th_all, th_sum = sum(recv_throughput), sum(send_throughput), sum(tot_throughput), sum(sum_throughput) + res = { + 'timings': { + 'recv': {'avg': recv_avg, 'max': recv_max, 'min': recv_min}, + 'send': {'avg': send_avg, 'max': send_max, 'min': send_min}, + 'sum': {'avg': sum_avg, 'max': sum_max, 'min': sum_min}, + 'all': {'avg': tot_avg, 'max': tot_max, 'min': tot_min}, + }, + 'throughput': { + 'recv': th_recv, + 'send': th_send, + 'all': th_all, + 'sum': th_sum, + } + } + rv.append(res) + return rv + + +async def main(): + for idx in range(RUNS): + await benchmark(idx + 1) + + res = build_results(MEASUREMENTS) + res.sort(key=lambda item: item['throughput']['sum']) + ridx = 2 if len(res) > 1 else 1 + print(json.dumps(res[-ridx])) + + +if __name__ == '__main__': + asyncio.run(main())