Skip to content

Commit

Permalink
Update run_standalone_temp.sh to the new folder specification (#1990)
Browse files Browse the repository at this point in the history
  • Loading branch information
mamcx authored Nov 12, 2024
1 parent 97bff92 commit 2976593
Showing 1 changed file with 7 additions and 21 deletions.
28 changes: 7 additions & 21 deletions run_standalone_temp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,11 @@
# Run a ephemeral database inside a `temp` folder
set -euo pipefail

SRC_TREE="$(dirname "$0")"
cd "$SRC_TREE"

declare -r DB_CONFIG_HOME="$HOME/.spacetime/db"
mkdir -p "$DB_CONFIG_HOME"

export SPACETIMEDB_JWT_PRIV_KEY="${SPACETIMEDB_JWT_PRIV_KEY:-$DB_CONFIG_HOME/standalone_id_ecdsa}"
export SPACETIMEDB_JWT_PUB_KEY="${SPACETIMEDB_JWT_PRIV_KEY}.pub"
cd "$(dirname "$0")"

cargo build -p spacetimedb-standalone

export STDB_PATH="${STDB_PATH:-$(mktemp -d)}"
mkdir -p "$STDB_PATH/logs"
STDB_PATH="${STDB_PATH:-$(mktemp -d)}"

function cleanup {
echo "Removing ${STDB_PATH}"
Expand All @@ -23,16 +15,10 @@ function cleanup {

trap cleanup EXIT

cp crates/standalone/log.conf "$STDB_PATH/log.conf"
# -i differs between GNU and BSD sed, so use a temp file
sed 's/spacetimedb=debug/spacetimedb=trace/g' "$STDB_PATH/log.conf" > "$STDB_PATH/log.conf.tmp" && \
mv "$STDB_PATH/log.conf.tmp" "$STDB_PATH/log.conf"

export SPACETIMEDB_LOG_CONFIG="$STDB_PATH/log.conf"
export SPACETIMEDB_LOGS_PATH="$STDB_PATH/logs"
export SPACETIMEDB_TRACY=1

echo "DATABASE AT ${STDB_PATH}"
echo "LOGS AT $STDB_PATH/logs"

cargo run -p spacetimedb-standalone -- start -l 127.0.0.1:3000 --enable-tracy
cargo run -p spacetimedb-standalone -- start \
--data-dir ${STDB_PATH} \
--jwt-pub-key-path "${STDB_PATH}/id_ecdsa.pub" \
--jwt-priv-key-path "${STDB_PATH}/id_ecdsa" \
-l 127.0.0.1:3000 --enable-tracy

2 comments on commit 2976593

@github-actions
Copy link

@github-actions github-actions bot commented on 2976593 Nov 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Criterion benchmark results

Criterion benchmark report

YOU SHOULD PROBABLY IGNORE THESE RESULTS.

Criterion is a wall time based benchmarking system that is extremely noisy when run on CI. We collect these results for longitudinal analysis, but they are not reliable for comparing individual PRs.

Go look at the callgrind report instead.

empty

db on disk new latency old latency new throughput old throughput
sqlite 💿 407.4±1.29ns 410.9±2.66ns - -
sqlite 🧠 403.2±2.17ns 402.2±1.31ns - -
stdb_raw 💿 774.7±4.03ns 768.7±1.04ns - -
stdb_raw 🧠 774.8±0.81ns 768.6±1.13ns - -

insert_1

db on disk schema indices preload new latency old latency new throughput old throughput

insert_bulk

db on disk schema indices preload count new latency old latency new throughput old throughput
sqlite 💿 u32_u64_str btree_each_column 2048 256 581.0±0.66µs 589.4±1.15µs 1721 tx/sec 1696 tx/sec
sqlite 💿 u32_u64_str unique_0 2048 256 148.8±0.48µs 144.5±0.36µs 6.6 Ktx/sec 6.8 Ktx/sec
sqlite 💿 u32_u64_u64 btree_each_column 2048 256 460.7±0.44µs 468.8±0.80µs 2.1 Ktx/sec 2.1 Ktx/sec
sqlite 💿 u32_u64_u64 unique_0 2048 256 141.3±15.39µs 136.1±0.43µs 6.9 Ktx/sec 7.2 Ktx/sec
sqlite 🧠 u32_u64_str btree_each_column 2048 256 442.3±0.59µs 451.7±3.21µs 2.2 Ktx/sec 2.2 Ktx/sec
sqlite 🧠 u32_u64_str unique_0 2048 256 118.6±0.31µs 118.7±2.03µs 8.2 Ktx/sec 8.2 Ktx/sec
sqlite 🧠 u32_u64_u64 btree_each_column 2048 256 361.2±0.46µs 367.6±0.32µs 2.7 Ktx/sec 2.7 Ktx/sec
sqlite 🧠 u32_u64_u64 unique_0 2048 256 103.3±0.34µs 106.2±0.28µs 9.5 Ktx/sec 9.2 Ktx/sec
stdb_raw 💿 u32_u64_str btree_each_column 2048 256 499.2±60.86µs 598.2±27.53µs 2003 tx/sec 1671 tx/sec
stdb_raw 💿 u32_u64_str unique_0 2048 256 486.2±31.81µs 488.7±23.23µs 2.0 Ktx/sec 2046 tx/sec
stdb_raw 💿 u32_u64_u64 btree_each_column 2048 256 372.4±7.61µs 378.8±7.32µs 2.6 Ktx/sec 2.6 Ktx/sec
stdb_raw 💿 u32_u64_u64 unique_0 2048 256 349.1±5.65µs 349.6±11.26µs 2.8 Ktx/sec 2.8 Ktx/sec
stdb_raw 🧠 u32_u64_str btree_each_column 2048 256 298.8±0.29µs 299.5±0.31µs 3.3 Ktx/sec 3.3 Ktx/sec
stdb_raw 🧠 u32_u64_str unique_0 2048 256 231.4±0.73µs 231.5±0.60µs 4.2 Ktx/sec 4.2 Ktx/sec
stdb_raw 🧠 u32_u64_u64 btree_each_column 2048 256 235.9±0.20µs 240.2±0.39µs 4.1 Ktx/sec 4.1 Ktx/sec
stdb_raw 🧠 u32_u64_u64 unique_0 2048 256 209.0±0.73µs 209.2±0.39µs 4.7 Ktx/sec 4.7 Ktx/sec

iterate

db on disk schema indices new latency old latency new throughput old throughput
sqlite 💿 u32_u64_str unique_0 23.1±0.36µs 22.9±0.12µs 42.2 Ktx/sec 42.7 Ktx/sec
sqlite 💿 u32_u64_u64 unique_0 21.2±0.18µs 21.9±0.13µs 46.0 Ktx/sec 44.7 Ktx/sec
sqlite 🧠 u32_u64_str unique_0 20.6±0.16µs 20.1±0.15µs 47.4 Ktx/sec 48.6 Ktx/sec
sqlite 🧠 u32_u64_u64 unique_0 18.7±0.15µs 19.1±0.10µs 52.1 Ktx/sec 51.0 Ktx/sec
stdb_raw 💿 u32_u64_str unique_0 4.9±0.00µs 4.9±0.00µs 199.7 Ktx/sec 200.1 Ktx/sec
stdb_raw 💿 u32_u64_u64 unique_0 4.8±0.00µs 4.8±0.00µs 204.5 Ktx/sec 204.4 Ktx/sec
stdb_raw 🧠 u32_u64_str unique_0 4.9±0.00µs 4.9±0.00µs 200.3 Ktx/sec 199.7 Ktx/sec
stdb_raw 🧠 u32_u64_u64 unique_0 4.8±0.00µs 4.8±0.00µs 204.4 Ktx/sec 204.4 Ktx/sec

find_unique

db on disk key type preload new latency old latency new throughput old throughput

filter

db on disk key type index strategy load count new latency old latency new throughput old throughput
sqlite 💿 string index 2048 256 70.4±0.29µs 72.5±0.15µs 13.9 Ktx/sec 13.5 Ktx/sec
sqlite 💿 u64 index 2048 256 67.4±0.29µs 67.9±0.25µs 14.5 Ktx/sec 14.4 Ktx/sec
sqlite 🧠 string index 2048 256 66.4±0.22µs 68.3±0.18µs 14.7 Ktx/sec 14.3 Ktx/sec
sqlite 🧠 u64 index 2048 256 60.9±0.23µs 61.4±0.25µs 16.0 Ktx/sec 15.9 Ktx/sec
stdb_raw 💿 string index 2048 256 4.9±0.00µs 4.9±0.00µs 198.1 Ktx/sec 198.2 Ktx/sec
stdb_raw 💿 u64 index 2048 256 4.9±0.00µs 4.9±0.00µs 201.3 Ktx/sec 199.9 Ktx/sec
stdb_raw 🧠 string index 2048 256 4.9±0.00µs 4.9±0.00µs 198.2 Ktx/sec 198.4 Ktx/sec
stdb_raw 🧠 u64 index 2048 256 4.8±0.00µs 4.9±0.00µs 201.5 Ktx/sec 200.1 Ktx/sec

serialize

schema format count new latency old latency new throughput old throughput
u32_u64_str bflatn_to_bsatn_fast_path 100 3.6±0.01µs 3.6±0.00µs 26.5 Mtx/sec 26.6 Mtx/sec
u32_u64_str bflatn_to_bsatn_slow_path 100 3.1±0.01µs 3.2±0.00µs 30.3 Mtx/sec 30.2 Mtx/sec
u32_u64_str bsatn 100 15.3±0.01ns 15.3±0.05ns 6.1 Gtx/sec 6.1 Gtx/sec
u32_u64_str bsatn 100 2.2±0.01µs 2.2±0.03µs 44.2 Mtx/sec 43.9 Mtx/sec
u32_u64_str json 100 5.1±0.03µs 5.0±0.03µs 18.8 Mtx/sec 19.1 Mtx/sec
u32_u64_str json 100 8.8±0.15µs 8.9±0.10µs 10.8 Mtx/sec 10.8 Mtx/sec
u32_u64_str product_value 100 1021.7±0.65ns 1021.6±0.61ns 93.3 Mtx/sec 93.3 Mtx/sec
u32_u64_u64 bflatn_to_bsatn_fast_path 100 947.1±2.88ns 946.1±8.58ns 100.7 Mtx/sec 100.8 Mtx/sec
u32_u64_u64 bflatn_to_bsatn_slow_path 100 2.4±0.00µs 2.4±0.00µs 39.5 Mtx/sec 39.6 Mtx/sec
u32_u64_u64 bsatn 100 1596.1±23.30ns 1658.1±62.51ns 59.7 Mtx/sec 57.5 Mtx/sec
u32_u64_u64 bsatn 100 6.8±0.03ns 15.1±0.11ns 13.7 Gtx/sec 6.2 Gtx/sec
u32_u64_u64 json 100 3.2±0.07µs 3.2±0.08µs 30.1 Mtx/sec 29.9 Mtx/sec
u32_u64_u64 json 100 6.0±0.04µs 6.3±0.11µs 15.8 Mtx/sec 15.2 Mtx/sec
u32_u64_u64 product_value 100 1016.2±0.82ns 1015.5±0.70ns 93.8 Mtx/sec 93.9 Mtx/sec
u64_u64_u32 bflatn_to_bsatn_fast_path 100 721.4±1.53ns 717.8±4.24ns 132.2 Mtx/sec 132.9 Mtx/sec
u64_u64_u32 bflatn_to_bsatn_slow_path 100 2.4±0.01µs 2.4±0.00µs 39.5 Mtx/sec 39.6 Mtx/sec
u64_u64_u32 bsatn 100 1589.9±22.68ns 1593.4±21.76ns 60.0 Mtx/sec 59.9 Mtx/sec
u64_u64_u32 bsatn 100 642.4±1.54ns 643.6±2.58ns 148.4 Mtx/sec 148.2 Mtx/sec
u64_u64_u32 json 100 3.3±0.10µs 3.4±0.04µs 28.8 Mtx/sec 28.3 Mtx/sec
u64_u64_u32 json 100 6.1±0.03µs 6.2±0.14µs 15.7 Mtx/sec 15.4 Mtx/sec
u64_u64_u32 product_value 100 1015.6±0.99ns 1015.5±0.40ns 93.9 Mtx/sec 93.9 Mtx/sec

stdb_module_large_arguments

arg size new latency old latency new throughput old throughput
64KiB 108.2±9.46µs 103.9±7.26µs - -

stdb_module_print_bulk

line count new latency old latency new throughput old throughput
1 56.1±4.74µs 55.7±7.50µs - -
100 602.9±11.35µs 603.2±6.21µs - -
1000 4.9±0.87ms 5.4±0.12ms - -

remaining

name new latency old latency new throughput old throughput
special/db_game/circles/load=10 50.7±4.71ms 44.2±6.05ms - -
special/db_game/circles/load=100 51.8±4.76ms 54.3±3.87ms - -
special/db_game/ia_loop/load=500 142.2±2.46ms 147.2±3.03ms - -
special/db_game/ia_loop/load=5000 5.4±0.03s 5.4±0.02s - -
sqlite/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 54.1±0.19µs 58.9±6.87µs 18.0 Ktx/sec 16.6 Ktx/sec
sqlite/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 45.4±0.08µs 49.6±0.19µs 21.5 Ktx/sec 19.7 Ktx/sec
sqlite/🧠/update_bulk/u32_u64_str/unique_0/load=2048/count=256 39.4±0.16µs 42.1±0.48µs 24.8 Ktx/sec 23.2 Ktx/sec
sqlite/🧠/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 34.5±0.13µs 38.6±0.17µs 28.3 Ktx/sec 25.3 Ktx/sec
stdb_module/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 1280.4±11.41µs 1267.5±45.16µs 780 tx/sec 788 tx/sec
stdb_module/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 1015.5±10.26µs 995.2±4.01µs 984 tx/sec 1004 tx/sec
stdb_raw/💿/update_bulk/u32_u64_str/unique_0/load=2048/count=256 641.7±13.24µs 622.7±21.94µs 1558 tx/sec 1605 tx/sec
stdb_raw/💿/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 456.3±31.70µs 465.7±30.44µs 2.1 Ktx/sec 2.1 Ktx/sec
stdb_raw/🧠/update_bulk/u32_u64_str/unique_0/load=2048/count=256 374.8±0.57µs 371.9±0.31µs 2.6 Ktx/sec 2.6 Ktx/sec
stdb_raw/🧠/update_bulk/u32_u64_u64/unique_0/load=2048/count=256 341.3±0.94µs 335.0±0.53µs 2.9 Ktx/sec 2.9 Ktx/sec

@github-actions
Copy link

@github-actions github-actions bot commented on 2976593 Nov 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Callgrind benchmark results

Callgrind Benchmark Report

These benchmarks were run using callgrind,
an instruction-level profiler. They allow comparisons between sqlite (sqlite), SpacetimeDB running through a module (stdb_module), and the underlying SpacetimeDB data storage engine (stdb_raw). Callgrind emulates a CPU to collect the below estimates.

Measurement changes larger than five percent are in bold.

In-memory benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6397 6397 0.00% 6493 6493 0.00%
sqlite 5579 5579 0.00% 5993 5993 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76591 76591 0.00% 77063 77063 0.00%
stdb_raw u32_u64_str no_index 64 128 2 string 119089 119089 0.00% 119711 119799 -0.07%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25082 25081 0.00% 25632 25631 0.00%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24049 24049 0.00% 24489 24489 0.00%
sqlite u32_u64_str no_index 64 128 2 string 144695 144695 0.00% 146111 146111 0.00%
sqlite u32_u64_str no_index 64 128 1 u64 124044 124044 0.00% 125258 125254 0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 131361 131361 0.00% 132791 132787 0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 134494 134494 0.00% 136062 136058 0.00%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 873590 874291 -0.08% 889708 921185 -3.42%
stdb_raw u32_u64_str btree_each_column 64 128 1022673 1020301 0.23% 1048073 1076833 -2.67%
sqlite u32_u64_str unique_0 64 128 398320 398320 0.00% 415516 415524 -0.00%
sqlite u32_u64_str btree_each_column 64 128 983643 983637 0.00% 1022689 1022671 0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153724 153724 0.00% 153840 153840 0.00%
stdb_raw u32_u64_str unique_0 64 16749 16749 0.00% 16845 16845 0.00%
sqlite u32_u64_str unique_0 1024 1067255 1067273 -0.00% 1070547 1070569 -0.00%
sqlite u32_u64_str unique_0 64 76201 76201 0.00% 77195 77195 0.00%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50286 50286 0.00%
64 bsatn 25509 25509 0.00% 27787 27787 0.00%
16 bsatn 8200 8200 0.00% 9594 9594 0.00%
16 json 12188 12188 0.00% 14194 14194 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 20011708 20011681 0.00% 20486134 20483199 0.01%
stdb_raw u32_u64_str unique_0 64 128 1280941 1281611 -0.05% 1316171 1347429 -2.32%
sqlite u32_u64_str unique_0 1024 1024 1802150 1802150 0.00% 1811422 1811422 0.00%
sqlite u32_u64_str unique_0 64 128 128496 128496 0.00% 131432 131432 0.00%
On-disk benchmarks

callgrind: empty transaction

db total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw 6402 6402 0.00% 6518 6518 0.00%
sqlite 5621 5621 0.00% 6103 6103 0.00%

callgrind: filter

db schema indices count preload _column data_type total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str no_index 64 128 1 u64 76596 76596 0.00% 77044 77068 -0.03%
stdb_raw u32_u64_str no_index 64 128 2 string 119094 119094 0.00% 119732 119752 -0.02%
stdb_raw u32_u64_str btree_each_column 64 128 2 string 25088 25087 0.00% 25622 25621 0.00%
stdb_raw u32_u64_str btree_each_column 64 128 1 u64 24054 24054 0.00% 24462 24462 0.00%
sqlite u32_u64_str no_index 64 128 1 u64 125965 125965 0.00% 127523 127523 0.00%
sqlite u32_u64_str no_index 64 128 2 string 146616 146616 0.00% 148380 148376 0.00%
sqlite u32_u64_str btree_each_column 64 128 2 string 136616 136616 0.00% 138690 138690 0.00%
sqlite u32_u64_str btree_each_column 64 128 1 u64 133457 133457 0.00% 135293 135293 0.00%

callgrind: insert bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 64 128 822629 823495 -0.11% 837943 869571 -3.64%
stdb_raw u32_u64_str btree_each_column 64 128 971436 970897 0.06% 1028118 1027081 0.10%
sqlite u32_u64_str unique_0 64 128 415857 415857 0.00% 432363 432359 0.00%
sqlite u32_u64_str btree_each_column 64 128 1021898 1021898 0.00% 1059252 1059248 0.00%

callgrind: iterate

db schema indices count total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 153729 153729 0.00% 153825 153821 0.00%
stdb_raw u32_u64_str unique_0 64 16754 16754 0.00% 16846 16846 0.00%
sqlite u32_u64_str unique_0 1024 1070323 1070323 0.00% 1074061 1074061 0.00%
sqlite u32_u64_str unique_0 64 77973 77973 0.00% 79167 79167 0.00%

callgrind: serialize_product_value

count format total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
64 json 47528 47528 0.00% 50286 50286 0.00%
64 bsatn 25509 25509 0.00% 27787 27787 0.00%
16 bsatn 8200 8200 0.00% 9594 9594 0.00%
16 json 12188 12188 0.00% 14194 14194 0.00%

callgrind: update bulk

db schema indices count preload total reads + writes old total reads + writes Δrw estimated cycles old estimated cycles Δcycles
stdb_raw u32_u64_str unique_0 1024 1024 18939889 18942249 -0.01% 19506595 19502989 0.02%
stdb_raw u32_u64_str unique_0 64 128 1233586 1234531 -0.08% 1300386 1299567 0.06%
sqlite u32_u64_str unique_0 1024 1024 1809711 1809711 0.00% 1818439 1818439 0.00%
sqlite u32_u64_str unique_0 64 128 132622 132622 0.00% 135634 135634 0.00%

Please sign in to comment.