diff --git a/README.md b/README.md index ebba1c9207..9d9f256c64 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ To introduce a new result for an existing system with a different usage scenario `index.html` can be re-generated using `./generate-results.sh`. The CI (GitHub Actions) does this automatically, this step is optional. -The shared driver (`lib/benchmark-common.sh`) runs a supplementary concurrent-QPS test (`BENCH_CONCURRENT_CONNECTIONS` workers for `BENCH_CONCURRENT_DURATION` seconds) after the main sweep. Single-process engines must set `BENCH_CONCURRENT_DURATION=0` in their `benchmark.sh` to skip it: each query forks a fresh full-machine process with no shared scheduler, so concurrent connections only oversubscribe RAM (and can OOM the run) instead of measuring throughput. Rule of thumb — skip when `./start` launches no shared server (the embedded CLIs and Spark variants); keep it for daemons and the in-process server wrappers (pandas/polars/`*-dataframe`), which share one process. See issue #946. +The shared driver (`lib/benchmark-common.sh`) runs a supplementary concurrent-QPS test (`BENCH_CONCURRENT_CONNECTIONS` workers for `BENCH_CONCURRENT_DURATION` seconds) after the main sweep. It runs for every system, including single-process engines, where each connection runs its own query process. If the machine starts swap thrashing (swap-in above `BENCH_CONCURRENT_SWAP_LIMIT` MiB/s, default 10, for `BENCH_CONCURRENT_SWAP_WINDOW` seconds, default 30), the test is terminated early and the result is recorded as `null`, because the number would measure paging rather than query throughput. See issue #946. All tests were originally run on AWS c6a.4xlarge EC2 VMs with 500 GB gp2 disks. With better automation, more EC2 machines were added later: c6a.2xlarge, c6a.metal, c8g.4xlarge, c6a.xlarge, c7a.metal-48xl, c6a.large, c8g.metal-48xl, and t3a.small. diff --git a/bqn/benchmark.sh b/bqn/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/bqn/benchmark.sh +++ b/bqn/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb-parquet-partitioned/benchmark.sh b/chdb-parquet-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/chdb-parquet-partitioned/benchmark.sh +++ b/chdb-parquet-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/chdb/benchmark.sh b/chdb/benchmark.sh index 75c3806085..f65cb3aa51 100755 --- a/chdb/benchmark.sh +++ b/chdb/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake-partitioned/benchmark.sh b/clickhouse-datalake-partitioned/benchmark.sh index d82911fa20..20e6e4820a 100755 --- a/clickhouse-datalake-partitioned/benchmark.sh +++ b/clickhouse-datalake-partitioned/benchmark.sh @@ -2,9 +2,4 @@ # Data is read directly from S3, no local download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-datalake/benchmark.sh b/clickhouse-datalake/benchmark.sh index d82911fa20..20e6e4820a 100755 --- a/clickhouse-datalake/benchmark.sh +++ b/clickhouse-datalake/benchmark.sh @@ -2,9 +2,4 @@ # Data is read directly from S3, no local download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet-partitioned/benchmark.sh b/clickhouse-parquet-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/clickhouse-parquet-partitioned/benchmark.sh +++ b/clickhouse-parquet-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/clickhouse-parquet/benchmark.sh b/clickhouse-parquet/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/clickhouse-parquet/benchmark.sh +++ b/clickhouse-parquet/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/ctrlb/benchmark.sh b/ctrlb/benchmark.sh index 5ae07af4f2..c5052317ff 100755 --- a/ctrlb/benchmark.sh +++ b/ctrlb/benchmark.sh @@ -10,7 +10,6 @@ export BENCH_RESTARTABLE=yes export BENCH_DURABLE=yes export BENCH_TRIES="${BENCH_TRIES:-3}" -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" # Run the shared ClickBench driver, capturing stdout so we can BOTH show it live # and turn it into the committed results JSON. `tee` preserves the standard text diff --git a/datafusion-partitioned/benchmark.sh b/datafusion-partitioned/benchmark.sh index 228e2176e4..c417bf4118 100755 --- a/datafusion-partitioned/benchmark.sh +++ b/datafusion-partitioned/benchmark.sh @@ -1,7 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# skip concurrent_qps tests by default as datafusion-cli is configured -# for single user/single process usage -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion-vortex-partitioned/benchmark.sh b/datafusion-vortex-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/datafusion-vortex-partitioned/benchmark.sh +++ b/datafusion-vortex-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion-vortex/benchmark.sh b/datafusion-vortex/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/datafusion-vortex/benchmark.sh +++ b/datafusion-vortex/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/datafusion/benchmark.sh b/datafusion/benchmark.sh index 4128a24124..3db3216601 100755 --- a/datafusion/benchmark.sh +++ b/datafusion/benchmark.sh @@ -1,7 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# skip concurrent_qps tests by default as datafusion-cli is configured -# for single user/single process usage -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/drill/benchmark.sh b/drill/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/drill/benchmark.sh +++ b/drill/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake-partitioned/benchmark.sh b/duckdb-datalake-partitioned/benchmark.sh index d82911fa20..20e6e4820a 100755 --- a/duckdb-datalake-partitioned/benchmark.sh +++ b/duckdb-datalake-partitioned/benchmark.sh @@ -2,9 +2,4 @@ # Data is read directly from S3, no local download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-datalake/benchmark.sh b/duckdb-datalake/benchmark.sh index d82911fa20..20e6e4820a 100755 --- a/duckdb-datalake/benchmark.sh +++ b/duckdb-datalake/benchmark.sh @@ -2,9 +2,4 @@ # Data is read directly from S3, no local download. export BENCH_DOWNLOAD_SCRIPT="" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet-partitioned/benchmark.sh b/duckdb-parquet-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/duckdb-parquet-partitioned/benchmark.sh +++ b/duckdb-parquet-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-parquet/benchmark.sh b/duckdb-parquet/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/duckdb-parquet/benchmark.sh +++ b/duckdb-parquet/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex-partitioned/benchmark.sh b/duckdb-vortex-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/duckdb-vortex-partitioned/benchmark.sh +++ b/duckdb-vortex-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex-tuned/benchmark.sh b/duckdb-vortex-tuned/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/duckdb-vortex-tuned/benchmark.sh +++ b/duckdb-vortex-tuned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb-vortex/benchmark.sh b/duckdb-vortex/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/duckdb-vortex/benchmark.sh +++ b/duckdb-vortex/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/duckdb/benchmark.sh b/duckdb/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/duckdb/benchmark.sh +++ b/duckdb/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/gendb/benchmark.sh b/gendb/benchmark.sh index 7128ac17b6..ed0823ecd8 100755 --- a/gendb/benchmark.sh +++ b/gendb/benchmark.sh @@ -10,9 +10,4 @@ # to start/stop); the data on disk in db/ is persistent across cold cycles. export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb-partitioned/benchmark.sh b/glaredb-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/glaredb-partitioned/benchmark.sh +++ b/glaredb-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/glaredb/benchmark.sh b/glaredb/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/glaredb/benchmark.sh +++ b/glaredb/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/infino/benchmark.sh b/infino/benchmark.sh index d9be72605c..3d2b8d450b 100755 --- a/infino/benchmark.sh +++ b/infino/benchmark.sh @@ -11,9 +11,6 @@ export BENCH_RESTARTABLE=yes # Data is durable on local disk; a restart reopens ./data, no reload. export BENCH_DURABLE=yes -# The server answers one query at a time, so the concurrent-QPS test would only -# oversubscribe it. Skip by default. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" # infino tuning (cache dir + budget). Shared with the raw load/start scripts so # the playground gets the same config. diff --git a/jd/benchmark.sh b/jd/benchmark.sh index bcccf0bab0..2e5741b394 100755 --- a/jd/benchmark.sh +++ b/jd/benchmark.sh @@ -3,9 +3,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh ijconsole with no shared -# scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/kdb/benchmark.sh b/kdb/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/kdb/benchmark.sh +++ b/kdb/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/lib/benchmark-common.sh b/lib/benchmark-common.sh index 0b376cb20a..041965a98c 100755 --- a/lib/benchmark-common.sh +++ b/lib/benchmark-common.sh @@ -56,15 +56,15 @@ # BENCH_CONCURRENT_DURATION # Wall-clock window for the QPS test, in seconds. # Default 600. Set to 0 to skip the test (emits -# null). Single-process engines should set 0: each -# query forks a fresh full-machine process with no -# shared scheduler, so N concurrent connections only -# oversubscribe RAM instead of measuring throughput. -# The rule: skip when ./start launches no shared -# server (the embedded CLIs and spark variants), keep -# for daemons and the in-process server wrappers -# (pandas/polars/*-dataframe), which do share one -# process. See issue #946. +# null). +# BENCH_CONCURRENT_SWAP_LIMIT +# Swap-in rate, in MiB/s, that counts as thrashing +# during the QPS test. Default 10. Set to 0 to +# disable the guard. See issue #946. +# BENCH_CONCURRENT_SWAP_WINDOW +# Seconds the swap-in rate must stay above the limit +# before the QPS test is terminated and recorded as +# null. Default 30. # BENCH_CONCURRENT_SEED Seed shared across systems so that connection # N hits the same query order on every engine # (the per-connection permutation is derived @@ -91,6 +91,8 @@ export HOME="${HOME:-/root}" : "${BENCH_CONCURRENT_CONNECTIONS:=10}" : "${BENCH_CONCURRENT_DURATION:=600}" : "${BENCH_CONCURRENT_SEED:=clickbench}" +: "${BENCH_CONCURRENT_SWAP_LIMIT:=10}" +: "${BENCH_CONCURRENT_SWAP_WINDOW:=30}" # Resolve the directory containing this script so we can find sibling # helpers (download-hits-*). @@ -389,6 +391,14 @@ bench_run_query() { # honest "successful queries / wall window"; only completely-zero # successes (e.g. the engine never came back) end up as "null". # +# Swap guard: N independent processes of a single-process engine (each +# sized for the whole machine) can oversubscribe RAM, and the window then +# measures how fast the box pages rather than query throughput (issue +# #946). The main shell samples the swap-in counter every 5 s; if the rate +# stays above BENCH_CONCURRENT_SWAP_LIMIT MiB/s for +# BENCH_CONCURRENT_SWAP_WINDOW seconds, the workers and the watchdog are +# killed and the test is recorded as null. +# # Emits two lines that sink.parser pulls into sink.results: # Concurrent QPS: successful queries / window length # Concurrent error ratio: <0.NNN> errored queries / total attempted @@ -397,8 +407,7 @@ bench_concurrent_qps() { local duration="${BENCH_CONCURRENT_DURATION}" local seed="${BENCH_CONCURRENT_SEED}" - # Duration 0 means the test is disabled (single-process engines set this; - # see the BENCH_CONCURRENT_DURATION docs). Emit null and skip the restart, + # Duration 0 means the test is disabled. Emit null and skip the restart, # cache flush, and worker setup entirely. if [ "$duration" -le 0 ]; then echo "Concurrent QPS: null" @@ -439,6 +448,14 @@ bench_concurrent_qps() { stats_dir=$(mktemp -d) local deadline=$(($(date +%s) + duration)) + # Job control puts the watchdog and every worker in its own process + # group, so the swap guard can kill each one together with the ./query + # (and client/engine) processes it has spawned. Job control also stops + # bash from pointing the background jobs' stdin at /dev/null, so do it + # explicitly. + set -m + local pids=() + # Watchdog: poll ./check; if the system goes down mid-test, bring it # back up without telling the workers. Workers keep firing through # the outage — errors during the dead window count toward @@ -461,7 +478,8 @@ bench_concurrent_qps() { fi fi done - ) & + ) < /dev/null & + pids+=($!) local c for c in $(seq 1 "$connections"); do @@ -500,10 +518,66 @@ PY idx=$(( (idx + 1) % nq )) done printf '%s %s\n' "$ok" "$err" > "$stats_dir/$c" - ) & + ) < /dev/null & + pids+=($!) done + set +m + + local thrashing=0 + if [ "$BENCH_CONCURRENT_SWAP_LIMIT" != "0" ]; then + local page_size limit_pages_per_sec strikes=0 needed_strikes + local prev_pages prev_time cur_pages cur_time p alive + page_size=$(getconf PAGESIZE) + limit_pages_per_sec=$((BENCH_CONCURRENT_SWAP_LIMIT * 1024 * 1024 / page_size)) + needed_strikes=$(( (BENCH_CONCURRENT_SWAP_WINDOW + 4) / 5 )) + prev_pages=$(awk '$1 == "pswpin" { print $2 }' /proc/vmstat) + prev_time=$(date +%s) + while :; do + alive=0 + for p in "${pids[@]:1}"; do + if kill -0 "$p" 2>/dev/null; then + alive=1 + break + fi + done + [ "$alive" -eq 1 ] || break + sleep 5 + cur_pages=$(awk '$1 == "pswpin" { print $2 }' /proc/vmstat) + cur_time=$(date +%s) + if [ "$cur_time" -gt "$prev_time" ] \ + && [ $(( (cur_pages - prev_pages) / (cur_time - prev_time) )) -gt "$limit_pages_per_sec" ]; then + strikes=$((strikes + 1)) + else + strikes=0 + fi + prev_pages=$cur_pages + prev_time=$cur_time + if [ "$strikes" -ge "$needed_strikes" ]; then + thrashing=1 + break + fi + done + fi + + if [ "$thrashing" -eq 1 ]; then + echo "Concurrent QPS test terminated: swap-in above ${BENCH_CONCURRENT_SWAP_LIMIT} MiB/s for ${BENCH_CONCURRENT_SWAP_WINDOW} s" >&2 + for p in "${pids[@]}"; do + kill -TERM -- "-$p" 2>/dev/null || true + done + sleep 5 + for p in "${pids[@]}"; do + kill -KILL -- "-$p" 2>/dev/null || true + done + fi wait + if [ "$thrashing" -eq 1 ]; then + rm -rf "$stats_dir" + echo "Concurrent QPS: null" + echo "Concurrent error ratio: null" + return 0 + fi + local total_ok=0 total_err=0 ok err for f in "$stats_dir"/*; do read -r ok err < "$f" diff --git a/octosql/benchmark.sh b/octosql/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/octosql/benchmark.sh +++ b/octosql/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx-parquet/benchmark.sh b/opteryx-parquet/benchmark.sh index ad5d0266f0..d1284676e7 100755 --- a/opteryx-parquet/benchmark.sh +++ b/opteryx-parquet/benchmark.sh @@ -6,7 +6,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" # lives in process memory, so ./load is not re-run. export BENCH_RESTARTABLE=yes export BENCH_DURABLE=yes -# Concurrent-QPS test stays off for now: server.py serves one request at a -# time, so N connections would measure a queue, not throughput. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx-skene/benchmark.sh b/opteryx-skene/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/opteryx-skene/benchmark.sh +++ b/opteryx-skene/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/opteryx/benchmark.sh b/opteryx/benchmark.sh index ad5d0266f0..d1284676e7 100755 --- a/opteryx/benchmark.sh +++ b/opteryx/benchmark.sh @@ -6,7 +6,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" # lives in process memory, so ./load is not re-run. export BENCH_RESTARTABLE=yes export BENCH_DURABLE=yes -# Concurrent-QPS test stays off for now: server.py serves one request at a -# time, so N connections would measure a queue, not throughput. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/pivot-parquet-partitioned/benchmark.sh b/pivot-parquet-partitioned/benchmark.sh index 1251353052..d5cf12db78 100755 --- a/pivot-parquet-partitioned/benchmark.sh +++ b/pivot-parquet-partitioned/benchmark.sh @@ -1,4 +1,3 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/pivot-parquet/benchmark.sh b/pivot-parquet/benchmark.sh index 93e24f228d..a33527ee30 100755 --- a/pivot-parquet/benchmark.sh +++ b/pivot-parquet/benchmark.sh @@ -1,4 +1,3 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/regolith/benchmark.sh b/regolith/benchmark.sh index 27ed18dae5..2bc7e32ed4 100755 --- a/regolith/benchmark.sh +++ b/regolith/benchmark.sh @@ -2,7 +2,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # Embedded engine: no daemon to restart, one process per query. export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process, so -# the concurrent-QPS test would only oversubscribe RAM (see issue #946). -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail-partitioned/benchmark.sh b/sail-partitioned/benchmark.sh index b7ae16ae72..c417bf4118 100755 --- a/sail-partitioned/benchmark.sh +++ b/sail-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sail/benchmark.sh b/sail/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/sail/benchmark.sh +++ b/sail/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/slatedb/benchmark.sh b/slatedb/benchmark.sh index 27ed18dae5..2bc7e32ed4 100755 --- a/slatedb/benchmark.sh +++ b/slatedb/benchmark.sh @@ -2,7 +2,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # Embedded engine: no daemon to restart, one process per query. export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process, so -# the concurrent-QPS test would only oversubscribe RAM (see issue #946). -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-auron/benchmark.sh b/spark-auron/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/spark-auron/benchmark.sh +++ b/spark-auron/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-comet/benchmark.sh b/spark-comet/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/spark-comet/benchmark.sh +++ b/spark-comet/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-gluten/benchmark.sh b/spark-gluten/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/spark-gluten/benchmark.sh +++ b/spark-gluten/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark-velox/benchmark.sh b/spark-velox/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/spark-velox/benchmark.sh +++ b/spark-velox/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/spark/benchmark.sh b/spark/benchmark.sh index 598f8a07ad..d670e12023 100755 --- a/spark/benchmark.sh +++ b/spark/benchmark.sh @@ -3,9 +3,4 @@ export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # Spark runs in-process per query — restart between queries is meaningless # (and would re-download nothing). Skip restart. export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/sqlite/benchmark.sh b/sqlite/benchmark.sh index 75c3806085..f65cb3aa51 100755 --- a/sqlite/benchmark.sh +++ b/sqlite/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/turso/benchmark.sh b/turso/benchmark.sh index 75c3806085..f65cb3aa51 100755 --- a/turso/benchmark.sh +++ b/turso/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/velox/benchmark.sh b/velox/benchmark.sh index fb3b4d1318..3db3216601 100755 --- a/velox/benchmark.sh +++ b/velox/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Single-process engine: each query forks a fresh full-machine process with no -# shared scheduler across connections, so the concurrent-QPS test only -# oversubscribes RAM rather than measuring throughput. Skip it by default; -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zighouse/benchmark.sh b/zighouse/benchmark.sh index 93a74ea989..74f39a70f4 100755 --- a/zighouse/benchmark.sh +++ b/zighouse/benchmark.sh @@ -4,7 +4,6 @@ set -e export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no -export BENCH_CONCURRENT_DURATION=0 ../lib/benchmark-common.sh || true diff --git a/zpq-partitioned/benchmark.sh b/zpq-partitioned/benchmark.sh index 0c0203dd7b..c417bf4118 100755 --- a/zpq-partitioned/benchmark.sh +++ b/zpq-partitioned/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_RESTARTABLE=no -# Each query runs a fresh zpq process: there is no server and no shared -# scheduler across connections, so the concurrent-QPS test would only -# oversubscribe the machine. Skip it by default, like DuckDB (Parquet); -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh diff --git a/zpq/benchmark.sh b/zpq/benchmark.sh index f3353c690e..3db3216601 100755 --- a/zpq/benchmark.sh +++ b/zpq/benchmark.sh @@ -1,9 +1,4 @@ #!/bin/bash export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" export BENCH_RESTARTABLE=no -# Each query runs a fresh zpq process: there is no server and no shared -# scheduler across connections, so the concurrent-QPS test would only -# oversubscribe the machine. Skip it by default, like DuckDB (Parquet); -# override BENCH_CONCURRENT_DURATION to re-enable. See issue #946. -export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" exec ../lib/benchmark-common.sh