diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index cc77e714d..501e04d4a 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -146,13 +146,14 @@ jobs: - name: Fetch main branch data for comparison run: | - # Save PR's site/data aside, get main's for comparison - cp -r site/data /tmp/pr_site_data + # main's site/data, read into /tmp and never checked out. The checkout this + # replaces staged main's copy of every file under site/data, and copying the + # PR's files back over does not delete, so a file the PR removed stayed in the + # index and rode along in the results commit. git fetch origin main --depth=1 - git checkout origin/main -- site/data/ 2>/dev/null || true - # Keep main's site/data for comparison, restore PR code - cp -r site/data /tmp/main_site_data - cp -r /tmp/pr_site_data/* site/data/ + rm -rf /tmp/main_site_data + mkdir -p /tmp/main_site_data + git archive origin/main:site/data | tar -x -C /tmp/main_site_data # Frameworks run sequentially — each needs the whole machine. A failure # doesn't stop the loop: the remaining frameworks still run, whatever was diff --git a/site/data/results/fulmine.js-tuned.json b/site/data/results/fulmine.js-tuned.json deleted file mode 100644 index 78661dda1..000000000 --- a/site/data/results/fulmine.js-tuned.json +++ /dev/null @@ -1,570 +0,0 @@ -{ - "framework": "fulmine.js-tuned", - "results": { - "8gbit-512": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 49364, - "avg_latency": "113.8us", - "p99_latency": "162.0us", - "cpu": "318.9%", - "memory": "2.7GiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "511.36MB/s", - "input_bw": "482.07MB/s", - "reconnects": 0, - "status_2xx": 246901, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "cpu_usec": 17059474, - "cpu_per_req_us": 69.0944, - "target_rate": 50000, - "rate_ratio": 0.9873, - "p99_9_latency": "5786.0us" - }, - "async-32000": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 2110747, - "avg_latency": "15.02ms", - "p99_latency": "17.20ms", - "cpu": "6367.9%", - "memory": "5.7GiB", - "connections": 32000, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "237.49MB/s", - "input_bw": "96.62MB/s", - "reconnects": 0, - "status_2xx": 21107472, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "async-db-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 239935, - "avg_latency": "3.76ms", - "p99_latency": "16.20ms", - "cpu": "5813.2%", - "memory": "6.2GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "919.00MB/s", - "input_bw": "16.02MB/s", - "reconnects": 95732, - "status_2xx": 2399356, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "baseline-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 1595557, - "avg_latency": "2.57ms", - "p99_latency": "3.56ms", - "cpu": "6551.1%", - "memory": "2.6GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "179.51MB/s", - "input_bw": "123.25MB/s", - "reconnects": 0, - "status_2xx": 7977787, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-16384": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 3538165, - "avg_latency": "4.59ms", - "p99_latency": "8.92ms", - "cpu": "6173.2%", - "memory": "2.5GiB", - "connections": 16384, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "24.12MB/s", - "reconnects": 0, - "status_2xx": 17690826, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 3772036, - "avg_latency": "1.08ms", - "p99_latency": "2.30ms", - "cpu": "6279.7%", - "memory": "2.4GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "25.24MB/s", - "reconnects": 0, - "status_2xx": 18860183, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-512": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 3701735, - "avg_latency": "137us", - "p99_latency": "360us", - "cpu": "6393.0%", - "memory": "2.3GiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "24.71MB/s", - "reconnects": 0, - "status_2xx": 18508679, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-limited-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 2553620, - "avg_latency": "1.06ms", - "p99_latency": "2.81ms", - "cpu": "6262.5%", - "memory": "2.4GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "57.50MB/s", - "reconnects": 1276988, - "status_2xx": 12768102, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-limited-512": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 2084011, - "avg_latency": "152us", - "p99_latency": "546us", - "cpu": "5672.1%", - "memory": "2.4GiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "46.89MB/s", - "reconnects": 1042019, - "status_2xx": 10420057, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-pipeline-16384": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 22945078, - "avg_latency": "11.22ms", - "p99_latency": "14.80ms", - "cpu": "6261.9%", - "memory": "2.6GiB", - "connections": 16384, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "153.65MB/s", - "reconnects": 0, - "status_2xx": 114725392, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-pipeline-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 24204617, - "avg_latency": "2.71ms", - "p99_latency": "5.17ms", - "cpu": "6533.3%", - "memory": "2.4GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "161.60MB/s", - "reconnects": 0, - "status_2xx": 121023088, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "echo-ws-pipeline-512": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 23989939, - "avg_latency": "340us", - "p99_latency": "941us", - "cpu": "6721.9%", - "memory": "2.3GiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "160.11MB/s", - "reconnects": 0, - "status_2xx": 119949696, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "fortunes-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 70705, - "avg_latency": "13.21ms", - "p99_latency": "26.00ms", - "cpu": "6506.8%", - "memory": "6.1GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "1.64GB/s", - "reconnects": 0, - "status_2xx": 353526, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "gateway-64-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 311169, - "avg_latency": "102.14ms", - "p99_latency": "861.84ms", - "cpu": "6032.4%", - "memory": "6.5GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "3.85GB/s", - "reconnects": 0, - "status_2xx": 1577630, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 473289, - "tpl_baseline": 315526, - "tpl_json": 552170, - "tpl_async_db": 236644, - "cpu_breakdown": "server: 2283% 4.8GiB | proxy: 3749% 1.7GiB" - }, - "gateway-64-512": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 314541, - "avg_latency": "54.90ms", - "p99_latency": "403.48ms", - "cpu": "5473.9%", - "memory": "5.1GiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "3.83GB/s", - "reconnects": 0, - "status_2xx": 1585291, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 475587, - "tpl_baseline": 317058, - "tpl_json": 554851, - "tpl_async_db": 237793, - "cpu_breakdown": "server: 2297% 3.7GiB | proxy: 3177% 1.4GiB" - }, - "gateway-h3-256": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 62553, - "avg_latency": "127.14ms", - "p99_latency": "756.13ms", - "cpu": "2434.9%", - "memory": "4.7GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "663.81MB/s", - "reconnects": 0, - "status_2xx": 314020, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 94206, - "tpl_baseline": 62804, - "tpl_json": 109907, - "tpl_async_db": 47103, - "cpu_breakdown": "server: 556% 2.8GiB | proxy: 1879% 1.8GiB" - }, - "gateway-h3-64": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 90285, - "avg_latency": "22.54ms", - "p99_latency": "85.02ms", - "cpu": "3252.8%", - "memory": "3.4GiB", - "connections": 64, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "981.28MB/s", - "reconnects": 0, - "status_2xx": 452331, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 135699, - "tpl_baseline": 90466, - "tpl_json": 158315, - "tpl_async_db": 67849, - "cpu_breakdown": "server: 749% 2.9GiB | proxy: 2504% 465MiB" - }, - "json-comp-16384": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 491530, - "avg_latency": "32.95ms", - "p99_latency": "77.90ms", - "cpu": "6245.6%", - "memory": "8.5GiB", - "connections": 16384, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "717.79MB/s", - "input_bw": "36.56MB/s", - "reconnects": 90895, - "status_2xx": 2457653, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "json-comp-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 474140, - "avg_latency": "8.62ms", - "p99_latency": "27.10ms", - "cpu": "6357.3%", - "memory": "7.6GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "692.39MB/s", - "input_bw": "35.27MB/s", - "reconnects": 93253, - "status_2xx": 2370704, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "json-tls-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 1030277, - "avg_latency": "3.97ms", - "p99_latency": "84.51ms", - "cpu": "6575.2%", - "memory": "8.2GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "3.52GB", - "reconnects": 0, - "status_2xx": 5255296, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "latency-10k-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 9982, - "avg_latency": "84.3us", - "p99_latency": "122.0us", - "cpu": "37.7%", - "memory": "2.1GiB", - "connections": 1024, - "threads": 64, - "duration": "20s", - "pipeline": 1, - "bandwidth": "1.17MB/s", - "reconnects": 0, - "status_2xx": 199656, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "cpu_usec": 7938241, - "cpu_per_req_us": 39.7596, - "target_rate": 10000, - "rate_ratio": 0.9982, - "p99_9_latency": "210.0us" - }, - "latency-1m-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 998214, - "avg_latency": "129.3us", - "p99_latency": "263.0us", - "cpu": "3670.3%", - "memory": "2.5GiB", - "connections": 1024, - "threads": 64, - "duration": "20s", - "pipeline": 1, - "bandwidth": "116.79MB/s", - "reconnects": 0, - "status_2xx": 19967495, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "cpu_usec": 666133858, - "cpu_per_req_us": 33.3609, - "target_rate": 1000000, - "rate_ratio": 0.9982, - "p99_9_latency": "792.0us" - }, - "limited-conn-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 1450810, - "avg_latency": "2.81ms", - "p99_latency": "11.30ms", - "cpu": "6354.8%", - "memory": "2.6GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "163.23MB/s", - "input_bw": "112.07MB/s", - "reconnects": 725027, - "status_2xx": 7254050, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "pipelined-4096": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 32750279, - "avg_latency": "2.00ms", - "p99_latency": "4.07ms", - "cpu": "6349.1%", - "memory": "2.1GiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "3.60GB/s", - "reconnects": 0, - "status_2xx": 163751397, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, - "production-stack-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 105395, - "avg_latency": "127.01ms", - "p99_latency": "3.84s", - "cpu": "2639.5%", - "memory": "4.4GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "1.19GB/s", - "reconnects": 0, - "status_2xx": 535408, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 160622, - "tpl_baseline": 53540, - "tpl_items": 267704, - "tpl_me": 53540, - "cpu_breakdown": "server: 463% 2.5GiB | authsvc: 710% 467MiB | cache: 214% 10MiB | edge: 1253% 1.4GiB" - }, - "production-stack-256": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 120770, - "avg_latency": "34.84ms", - "p99_latency": "139.34ms", - "cpu": "2878.3%", - "memory": "3.7GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "1.31GB/s", - "reconnects": 0, - "status_2xx": 607478, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0, - "tpl_static": 182243, - "tpl_baseline": 60747, - "tpl_items": 303739, - "tpl_me": 60747, - "cpu_breakdown": "server: 636% 2.5GiB | authsvc: 765% 140MiB | cache: 214% 20MiB | edge: 1263% 1.1GiB" - }, - "static-tls-1024": { - "framework": "fulmine.js-tuned", - "language": "JS", - "rps": 595686, - "avg_latency": "1.72ms", - "p99_latency": "14.93ms", - "cpu": "6585.4%", - "memory": "3.3GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "9.18GB", - "reconnects": 0, - "status_2xx": 3038156, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - } - } -}