From 2fc36b47500acc2841694b6172ea27a2178c4895 Mon Sep 17 00:00:00 2001 From: Jose Quintana <1700322+joseluisq@users.noreply.github.com> Date: Thu, 20 Oct 2022 16:12:38 +0200 Subject: [PATCH] chore: benchmarks [skip ci] (#155) - lighttpd 1.4.67 (default config) - nginx 1.22.0 (default config + worker_processes=4) - sws 2.13.1 (default config) - apache 2.4.54 (default config) - caddy 2.6.1 (default config) - binserve 0.2.0 (default config + fast_mem_cache=false, enable_hot_reload=false, enable_logging=false, enable_directory_listing=false) --- .gitignore | 3 +++ Makefile | 7 ++++++- README.md | 6 ++++++ benchmark/BENCHMARKS.md | 24 ++++++++++++++++++++++++ benchmark/sws_benchmarks.csv | 7 +++++++ benchmark/sws_benchmarks.png | Bin 0 -> 26580 bytes benchmark/wrk_collector.lua | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 benchmark/BENCHMARKS.md create mode 100644 benchmark/sws_benchmarks.csv create mode 100644 benchmark/sws_benchmarks.png create mode 100644 benchmark/wrk_collector.lua diff --git a/.gitignore b/.gitignore index b8d036a..5a084e5 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ **/*.out* **/*.perf* **/*empty* +**/*.json +**/*.csv release .vscode TODO @@ -28,3 +30,4 @@ docs/*/**.html !sample.env !/docs !/tests/fixtures/**/* +!benchmark/sws_benchmarks*.csv diff --git a/Makefile b/Makefile index 6028b9f..64fc326 100644 --- a/Makefile +++ b/Makefile @@ -282,12 +282,17 @@ promote: .PHONY: promote loadtest: + @vegeta --version @echo "GET http://localhost:8787" | \ - vegeta -cpus=12 attack -workers=10 -duration=60s -connections=10000 -rate=200 -http2=false > results.bin + vegeta -cpus=12 attack -workers=12 -connections=500 -rate=6500/s -duration=10s -http2=false > results.bin @cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]' @cat results.bin | vegeta plot > plot.html .PHONY: loadtest +wrk: + @wrk -c 500 -t 12 -d 10s --latency -s benchmark/wrk_collector.lua $(WRK_URL) +.PHONY: wrk + man: @asciidoctor --doctype=manpage --backend=manpage docs/man/static-web-server.1.rst .PHONY: man diff --git a/README.md b/README.md index 85288ae..7970799 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,12 @@ For more details about the API, usage and examples please have a look at [The Do - [Release Binaries](https://github.com/joseluisq/static-web-server/releases) - [Platforms/Architectures Supported](https://sws.joseluisq.net/platforms-architectures/) +## Benchmarks + + + +See more details on [benchmark/BENCHMARKS.md](benchmark/BENCHMARKS.md) + ## Notes - If you're looking for `v1` please go to [1.x](https://github.com/joseluisq/static-web-server/tree/1.x) branch. diff --git a/benchmark/BENCHMARKS.md b/benchmark/BENCHMARKS.md new file mode 100644 index 0000000..17ac20c --- /dev/null +++ b/benchmark/BENCHMARKS.md @@ -0,0 +1,24 @@ +# SWS - Benchmarks 2022 + +> A benchmark suite which measures the requests per second and latency in average for several web servers. + + + +## How to use + +Change `WRK_URL` with the corresponding server URL to export the wrk metrics. + +```sh +WRK_URL="http://localhost" make wrk +``` + +## System + +- **OS:** Arch Linux +- **Kernel:** 5.19.13-arch1-1 (64 bits) +- **Processor:** 4 × Intel® Core™ i7-6500U +- **RAM:** 8 GiB + +## Data + +For data used see [data](./data/) directory for more details. diff --git a/benchmark/sws_benchmarks.csv b/benchmark/sws_benchmarks.csv new file mode 100644 index 0000000..bc155e9 --- /dev/null +++ b/benchmark/sws_benchmarks.csv @@ -0,0 +1,7 @@ +server,requests,duration_ms,requests_per_sec,bytes,bytes_transfer_per_sec,connect_errors,read_errors,write_errors,http_errors,timeouts,latency_min,latency_max,latency_mean_ms,latency_stdev +lighttpd 1.4.67,484277,"10,088.91 ms",48000.91,"1,094.53 mb",10848848.82,0,0,0,0,151,29,482359,7.36 ms,6710.11 +nginx 1.22.0,439964,"10,096.09 ms",43577.64,"3,752.89 mb",37171731.17,0,0,0,0,0,163,286376,11.44 ms,7715.38 +sws 2.13.1,413213,"10,076.99 ms",41005.59,"2,958.61 mb",29360002.27,0,0,0,0,0,74,94650,12.85 ms,9118.04 +apache 2.4.54,266118,"10,098.43 ms",26352.4,630.98 mb,6248330.38,0,5411,0,0,0,65,808201,89.82 ms,132231.85 +caddy 2.6.1,191447,"10,079.89 ms",18992.97,"3,011.53 mb",29876599.7,0,0,0,0,0,99,418180,38.44 ms,46037.13 +binserve 0.2.0,95837,"10,100.44 ms",9488.4,"1,632.43 mb",16161969.78,0,0,0,0,0,100,142686,51.15 ms,16476.27 diff --git a/benchmark/sws_benchmarks.png b/benchmark/sws_benchmarks.png new file mode 100644 index 0000000..fc9a6c4 Binary files /dev/null and b/benchmark/sws_benchmarks.png differ diff --git a/benchmark/wrk_collector.lua b/benchmark/wrk_collector.lua new file mode 100644 index 0000000..c28fdf8 --- /dev/null +++ b/benchmark/wrk_collector.lua @@ -0,0 +1,77 @@ +-- Script that runs at wrk's "done" stage collecting the stats in JSON and CSV formats. +-- https://github.com/wg/wrk/blob/master/SCRIPTING +done = function(summary, latency, requests) + local json = string.format("{\n") + json = json .. string.format('\t"requests": %d,\n', summary.requests) + json = json .. string.format('\t"duration_ms": %0.2f,\n', summary.duration / 1000) + json = json .. string.format('\t"requests_per_sec": %0.2f,\n', (summary.requests / summary.duration) * 1e6) + json = json .. string.format('\t"bytes": %d,\n', summary.bytes) + json = json .. string.format('\t"bytes_transfer_per_sec": %0.2f,\n', (summary.bytes / summary.duration) * 1e6) + json = json .. string.format('\t"connect_errors": %d,\n', summary.errors.connect) + json = json .. string.format('\t"read_errors": %d,\n', summary.errors.read) + json = json .. string.format('\t"write_errors": %d,\n', summary.errors.write) + json = json .. string.format('\t"http_errors": %d,\n', summary.errors.status) + json = json .. string.format('\t"timeouts": %d,\n', summary.errors.timeout) + json = json .. string.format('\t"latency_min": %0.2f,\n', latency.min) + json = json .. string.format('\t"latency_max": %0.2f,\n', latency.max) + json = json .. string.format('\t"latency_mean_ms": %0.2f,\n', latency.mean / 1000) + json = json .. string.format('\t"latency_stdev": %0.2f,\n', latency.stdev) + + json = json .. string.format('\t"latency_distribution": [\n') + for _, pair in pairs({50, 75, 90, 99}) do + json = json .. string.format("\t\t{\n") + + local percent = latency:percentile(pair) + json = json .. string.format('\t\t\t"percentile": %g,\n\t\t\t"latency_ms": %0.2f\n', pair, percent / 1000) + json = json .. string.format("\t\t}%s\n", pair > 90 and "" or ",") + end + + json = json .. string.format("\t]\n}\n") + + local file, err = io.open("benchmark/wrk_results.json", "w") + if file then + file:write(json) + file:close() + else + print("error saving json results file:", err) + end + + local csv = '' + csv = csv .. string.format('requests,') + csv = csv .. string.format('duration_ms,') + csv = csv .. string.format('requests_per_sec,') + csv = csv .. string.format('bytes,') + csv = csv .. string.format('bytes_transfer_per_sec,') + csv = csv .. string.format('connect_errors,') + csv = csv .. string.format('read_errors,') + csv = csv .. string.format('write_errors,') + csv = csv .. string.format('http_errors,') + csv = csv .. string.format('timeouts,') + csv = csv .. string.format('latency_min,') + csv = csv .. string.format('latency_max,') + csv = csv .. string.format('latency_mean_ms,') + csv = csv .. string.format('latency_stdev\n') + + csv = csv .. string.format('%d,', summary.requests) + csv = csv .. string.format('%0.2f,', summary.duration / 1000) + csv = csv .. string.format('%0.2f,', (summary.requests / summary.duration) * 1e6) + csv = csv .. string.format('%d,', summary.bytes) + csv = csv .. string.format('%0.2f,', (summary.bytes / summary.duration) * 1e6) + csv = csv .. string.format('%d,', summary.errors.connect) + csv = csv .. string.format('%d,', summary.errors.read) + csv = csv .. string.format('%d,', summary.errors.write) + csv = csv .. string.format('%d,', summary.errors.status) + csv = csv .. string.format('%d,', summary.errors.timeout) + csv = csv .. string.format('%0.2f,', latency.min) + csv = csv .. string.format('%0.2f,', latency.max) + csv = csv .. string.format('%0.2f,', latency.mean / 1000) + csv = csv .. string.format('%0.2f\n', latency.stdev) + + local file, err = io.open("benchmark/wrk_results.csv", "w") + if file then + file:write(csv) + file:close() + else + print("error saving csv results file:", err) + end +end -- libgit2 1.7.2