From a345df3eabfd66d21517ce7f9e0efff48c02658b Mon Sep 17 00:00:00 2001 From: Jose Quintana Date: Sat, 29 Apr 2023 15:26:58 +0200 Subject: [PATCH] chore: benchmarks 2023-04 https://github.com/static-web-server/benchmarks --- Makefile | 12 ------------ README.md | 4 ++-- benchmark/BENCHMARKS.md | 51 --------------------------------------------------- benchmark/sws_benchmarks.csv | 7 ------- benchmark/sws_benchmarks.png | Bin 27125 -> 0 bytes benchmark/wrk_collector.lua | 77 ----------------------------------------------------------------------------- docs/content/index.md | 4 ++-- 7 files changed, 4 insertions(+), 151 deletions(-) delete mode 100644 benchmark/BENCHMARKS.md delete mode 100644 benchmark/sws_benchmarks.csv delete mode 100644 benchmark/sws_benchmarks.png delete mode 100644 benchmark/wrk_collector.lua diff --git a/Makefile b/Makefile index ccfa8d8..6c0e639 100644 --- a/Makefile +++ b/Makefile @@ -278,18 +278,6 @@ docs-deploy: @git checkout master .PHONY: docs-deploy -vegeta: - @vegeta --version - @echo "GET http://localhost:8787" | \ - vegeta -cpus=12 attack -workers=12 -connections=500 -rate=6500/s -duration=10s -http2=false > results.bin - @cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]' - @cat results.bin | vegeta plot > plot.html -.PHONY: vegeta - -wrk: - @wrk -c 500 -t 12 -d 10s --latency -s benchmark/wrk_collector.lua $(WRK_URL) -.PHONY: wrk - man: @asciidoctor --doctype=manpage --backend=manpage docs/man/static-web-server.1.rst .PHONY: man diff --git a/README.md b/README.md index a1c03c7..07e7668 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,9 @@ Please refer to [The Documentation Website](https://static-web-server.net/) for ## Benchmarks - + -See more details on [benchmark/BENCHMARKS.md](benchmark/BENCHMARKS.md) +For more details see [benchmarks repository](https://github.com/static-web-server/benchmarks) ## Notes diff --git a/benchmark/BENCHMARKS.md b/benchmark/BENCHMARKS.md deleted file mode 100644 index 42d5cb6..0000000 --- a/benchmark/BENCHMARKS.md +++ /dev/null @@ -1,51 +0,0 @@ -# SWS - Benchmarks 2022 - -> A benchmark suite which measures the requests per second and latency on average for several web servers. - - - -## Motivation and context - -We wanted to prepare some benchmarks to have a general idea about the SWS performance path over time as well as an excuse to improve/optimize SWS in that regard. - -## Important Remarks - -- Note that this is a rough reference about some particular performance aspects (requests per second and latency on average) so take it *at your own risk*. -- Be also aware that **_this benchmark disables in-memory cache_** for some servers intentionally since SWS doesn't provide such a feature yet.
-In the case of some web servers providing that by default then it gets disabled explicitly. See for example [*binserve's author comment*](https://github.com/static-web-server/static-web-server/issues/156#issuecomment-1293941692) in this context. -- Have also in mind that some web servers follow approaches like [Nignx's hybrid disk‑and‑memory cache strategy](https://www.nginx.com/blog/cache-placement-strategies-nginx-plus#Where-is-Cached-Data-Stored) which could work in its favor. - -## Web servers setup - -- **lighttpd** `1.4.67` (default config) -- **nginx** `1.22.0` (default config + worker_processes=4, access_log=off) -- **sws** `2.13.1` (default config) -- **apache** `2.4.54` (default config) -- **caddy** `2.6.1` (default config) -- **binserve** `0.2.0` (default config + **fast_mem_cache=false**, enable_hot_reload=false, enable_logging=false, enable_directory_listing=false) - -## How to measure - -Change `WRK_URL` env with the corresponding server URL to export the [wrk](https://github.com/wg/wrk) metrics. - -```sh -WRK_URL="http://localhost" make wrk -# wrk -c 500 -t 12 -d 10s --latency http://localhost -``` - -After that, wrk will save two files (JSON/CSV) on disk with the resulting metrics. - -## Data used - -The data used can be found in [sws_benchmarks.csv](sws_benchmarks.csv). - -## System used - -- **OS:** Arch Linux -- **Kernel:** 5.19.13-arch1-1 (64 bits) -- **Processor:** 4 × Intel® Core™ i7-6500U -- **RAM:** 8 GiB - -## Contributions - -Of course, if you find any issue or remark/improvement to do please don't hesitate to file an [issue](https://github.com/static-web-server/static-web-server/issues) or send a [Pull request](https://github.com/static-web-server/static-web-server/pulls). diff --git a/benchmark/sws_benchmarks.csv b/benchmark/sws_benchmarks.csv deleted file mode 100644 index bc155e9..0000000 --- a/benchmark/sws_benchmarks.csv +++ /dev/null @@ -1,7 +0,0 @@ -server,requests,duration_ms,requests_per_sec,bytes,bytes_transfer_per_sec,connect_errors,read_errors,write_errors,http_errors,timeouts,latency_min,latency_max,latency_mean_ms,latency_stdev -lighttpd 1.4.67,484277,"10,088.91 ms",48000.91,"1,094.53 mb",10848848.82,0,0,0,0,151,29,482359,7.36 ms,6710.11 -nginx 1.22.0,439964,"10,096.09 ms",43577.64,"3,752.89 mb",37171731.17,0,0,0,0,0,163,286376,11.44 ms,7715.38 -sws 2.13.1,413213,"10,076.99 ms",41005.59,"2,958.61 mb",29360002.27,0,0,0,0,0,74,94650,12.85 ms,9118.04 -apache 2.4.54,266118,"10,098.43 ms",26352.4,630.98 mb,6248330.38,0,5411,0,0,0,65,808201,89.82 ms,132231.85 -caddy 2.6.1,191447,"10,079.89 ms",18992.97,"3,011.53 mb",29876599.7,0,0,0,0,0,99,418180,38.44 ms,46037.13 -binserve 0.2.0,95837,"10,100.44 ms",9488.4,"1,632.43 mb",16161969.78,0,0,0,0,0,100,142686,51.15 ms,16476.27 diff --git a/benchmark/sws_benchmarks.png b/benchmark/sws_benchmarks.png deleted file mode 100644 index 3af190d..0000000 Binary files a/benchmark/sws_benchmarks.png and /dev/null differ diff --git a/benchmark/wrk_collector.lua b/benchmark/wrk_collector.lua deleted file mode 100644 index c28fdf8..0000000 --- a/benchmark/wrk_collector.lua +++ /dev/null @@ -1,77 +0,0 @@ --- Script that runs at wrk's "done" stage collecting the stats in JSON and CSV formats. --- https://github.com/wg/wrk/blob/master/SCRIPTING -done = function(summary, latency, requests) - local json = string.format("{\n") - json = json .. string.format('\t"requests": %d,\n', summary.requests) - json = json .. string.format('\t"duration_ms": %0.2f,\n', summary.duration / 1000) - json = json .. string.format('\t"requests_per_sec": %0.2f,\n', (summary.requests / summary.duration) * 1e6) - json = json .. string.format('\t"bytes": %d,\n', summary.bytes) - json = json .. string.format('\t"bytes_transfer_per_sec": %0.2f,\n', (summary.bytes / summary.duration) * 1e6) - json = json .. string.format('\t"connect_errors": %d,\n', summary.errors.connect) - json = json .. string.format('\t"read_errors": %d,\n', summary.errors.read) - json = json .. string.format('\t"write_errors": %d,\n', summary.errors.write) - json = json .. string.format('\t"http_errors": %d,\n', summary.errors.status) - json = json .. string.format('\t"timeouts": %d,\n', summary.errors.timeout) - json = json .. string.format('\t"latency_min": %0.2f,\n', latency.min) - json = json .. string.format('\t"latency_max": %0.2f,\n', latency.max) - json = json .. string.format('\t"latency_mean_ms": %0.2f,\n', latency.mean / 1000) - json = json .. string.format('\t"latency_stdev": %0.2f,\n', latency.stdev) - - json = json .. string.format('\t"latency_distribution": [\n') - for _, pair in pairs({50, 75, 90, 99}) do - json = json .. string.format("\t\t{\n") - - local percent = latency:percentile(pair) - json = json .. string.format('\t\t\t"percentile": %g,\n\t\t\t"latency_ms": %0.2f\n', pair, percent / 1000) - json = json .. string.format("\t\t}%s\n", pair > 90 and "" or ",") - end - - json = json .. string.format("\t]\n}\n") - - local file, err = io.open("benchmark/wrk_results.json", "w") - if file then - file:write(json) - file:close() - else - print("error saving json results file:", err) - end - - local csv = '' - csv = csv .. string.format('requests,') - csv = csv .. string.format('duration_ms,') - csv = csv .. string.format('requests_per_sec,') - csv = csv .. string.format('bytes,') - csv = csv .. string.format('bytes_transfer_per_sec,') - csv = csv .. string.format('connect_errors,') - csv = csv .. string.format('read_errors,') - csv = csv .. string.format('write_errors,') - csv = csv .. string.format('http_errors,') - csv = csv .. string.format('timeouts,') - csv = csv .. string.format('latency_min,') - csv = csv .. string.format('latency_max,') - csv = csv .. string.format('latency_mean_ms,') - csv = csv .. string.format('latency_stdev\n') - - csv = csv .. string.format('%d,', summary.requests) - csv = csv .. string.format('%0.2f,', summary.duration / 1000) - csv = csv .. string.format('%0.2f,', (summary.requests / summary.duration) * 1e6) - csv = csv .. string.format('%d,', summary.bytes) - csv = csv .. string.format('%0.2f,', (summary.bytes / summary.duration) * 1e6) - csv = csv .. string.format('%d,', summary.errors.connect) - csv = csv .. string.format('%d,', summary.errors.read) - csv = csv .. string.format('%d,', summary.errors.write) - csv = csv .. string.format('%d,', summary.errors.status) - csv = csv .. string.format('%d,', summary.errors.timeout) - csv = csv .. string.format('%0.2f,', latency.min) - csv = csv .. string.format('%0.2f,', latency.max) - csv = csv .. string.format('%0.2f,', latency.mean / 1000) - csv = csv .. string.format('%0.2f\n', latency.stdev) - - local file, err = io.open("benchmark/wrk_results.csv", "w") - if file then - file:write(csv) - file:close() - else - print("error saving csv results file:", err) - end -end diff --git a/docs/content/index.md b/docs/content/index.md index e537b82..e93ac7e 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -73,6 +73,6 @@ It's cross-platform and available for `Linux`, `macOS`, `Windows` and `FreeBSD` ## Benchmarks - + -For more details see [benchmark/BENCHMARKS.md](https://github.com/static-web-server/static-web-server/blob/master/benchmark/BENCHMARKS.md). +For more details see [benchmarks repository](https://github.com/static-web-server/benchmarks) -- libgit2 1.7.2