chore: benchmarks [skip ci] (#155)
- lighttpd 1.4.67 (default config)
- nginx 1.22.0 (default config + worker_processes=4)
- sws 2.13.1 (default config)
- apache 2.4.54 (default config)
- caddy 2.6.1 (default config)
- binserve 0.2.0 (default config + fast_mem_cache=false, enable_hot_reload=false, enable_logging=false, enable_directory_listing=false)
Diff
.gitignore | 3 ++-
Makefile | 7 +++-
README.md | 6 ++++-
benchmark/BENCHMARKS.md | 24 ++++++++++++++-
benchmark/sws_benchmarks.csv | 7 ++++-
benchmark/sws_benchmarks.png | Bin 0 -> 26580 bytes
benchmark/wrk_collector.lua | 77 +++++++++++++++++++++++++++++++++++++++++++++-
7 files changed, 123 insertions(+), 1 deletion(-)
@@ -18,6 +18,8 @@
**/*.out*
**/*.perf*
**/*empty*
**/*.json
**/*.csv
release
.vscode
TODO
@@ -28,3 +30,4 @@ docs/*/**.html
!sample.env
!/docs
!/tests/fixtures/**/*
!benchmark/sws_benchmarks*.csv
@@ -282,12 +282,17 @@ promote:
.PHONY: promote
loadtest:
@vegeta --version
@echo "GET http://localhost:8787" | \
vegeta -cpus=12 attack -workers=10 -duration=60s -connections=10000 -rate=200 -http2=false > results.bin
vegeta -cpus=12 attack -workers=12 -connections=500 -rate=6500/s -duration=10s -http2=false > results.bin
@cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]'
@cat results.bin | vegeta plot > plot.html
.PHONY: loadtest
wrk:
@wrk -c 500 -t 12 -d 10s --latency -s benchmark/wrk_collector.lua $(WRK_URL)
.PHONY: wrk
man:
@asciidoctor --doctype=manpage --backend=manpage docs/man/static-web-server.1.rst
.PHONY: man
@@ -74,6 +74,12 @@ For more details about the API, usage and examples please have a look at [The Do
- [Release Binaries](https://github.com/joseluisq/static-web-server/releases)
- [Platforms/Architectures Supported](https://sws.joseluisq.net/platforms-architectures/)
## Benchmarks
<img title="SWS - Benchmarks 2022" src="benchmark/sws_benchmarks.png" width="860">
See more details on [benchmark/BENCHMARKS.md](benchmark/BENCHMARKS.md)
## Notes
- If you're looking for `v1` please go to [1.x](https://github.com/joseluisq/static-web-server/tree/1.x) branch.
@@ -0,0 +1,24 @@
# SWS - Benchmarks 2022
> A benchmark suite which measures the requests per second and latency in average for several web servers.
<img title="SWS - Benchmarks 2022" src="data/sws_benchmarks.png" width="860">
## How to use
Change `WRK_URL` with the corresponding server URL to export the wrk metrics.
```sh
WRK_URL="http://localhost" make wrk
```
## System
- **OS:** Arch Linux
- **Kernel:** 5.19.13-arch1-1 (64 bits)
- **Processor:** 4 × Intel® Core™ i7-6500U
- **RAM:** 8 GiB
## Data
For data used see [data](./data/) directory for more details.
@@ -0,0 +1,7 @@
server,requests,duration_ms,requests_per_sec,bytes,bytes_transfer_per_sec,connect_errors,read_errors,write_errors,http_errors,timeouts,latency_min,latency_max,latency_mean_ms,latency_stdev
lighttpd 1.4.67,484277,"10,088.91 ms",48000.91,"1,094.53 mb",10848848.82,0,0,0,0,151,29,482359,7.36 ms,6710.11
nginx 1.22.0,439964,"10,096.09 ms",43577.64,"3,752.89 mb",37171731.17,0,0,0,0,0,163,286376,11.44 ms,7715.38
sws 2.13.1,413213,"10,076.99 ms",41005.59,"2,958.61 mb",29360002.27,0,0,0,0,0,74,94650,12.85 ms,9118.04
apache 2.4.54,266118,"10,098.43 ms",26352.4,630.98 mb,6248330.38,0,5411,0,0,0,65,808201,89.82 ms,132231.85
caddy 2.6.1,191447,"10,079.89 ms",18992.97,"3,011.53 mb",29876599.7,0,0,0,0,0,99,418180,38.44 ms,46037.13
binserve 0.2.0,95837,"10,100.44 ms",9488.4,"1,632.43 mb",16161969.78,0,0,0,0,0,100,142686,51.15 ms,16476.27
Binary files /dev/null and b/benchmark/sws_benchmarks.png differ
@@ -0,0 +1,77 @@
done = function(summary, latency, requests)
local json = string.format("{\n")
json = json .. string.format('\t"requests": %d,\n', summary.requests)
json = json .. string.format('\t"duration_ms": %0.2f,\n', summary.duration / 1000)
json = json .. string.format('\t"requests_per_sec": %0.2f,\n', (summary.requests / summary.duration) * 1e6)
json = json .. string.format('\t"bytes": %d,\n', summary.bytes)
json = json .. string.format('\t"bytes_transfer_per_sec": %0.2f,\n', (summary.bytes / summary.duration) * 1e6)
json = json .. string.format('\t"connect_errors": %d,\n', summary.errors.connect)
json = json .. string.format('\t"read_errors": %d,\n', summary.errors.read)
json = json .. string.format('\t"write_errors": %d,\n', summary.errors.write)
json = json .. string.format('\t"http_errors": %d,\n', summary.errors.status)
json = json .. string.format('\t"timeouts": %d,\n', summary.errors.timeout)
json = json .. string.format('\t"latency_min": %0.2f,\n', latency.min)
json = json .. string.format('\t"latency_max": %0.2f,\n', latency.max)
json = json .. string.format('\t"latency_mean_ms": %0.2f,\n', latency.mean / 1000)
json = json .. string.format('\t"latency_stdev": %0.2f,\n', latency.stdev)
json = json .. string.format('\t"latency_distribution": [\n')
for _, pair in pairs({50, 75, 90, 99}) do
json = json .. string.format("\t\t{\n")
local percent = latency:percentile(pair)
json = json .. string.format('\t\t\t"percentile": %g,\n\t\t\t"latency_ms": %0.2f\n', pair, percent / 1000)
json = json .. string.format("\t\t}%s\n", pair > 90 and "" or ",")
end
json = json .. string.format("\t]\n}\n")
local file, err = io.open("benchmark/wrk_results.json", "w")
if file then
file:write(json)
file:close()
else
print("error saving json results file:", err)
end
local csv = ''
csv = csv .. string.format('requests,')
csv = csv .. string.format('duration_ms,')
csv = csv .. string.format('requests_per_sec,')
csv = csv .. string.format('bytes,')
csv = csv .. string.format('bytes_transfer_per_sec,')
csv = csv .. string.format('connect_errors,')
csv = csv .. string.format('read_errors,')
csv = csv .. string.format('write_errors,')
csv = csv .. string.format('http_errors,')
csv = csv .. string.format('timeouts,')
csv = csv .. string.format('latency_min,')
csv = csv .. string.format('latency_max,')
csv = csv .. string.format('latency_mean_ms,')
csv = csv .. string.format('latency_stdev\n')
csv = csv .. string.format('%d,', summary.requests)
csv = csv .. string.format('%0.2f,', summary.duration / 1000)
csv = csv .. string.format('%0.2f,', (summary.requests / summary.duration) * 1e6)
csv = csv .. string.format('%d,', summary.bytes)
csv = csv .. string.format('%0.2f,', (summary.bytes / summary.duration) * 1e6)
csv = csv .. string.format('%d,', summary.errors.connect)
csv = csv .. string.format('%d,', summary.errors.read)
csv = csv .. string.format('%d,', summary.errors.write)
csv = csv .. string.format('%d,', summary.errors.status)
csv = csv .. string.format('%d,', summary.errors.timeout)
csv = csv .. string.format('%0.2f,', latency.min)
csv = csv .. string.format('%0.2f,', latency.max)
csv = csv .. string.format('%0.2f,', latency.mean / 1000)
csv = csv .. string.format('%0.2f\n', latency.stdev)
local file, err = io.open("benchmark/wrk_results.csv", "w")
if file then
file:write(csv)
file:close()
else
print("error saving csv results file:", err)
end
end
