diff --git a/.gitignore b/.gitignore index f2612c3..ea9ee7e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,7 @@ docs/ .env .DS_Store certs/ + +# Benchmark artifacts (certs generated by run.sh, results are local-only) +benchmarks/certs/ +benchmarks/results/ diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..2c31886 --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,97 @@ +# Benchmarks + +Comparative benchmarks of tiny-proxy, nginx, and Caddy as reverse proxies. + +All three proxies were configured with equivalent routing rules, forwarding to the same backend (hashicorp/http-echo). The only variable is the proxy implementation. + +## Environment + +- **Host:** Docker Desktop on macOS (Apple Silicon, M-series) +- **Tool:** [hey](https://github.com/rakyll/hey) +- **Proxies:** tiny-proxy 0.3.0 (Rust, rustls), nginx (alpine), Caddy (alpine) +- **Backend:** hashicorp/http-echo (minimal response overhead) +- **Method:** 10 000 requests, 100 concurrent connections, best of 3 runs +- **Warmup:** 200 requests before each measurement +- **Containers:** each proxy and the backend ran in separate Docker containers on the same host + +> **Note:** These are local benchmarks run through Docker networking on a single machine. Absolute numbers will differ on dedicated hardware. The relative comparison between proxies is what's useful here. + +## Results + +### 1. Plain Text (~11 bytes response) + +| Proxy | RPS | Avg | p50 | p90 | p95 | p99 | +|-------|-----|-----|-----|-----|-----|-----| +| tiny-proxy | 16 275 | 5.8ms | 5.5ms | 8.6ms | 10.1ms | 22.2ms | +| nginx | 16 418 | 5.9ms | 5.1ms | 9.5ms | 11.2ms | 19.5ms | +| Caddy | 17 542 | 5.5ms | 4.8ms | 9.1ms | 10.5ms | 22.4ms | + +All three proxies perform within ~7% of each other. The differences are within run-to-run variance. + +### 2. JSON API (~200 bytes response) + +| Proxy | RPS | Avg | p50 | p90 | p95 | p99 | +|-------|-----|-----|-----|-----|-----|-----| +| tiny-proxy | 17 184 | 5.7ms | 5.3ms | 7.9ms | 8.9ms | 16.2ms | +| nginx | 20 273 | 4.8ms | 4.2ms | 7.0ms | 8.3ms | 19.8ms | +| Caddy | 17 596 | 5.5ms | 4.6ms | 9.3ms | 10.9ms | 24.7ms | + +nginx shows higher throughput on this scenario (~18% more RPS than tiny-proxy). Latency distributions are comparable at p50, with tiny-proxy showing tighter p99. + +### 3. TLS Termination + +Each request establishes a new TCP connection and TLS handshake (`-disable-keepalive`), making this the most demanding scenario. + +| Proxy | RPS | Avg | p50 | p90 | p95 | p99 | +|-------|-----|-----|-----|-----|-----|-----| +| tiny-proxy | 2 672 | 36.9ms | 36.0ms | 49.1ms | 53.6ms | 61.5ms | +| nginx | 2 437 | 39.3ms | 33.6ms | 70.1ms | 88.0ms | 117.4ms | +| Caddy | 2 129 | 44.2ms | 33.5ms | 90.2ms | 115.5ms | 157.8ms | + +tiny-proxy shows ~10% higher throughput than nginx and ~25% higher than Caddy. The p99 latency difference is more pronounced: tiny-proxy at 61ms vs nginx at 117ms and Caddy at 158ms. This is likely due to rustls (with aws-lc-rs backend) handling TLS handshakes efficiently. + +## Configuration Details + +All proxies used the same routing: two paths (`/text/`, `/json/`) forwarding to separate backend instances. + +**tiny-proxy:** +``` +localhost:8080 { + handle_path /text/* { reverse_proxy backend:9000 } + handle_path /json/* { reverse_proxy backend-json:9000 } +} +``` + +**nginx:** +```nginx +location /text/ { proxy_pass http://backend:9000/; } +location /json/ { proxy_pass http://backend-json:9000/; } +``` + +**Caddy:** +``` +:8082 { + reverse_proxy /text/* backend:9000 + reverse_proxy /json/* backend-json:9000 +} +``` + +All proxies ran with out-of-the-box defaults — no worker tuning, buffer sizing, keepalive optimization, or OS-level tweaks. Both nginx and Caddy have extensive tuning options (e.g., `worker_processes`, `worker_connections`, `proxy_buffer_size`, `keepalive_timeout`) that can meaningfully improve their numbers. These benchmarks reflect a fair "zero-config" comparison, not maximum achievable performance for any of the proxies. + +## Reproduce + +```bash +cd benchmarks +docker compose up -d +./run.sh +``` + +See `benchmarks/` for compose file, proxy configs, and the runner script. + +## Image Sizes + +| Proxy | Docker image size | +|-------|-------------------| +| tiny-proxy | 22 MB (Alpine + 4.6 MB binary) | +| nginx | 43 MB (nginx:alpine) | +| Caddy | 40 MB (caddy:alpine) | diff --git a/benchmarks/compose.yml b/benchmarks/compose.yml new file mode 100644 index 0000000..3858e44 --- /dev/null +++ b/benchmarks/compose.yml @@ -0,0 +1,66 @@ +# Benchmark: tiny-proxy vs nginx vs Caddy +# +# 3 equivalent reverse proxies → same backend → measure proxy overhead +# +# Ports: +# 9001 backend (http-echo, plain text "Hello World") +# 9002 backend-json (http-echo, JSON response ~200 bytes) +# 8080 tiny-proxy +# 8081 nginx +# 8082 Caddy +# 8443 tiny-proxy (TLS) +# 8444 nginx (TLS) +# 8445 Caddy (TLS) + +services: + backend: + image: hashicorp/http-echo + command: ["-text=Hello World", "-listen=:9000"] + expose: + - "9000" + + backend-json: + image: hashicorp/http-echo + command: ["-text={\"status\":\"ok\",\"items\":[1,2,3,4,5,6,7,8,9,10],\"page\":1,\"total\":100}", "-listen=:9000"] + expose: + - "9000" + + # --- Proxies (HTTP) --- + + tiny-proxy: + build: + context: .. + dockerfile: Dockerfile + ports: + - "8080:8080" + - "8443:8443" + volumes: + - ./proxies/tiny-proxy.caddy:/etc/tiny-proxy/config.caddy:ro + - ./certs:/etc/ssl/tiny-proxy:ro + depends_on: + - backend + - backend-json + + nginx: + image: nginx:alpine + ports: + - "8081:80" + - "8444:443" + volumes: + - ./proxies/nginx.conf:/etc/nginx/nginx.conf:ro + - ./certs:/etc/ssl/bench:ro + depends_on: + - backend + - backend-json + + caddy: + image: caddy:alpine + ports: + - "8082:8082" + - "8445:8445" + volumes: + - ./proxies/Caddyfile:/etc/caddy/Caddyfile:ro + - ./certs:/etc/ssl/bench:ro + depends_on: + - backend + - backend-json diff --git a/benchmarks/proxies/Caddyfile b/benchmarks/proxies/Caddyfile new file mode 100644 index 0000000..6e01fc8 --- /dev/null +++ b/benchmarks/proxies/Caddyfile @@ -0,0 +1,10 @@ +:8082 { + reverse_proxy /text/* backend:9000 + reverse_proxy /json/* backend-json:9000 +} + +:8445 { + tls /etc/ssl/bench/cert.pem /etc/ssl/bench/key.pem + reverse_proxy /text/* backend:9000 + reverse_proxy /json/* backend-json:9000 +} diff --git a/benchmarks/proxies/nginx.conf b/benchmarks/proxies/nginx.conf new file mode 100644 index 0000000..cbb22cb --- /dev/null +++ b/benchmarks/proxies/nginx.conf @@ -0,0 +1,42 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + server { + listen 80; + + location /text/ { + proxy_pass http://backend:9000/; + proxy_set_header Host $host; + proxy_http_version 1.1; + } + + location /json/ { + proxy_pass http://backend-json:9000/; + proxy_set_header Host $host; + proxy_http_version 1.1; + } + } + + server { + listen 443 ssl; + + ssl_certificate /etc/ssl/bench/cert.pem; + ssl_certificate_key /etc/ssl/bench/key.pem; + + location /text/ { + proxy_pass http://backend:9000/; + proxy_set_header Host $host; + proxy_http_version 1.1; + } + + location /json/ { + proxy_pass http://backend-json:9000/; + proxy_set_header Host $host; + proxy_http_version 1.1; + } + } +} diff --git a/benchmarks/proxies/tiny-proxy.caddy b/benchmarks/proxies/tiny-proxy.caddy new file mode 100644 index 0000000..bc607da --- /dev/null +++ b/benchmarks/proxies/tiny-proxy.caddy @@ -0,0 +1,18 @@ +localhost:8080 { + handle_path /text/* { + reverse_proxy backend:9000 + } + handle_path /json/* { + reverse_proxy backend-json:9000 + } +} + +localhost:8443 { + tls /etc/ssl/tiny-proxy/cert.pem /etc/ssl/tiny-proxy/key.pem + handle_path /text/* { + reverse_proxy backend:9000 + } + handle_path /json/* { + reverse_proxy backend-json:9000 + } +} diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 0000000..6b698bf --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# Benchmark: tiny-proxy vs nginx vs Caddy +# +# Usage: +# cd benchmarks +# ./run.sh # run all tests +# ./run.sh --skip-tls # skip TLS tests +# +# Prerequisites: +# brew install hey + +set -uo pipefail + +cd "$(dirname "$0")" + +# --- Config --- +REQUESTS=10000 +CONNECTIONS=100 +RUNS=3 +RESULTS_DIR="results" + +log() { echo -e "\033[0;32m[BENCH]\033[0m $*" >&2; } + +# --- Check hey --- +if ! command -v hey &>/dev/null; then + echo "error: 'hey' not found. Install: brew install hey" >&2 + exit 1 +fi + +# --- Generate certs if needed --- +if [ ! -f certs/cert.pem ]; then + log "Generating self-signed certificates..." + mkdir -p certs + openssl req -x509 -newkey rsa:2048 \ + -keyout certs/key.pem -out certs/cert.pem \ + -days 365 -nodes -subj "/CN=localhost" 2>/dev/null +fi + +# --- Start services --- +log "Starting services..." +docker compose -f compose.yml up -d --build + +log "Waiting for services to be ready..." +sleep 3 + +for port in 8080 8081 8082; do + for attempt in $(seq 1 10); do + if curl -sf "http://localhost:$port/text/" -o /dev/null 2>/dev/null; then + log "HTTP port $port ✓" + break + fi + if [ "$attempt" -eq 10 ]; then + echo "error: HTTP port $port failed to respond" >&2 + docker compose -f compose.yml logs --tail 10 + exit 1 + fi + sleep 1 + done +done + +if [[ "${1:-}" != "--skip-tls" ]]; then + for port in 8443 8444 8445; do + for attempt in $(seq 1 10); do + if curl -skf "https://localhost:$port/text/" -o /dev/null 2>/dev/null; then + log "TLS port $port ✓" + break + fi + if [ "$attempt" -eq 10 ]; then + echo "error: TLS port $port failed to respond" >&2 + docker compose -f compose.yml logs --tail 10 + exit 1 + fi + sleep 1 + done + done +fi + +# --- Prepare results --- +mkdir -p "$RESULTS_DIR" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +SUMMARY_FILE="$RESULTS_DIR/summary_${TIMESTAMP}.md" + +# --- Parse hey output --- +parse_hey() { + local output="$1" + local rps avg p50 p90 p95 p99 + + rps=$(echo "$output" | grep "Requests/sec" | awk '{printf "%.0f", $2}') + avg=$(echo "$output" | grep "Average:" | awk '{print $2}') + p50=$(echo "$output" | awk '/50%/ {print $3; exit}') + p90=$(echo "$output" | awk '/90%/ {print $3; exit}') + p95=$(echo "$output" | awk '/95%/ {print $3; exit}') + p99=$(echo "$output" | awk '/99%/ {print $3; exit}') + + echo "${rps:-0}|${avg:-0}|${p50:-0}|${p90:-0}|${p95:-0}|${p99:-0}" +} + +# --- Run a single benchmark --- +# Only the final result line goes to stdout (captured by caller). +# All progress messages go to stderr. +run_bench() { + local label=$1 + local url=$2 + shift 2 + + local best_rps=0 + local best_parsed="" + + for i in $(seq 1 $RUNS); do + # Warmup on first run + if [ "$i" -eq 1 ]; then + hey -n 200 -c 10 "$@" "$url" >/dev/null 2>&1 || true + sleep 1 + fi + + local output + output=$(hey -n "$REQUESTS" -c "$CONNECTIONS" "$@" "$url" 2>&1) || true + + local parsed + parsed=$(parse_hey "$output") + local rps + rps=$(echo "$parsed" | cut -d'|' -f1) + + if ! echo "$output" | grep -q '\[200\]'; then + log " $label run $i: FAILED (no HTTP 200 — check hey output below)" >&2 + echo "$output" >&2 + rps=0 + elif [ "${rps:-0}" -eq 0 ] 2>/dev/null; then + log " $label run $i: FAILED (0 RPS — check hey output below)" >&2 + echo "$output" >&2 + else + log " $label run $i: ${rps} RPS" + fi + + if [ "${rps:-0}" -gt "${best_rps:-0}" ] 2>/dev/null; then + best_rps=$rps + best_parsed=$parsed + fi + + sleep 1 + done + + if [ "${best_rps:-0}" -eq 0 ] 2>/dev/null; then + echo "error: $label produced 0 RPS in all runs" >&2 + exit 1 + fi + + echo "${label}|${best_parsed}" +} + +# --- Scenarios --- +log "Running benchmarks..." +echo "" >&2 + +declare -a ALL_RESULTS + +log "Scenario 1: Plain text reverse proxy" +ALL_RESULTS+=("$(run_bench "tiny-proxy" "http://localhost:8080/text/")") +ALL_RESULTS+=("$(run_bench "nginx" "http://localhost:8081/text/")") +ALL_RESULTS+=("$(run_bench "caddy" "http://localhost:8082/text/")") +echo "" >&2 + +log "Scenario 2: JSON API proxy" +ALL_RESULTS+=("$(run_bench "tiny-proxy" "http://localhost:8080/json/")") +ALL_RESULTS+=("$(run_bench "nginx" "http://localhost:8081/json/")") +ALL_RESULTS+=("$(run_bench "caddy" "http://localhost:8082/json/")") +echo "" >&2 + +if [[ "${1:-}" != "--skip-tls" ]]; then + # Non-standard TLS ports: hey puts "host:port" in SNI by default, which rustls + # rejects (RFC 6066). -host sets SNI to the bare hostname. + log "Scenario 3: TLS termination (-disable-keepalive -host localhost)" + ALL_RESULTS+=("$(run_bench "tiny-proxy" "https://localhost:8443/text/" -disable-keepalive -host localhost)") + ALL_RESULTS+=("$(run_bench "nginx" "https://localhost:8444/text/" -disable-keepalive -host localhost)") + ALL_RESULTS+=("$(run_bench "caddy" "https://localhost:8445/text/" -disable-keepalive -host localhost)") + echo "" >&2 +fi + +# --- Generate summary --- +log "Generating summary..." + +print_table() { + local title=$1 + local start=$2 + local count=$3 + + echo "### $title" + echo "" + echo "| Proxy | RPS | Avg | p50 | p90 | p95 | p99 |" + echo "|-------|-----|-----|-----|-----|-----|-----|" + + for r in "${ALL_RESULTS[@]:$start:$count}"; do + IFS='|' read -r name rps avg p50 p90 p95 p99 <<< "$r" + printf "| %s | %s | %s | %s | %s | %s | %s |\n" \ + "$name" "${rps:---}" "${avg:---}" "${p50:---}" "${p90:---}" "${p95:---}" "${p99:---}" + done + echo "" +} + +{ + echo "# Benchmark Results" + echo "" + echo "**Date:** $(date)" + echo "**Tool:** hey — $REQUESTS requests, $CONNECTIONS connections, best of $RUNS runs" + echo "**Environment:** Docker Desktop on $(uname -sm)" + echo "" + + print_table "1. Plain Text (~11 bytes response)" 0 3 + print_table "2. JSON API (~200 bytes response)" 3 3 + + if [[ "${1:-}" != "--skip-tls" ]] && [ ${#ALL_RESULTS[@]} -gt 6 ]; then + print_table "3. TLS Termination" 6 3 + fi + + echo "## Methodology" + echo "" + echo "- **Tool:** [hey](https://github.com/rakyll/hey)" + echo "- **Warmup:** 200 requests before measurement" + echo "- **Measurement:** $REQUESTS requests, $CONNECTIONS concurrent" + echo "- **Best** of $RUNS runs reported" + echo "- **Backend:** hashicorp/http-echo (minimal overhead)" + if [[ "${1:-}" != "--skip-tls" ]]; then + echo "- **TLS:** \`-disable-keepalive\` (new TCP+TLS handshake per request)" + echo "- **SNI:** \`-host localhost\` (hey otherwise sends \`localhost:844x\` in SNI; rustls rejects that)" + fi + echo "" + echo "## Reproduce" + echo "" + echo '```bash' + echo "cd benchmarks" + echo "docker compose up -d" + echo "./run.sh" + echo '```' +} > "$SUMMARY_FILE" + +echo "" +cat "$SUMMARY_FILE" + +log "Results saved to $SUMMARY_FILE" + +log "Stopping services..." +docker compose -f compose.yml down + +log "Done!"