diff --git a/crates/yip-bench/cpu-bound-regime.md b/crates/yip-bench/cpu-bound-regime.md new file mode 100644 index 0000000..dc5ea41 --- /dev/null +++ b/crates/yip-bench/cpu-bound-regime.md @@ -0,0 +1,64 @@ +# CPU-bound-regime spike (#4 throughput campaign go/no-go) + +Script: `crates/yip-bench/tests/run-cpu-regime.sh` +Run: `sudo bash crates/yip-bench/tests/run-cpu-regime.sh target/release/yipd` + +**Question:** can the yip receiver ever become CPU-bound (its packet-processing +core pegged), or is throughput always RTT/window-bound? This is the go/no-go for +the #4 CPU-optimization work (codec/MAC swaps): if the receiver never saturates +a core, those wins can never move end-to-end throughput. + +**Method:** two netns + veth, yip tunnel between them. The receiving yipd is +`taskset`-pinned to ONE core (models the 1-core EPYC target); sender yipd + iperf +get other cores. Push data B->A (UDP blast, then TCP -P 8) while sweeping netem +RTT; record tunnel throughput + the pinned RX core's utilization (1.0 = one core +saturated). Dev box: Ryzen 5 7640U, 12 core. + +## Result + +| RTT | flow | Gbps | rx_cores | udp_loss | +|------|------|------:|---------:|---------:| +| 0ms | udp | 1.196 | 0.96 | 80.1% | +| 0ms | tcp | 0.969 | 0.86 | - | +| 1ms | udp | 1.250 | 0.98 | 79.5% | +| 1ms | tcp | 0.798 | 0.76 | - | +| 5ms | udp | 1.222 | 0.97 | 79.8% | +| 5ms | tcp | 0.627 | 0.43 | - | +| 12ms | udp | 1.243 | 0.97 | 79.6% | +| 12ms | tcp | 0.504 | 0.48 | - | +| 24ms | udp | 1.049 | 0.96 | 82.9% | +| 24ms | tcp | 0.424 | 0.38 | - | + +## Verdict: a CPU-bound regime EXISTS + +- **UDP blast pins the RX core at ~0.97 and caps at ~1.2 Gbps with ~80% loss at + every RTT.** The single core is a hard ~1.2 Gbps processing ceiling; offered + load above it is dropped. Unambiguous CPU bound. +- **TCP is CPU-bound at low RTT, window-bound at high RTT.** 0ms: 0.97 Gbps @ + core 0.86 (near-ceiling). 24ms: 0.42 Gbps @ core 0.38 (core idle, window caps). +- The earlier "CPU wins don't move throughput" verdict (4b, #58) was + **regime-specific to the 24ms single-flow WAN path** — window-bound, core idle. + +## Implication for #4 + +CPU optimizations DO translate to throughput in the CPU-bound regime: +short-RTT / regional paths and aggregate / many-parallel-flow traffic, where the +1-core receiver ceiling (~1.2 Gbps here) is the binding constraint. In that +regime a 4% MAC saving ~= +50 Mbps, and larger codec-path wins scale directly. +They do NOT help the high-RTT single-flow case (window-bound). + +**Caveat (NIC):** veth has no real NIC, so absolute Gbps is optimistic — a real +NIC adds its own ceiling. But the per-core processing ceiling is real and still +binds a fast-NIC / high-parallelism box. The finding (a CPU regime exists) is a +strong positive despite the optimistic absolute number. + +**Caveat (measurement):** `rx_cores` is `utime+stime` from `/proc/PID/stat`, which +excludes softirq / kernel-side receive work not charged to the yipd pid. True +per-core cost is therefore *higher* than reported — which only strengthens the +"saturated" conclusion (0.96–0.98 is already at the ceiling). + +## Where the RX core actually goes (next step if pursuing #4) + +The ~1.2 Gbps ceiling is the sum of the receive path: TUN-write (~20% in the 4b +profile) + AEAD decrypt + FEC decode + SipHash. To raise the ceiling, attack the +biggest components first (TUN-write, then AEAD), not just the ~9% SipHash. diff --git a/crates/yip-bench/tests/run-cpu-regime.sh b/crates/yip-bench/tests/run-cpu-regime.sh new file mode 100755 index 0000000..a6e87c2 --- /dev/null +++ b/crates/yip-bench/tests/run-cpu-regime.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# CPU-bound-regime spike for the #4 throughput campaign. +# +# Question: can the yip RECEIVER ever become CPU-bound (its packet-processing +# core pegged at ~100%), or is throughput always RTT/window-bound? If the +# receiver core never saturates even under a UDP blast at ~0 RTT, then CPU +# optimizations (codec/MAC swaps) can NEVER move end-to-end throughput and the +# rest of #4 should yield to feature work. +# +# Method: two netns joined by a veth, a yip tunnel between them. The RECEIVING +# yipd (yipd_A) is taskset-pinned to a SINGLE core to model the 1-core EPYC +# target; everything else (sender yipd_B, iperf) gets other cores so they are +# never the bottleneck. We push data B->A with iperf3 (UDP blast, then TCP -P N) +# while sweeping netem RTT, and at each point record tunnel throughput plus the +# pinned RX core's utilization (cores busy: 1.0 == one core saturated). +# +# Caveat: veth has no real NIC, so absolute Gbps is optimistic — which only +# makes CPU-bound HARDER to reach. A negative result here is a strong negative. +# +# Usage: sudo run-cpu-regime.sh [path-to-yipd-binary] +set -euo pipefail + +if [ $# -ge 1 ] && [ -n "${1:-}" ]; then + YIPD="$1" +else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + WORKSPACE_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + echo "[build] cargo build --release -p yipd" + cargo build --release -p yipd --quiet --manifest-path "$WORKSPACE_ROOT/Cargo.toml" + YIPD="$WORKSPACE_ROOT/target/release/yipd" +fi + +# ── core assignment (models a 1-core receiver) ─────────────────────────────── +NCORES="$(nproc)" +if [ "$NCORES" -lt 6 ]; then + echo "[error] need >= 6 cores to isolate a 1-core receiver cleanly; have $NCORES" + exit 1 +fi +RX_CORE=2 # yipd_A (receiver under test) — pinned here, alone +IPERF_SRV_CORE=3 # iperf3 server in NS_A (decrypted-side sink) +TX_CORES="6,7,8" # yipd_A's peer (sender encrypt path) +IPERF_CLI_CORES="9,10,11" # iperf3 client (traffic source) + +TMPDIR_TEST="$(mktemp -d /tmp/yipd-cpuregime.XXXXXX)" +NS_A="yipA"; NS_B="yipB" +VETH_A="vethA"; VETH_B="vethB" +VETH_A_IP="10.0.0.1"; VETH_B_IP="10.0.0.2" +TUN_A_IP="10.9.0.1"; TUN_B_IP="10.9.0.2" +PORT_A="51820"; PORT_B="51821" +TUN_DEV="yip0" +TUN_MTU=1380 +DUR=8 # seconds per iperf run +UDP_RATE="6G" # UDP blast target (well above any plausible plateau) +TCP_STREAMS=8 + +PID_A=""; PID_B=""; IPERF_SRV_PID="" + +cleanup() { + echo "[cleanup] tearing down" + [ -n "$IPERF_SRV_PID" ] && kill "$IPERF_SRV_PID" 2>/dev/null || true + [ -n "$PID_A" ] && kill "$PID_A" 2>/dev/null || true + [ -n "$PID_B" ] && kill "$PID_B" 2>/dev/null || true + sleep 0.2 + [ -n "$PID_A" ] && kill -9 "$PID_A" 2>/dev/null || true + [ -n "$PID_B" ] && kill -9 "$PID_B" 2>/dev/null || true + ip netns del "$NS_A" 2>/dev/null || true + ip netns del "$NS_B" 2>/dev/null || true + rm -rf "$TMPDIR_TEST" +} +trap cleanup EXIT + +# ── keys + configs ─────────────────────────────────────────────────────────── +GENKEY_A="$("$YIPD" --genkey)"; GENKEY_B="$("$YIPD" --genkey)" +PRIV_A="$(echo "$GENKEY_A" | grep '^private=' | cut -d= -f2)" +PUB_A="$(echo "$GENKEY_A" | grep '^public=' | cut -d= -f2)" +PRIV_B="$(echo "$GENKEY_B" | grep '^private=' | cut -d= -f2)" +PUB_B="$(echo "$GENKEY_B" | grep '^public=' | cut -d= -f2)" + +CFG_A="$TMPDIR_TEST/yipA.conf"; CFG_B="$TMPDIR_TEST/yipB.conf" +cat > "$CFG_A" < "$CFG_B" <"$LOG_A" 2>&1 & +PID_A=$! +ip netns exec "$NS_B" taskset -c "$TX_CORES" "$YIPD" "$CFG_B" >"$LOG_B" 2>&1 & +PID_B=$! + +# ── wait for TUN devices ───────────────────────────────────────────────────── +echo "[wait] TUN devices" +elapsed=0 +while true; do + A_UP=0; B_UP=0 + ip netns exec "$NS_A" ip link show "$TUN_DEV" >/dev/null 2>&1 && A_UP=1 || true + ip netns exec "$NS_B" ip link show "$TUN_DEV" >/dev/null 2>&1 && B_UP=1 || true + [ "$A_UP" -eq 1 ] && [ "$B_UP" -eq 1 ] && break + kill -0 "$PID_A" 2>/dev/null || { echo "[error] yipd_A died"; cat "$LOG_A"; exit 1; } + kill -0 "$PID_B" 2>/dev/null || { echo "[error] yipd_B died"; cat "$LOG_B"; exit 1; } + elapsed=$(awk "BEGIN {print $elapsed + 0.25}") + awk "BEGIN {exit ($elapsed >= 20) ? 0 : 1}" && { echo "[error] TUN timeout"; cat "$LOG_A" "$LOG_B"; exit 1; } + sleep 0.25 +done + +# ── tunnel IPs + MTU ───────────────────────────────────────────────────────── +ip netns exec "$NS_A" ip addr add "${TUN_A_IP}/24" dev "$TUN_DEV" +ip netns exec "$NS_B" ip addr add "${TUN_B_IP}/24" dev "$TUN_DEV" +ip netns exec "$NS_A" ip link set "$TUN_DEV" mtu "$TUN_MTU" up +ip netns exec "$NS_B" ip link set "$TUN_DEV" mtu "$TUN_MTU" up +sleep 0.5 +ip netns exec "$NS_B" ping -c 3 -W 5 "$TUN_A_IP" >/dev/null || { echo "[error] baseline ping failed"; cat "$LOG_A" "$LOG_B"; exit 1; } + +# read total CPU ticks (utime+stime) for a pid +cpu_ticks() { awk '{print $14+$15}' "/proc/$1/stat" 2>/dev/null || echo 0; } + +run_iperf() { # $1=label $2..=iperf client args + local label="$1"; shift + # fresh server for this run + ip netns exec "$NS_A" taskset -c "$IPERF_SRV_CORE" iperf3 -s -1 -p 5201 >/dev/null 2>&1 & + local srv=$! + sleep 0.3 + local t0 c0 out c1 t1 cores gbps loss + t0="$(date +%s.%N)"; c0="$(cpu_ticks "$PID_A")" + out="$(ip netns exec "$NS_B" taskset -c "$IPERF_CLI_CORES" iperf3 -c "$TUN_A_IP" -p 5201 -J "$@" 2>/dev/null || true)" + c1="$(cpu_ticks "$PID_A")"; t1="$(date +%s.%N)" + kill "$srv" 2>/dev/null || true + cores="$(awk "BEGIN {w=$t1-$t0; if (w<=0) w=1; printf \"%.2f\", ($c1-$c0)/100.0/w}")" + gbps="$(echo "$out" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin); e=d['end'] + r=e.get('sum_received') or e.get('sum') + print('%.3f' % (r['bits_per_second']/1e9)) +except Exception: print('ERR') +" 2>/dev/null || echo ERR)" + loss="$(echo "$out" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin); s=d['end'].get('sum',{}) + print('%.1f%%' % s['lost_percent']) if 'lost_percent' in s else print('-') +except Exception: print('-') +" 2>/dev/null || echo -)" + printf "%-6s %-8s %-10s %-10s %-8s\n" "$RTT_LABEL" "$label" "$gbps" "$cores" "$loss" +} + +# ── RTT sweep ──────────────────────────────────────────────────────────────── +echo "" +echo "================================================================" +echo " yip receiver CPU-bound-regime sweep" +echo " data flows NS_B -> NS_A; yipd_A RX pinned to core ${RX_CORE}" +echo " rx_cores = cores busy on the pinned RX process (1.0 = saturated)" +echo "================================================================" +printf "%-6s %-8s %-10s %-10s %-8s\n" "RTT" "flow" "Gbps" "rx_cores" "udp_loss" +echo "----------------------------------------------------------------" + +for RTT in 0 1 5 12 24; do + RTT_LABEL="${RTT}ms" + if [ "$RTT" -eq 0 ]; then + ip netns exec "$NS_A" tc qdisc del dev "$VETH_A" root 2>/dev/null || true + ip netns exec "$NS_B" tc qdisc del dev "$VETH_B" root 2>/dev/null || true + else + HALF="$(awk "BEGIN {printf \"%.3f\", $RTT/2.0}")" + ip netns exec "$NS_A" tc qdisc replace dev "$VETH_A" root netem delay "${HALF}ms" + ip netns exec "$NS_B" tc qdisc replace dev "$VETH_B" root netem delay "${HALF}ms" + fi + run_iperf "udp" -u -b "$UDP_RATE" -t "$DUR" -O 1 + run_iperf "tcp" -P "$TCP_STREAMS" -t "$DUR" -O 1 +done + +echo "================================================================" +echo "[verdict] If rx_cores approaches ~1.0 at low RTT (<=5ms) while Gbps" +echo " plateaus, a CPU-bound regime EXISTS -> #4 is justified." +echo " If rx_cores stays well below 1.0 everywhere Gbps plateaus," +echo " the receiver is window/other-bound -> #4 CPU levers are dead." +echo "[done]"