feat: initial build of kernel tuning and eBPF observability toolkit

.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+# CI for linux-kernel-optimization.
+#
+# Two jobs:
+#   1. shellcheck - lint every Bash script in the repository.
+#   2. ebpf-compile - build the CO-RE BPF objects with clang to catch type and
+#      CO-RE errors early. Loading/attaching is intentionally NOT attempted:
+#      it requires privileges and a matching kernel, which CI does not provide.
+name: ci
+
+on:
+  push:
+    branches: [main, 'feat/**', 'fix/**']
+  pull_request:
+    branches: [main, 'feat/**', 'fix/**']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  shellcheck:
+    name: shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Lint all shell scripts
+        run: |
+          set -euo pipefail
+          mapfile -t scripts < <(git ls-files '*.sh')
+          if [ "${#scripts[@]}" -eq 0 ]; then
+            echo "No shell scripts found." >&2
+            exit 1
+          fi
+          printf 'Linting %d script(s)\n' "${#scripts[@]}"
+          shellcheck --version
+          # severity=warning: advisory style/info notes (for example SC1091 for
+          # dynamically computed source paths) must not fail the build, but real
+          # warnings and errors do.
+          shellcheck --severity=warning --external-sources "${scripts[@]}"
+
+  ebpf-compile:
+    name: eBPF compile check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install build toolchain
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y clang llvm libelf-dev libbpf-dev build-essential
+
+          # Ubuntu ships no standalone 'bpftool' package. The binary comes from
+          # the per-kernel linux-tools packages, and the 'linux-tools-common'
+          # wrapper only works when the tools package matching the running kernel
+          # is installed - which is often not the case on hosted runners. Try the
+          # plausible sources, then locate the real binary and put it on PATH.
+          for pkg in \
+              "linux-tools-$(uname -r)" \
+              linux-tools-azure \
+              linux-tools-generic \
+              linux-tools-common; do
+            sudo apt-get install -y --no-install-recommends "${pkg}" \
+              || echo "note: ${pkg} is not installable on this runner"
+          done
+
+          if ! bpftool version >/dev/null 2>&1; then
+            real="$(find /usr/lib/linux-tools -type f -name bpftool 2>/dev/null | head -n1 || true)"
+            if [ -n "${real}" ]; then
+              echo "using ${real} as bpftool"
+              sudo ln -sf "${real}" /usr/local/bin/bpftool
+            fi
+          fi
+
+          command -v bpftool
+          bpftool version
+
+      - name: Show toolchain versions
+        run: |
+          clang --version
+
+      - name: Compile tcp_retrans BPF object
+        working-directory: ebpf/tcp_retrans
+        run: make check
+
+      - name: Compile biolatency BPF object
+        working-directory: ebpf/biolatency
+        run: make check

README.md

@@ -1,2 +1,116 @@
 # linux-kernel-optimization
-Linux kernel optimization and performance tuning experiments with eBPF and system observability.
+
+Tooling for measuring and tuning Linux kernel behaviour: CO-RE eBPF tracers for
+TCP retransmissions and block I/O latency, commented sysctl tuning profiles with
+a safe apply/revert script, and a repeatable benchmarking methodology.
+
+> **No performance numbers live in this repository.** Any claim about a tuning
+> change must be produced on your own hardware with the scripts in `bench/` and
+> reported using the rules in [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md).
+
+## Layout
+
+```
+ebpf/
+  tcp_retrans/     Trace TCP retransmits, aggregated per destination address
+  biolatency/      Block I/O latency histogram (log2 microseconds)
+tuning/
+  network-latency.conf   sysctl profile for latency-sensitive workloads
+  high-throughput.conf   sysctl profile for bulk/throughput workloads
+  apply-profile.sh       Apply a profile, backing up prior values; --revert
+bench/
+  common.sh              Shared helpers (run dirs, metadata, logging)
+  run-iperf3.sh          Capture iperf3 network measurements
+  run-fio.sh             Capture fio block I/O measurements
+  run-perf-stat.sh       Capture perf stat counters for a command
+systemd/
+  tuning.service         Apply the selected profile at boot
+docs/
+  METHODOLOGY.md         Warmup, repeats, and p99 reporting rules
+.github/workflows/ci.yml shellcheck + eBPF compile check
+```
+
+## Requirements
+
+- Linux with BTF (`/sys/kernel/btf/vmlinux`) for CO-RE.
+- Kernel 5.8+ recommended.
+- `clang`, `llvm`, `bpftool`, `libbpf`, `libelf`, `zlib` for building the eBPF
+  programs and their userspace loaders.
+- `root` (or `CAP_BPF` + `CAP_PERFMON`) to load and attach BPF programs.
+- For benchmarks: `iperf3`, `fio`, `perf` (linux-tools), `sysstat` (optional).
+
+## Observability
+
+Two CO-RE tracers are provided. Both use kprobes rather than tracepoints: the
+tracepoint context types (`trace_event_raw_*`) are not part of BTF and so are
+absent from `vmlinux.h`.
+
+- `tcp_retrans` attaches to `tcp_retransmit_skb` and counts retransmissions per
+  destination address.
+- `biolatency` attaches to `blk_account_io_start`/`blk_account_io_done` and
+  builds a log2 histogram of block I/O latency in microseconds.
+
+If a kprobe cannot be attached on your kernel (the function may be absent or
+inlined), the loader reports the error from libbpf; run it as root and check
+`/proc/kallsyms` for the symbol.
+
+## Quick start
+
+Build a tracer:
+
+```bash
+cd ebpf/tcp_retrans
+make            # generates vmlinux.h, compiles the .bpf.o and the loader
+sudo ./tcp_retrans -i 2 -n 10
+```
+
+Block I/O latency histogram, optionally filtered to one device:
+
+```bash
+cd ebpf/biolatency
+make
+sudo ./biolatency --device 259:0
+```
+
+Apply a sysctl profile (records the previous values first):
+
+```bash
+sudo tuning/apply-profile.sh --profile tuning/network-latency.conf
+sudo tuning/apply-profile.sh --revert
+```
+
+Run a before/after comparison:
+
+```bash
+bench/run-iperf3.sh --server 10.0.0.5 --phase before
+sudo tuning/apply-profile.sh --profile tuning/network-latency.conf
+bench/run-iperf3.sh --server 10.0.0.5 --phase after
+```
+
+Results are written under `bench/results/<UTC-timestamp>-<tool>-<phase>/`
+together with a `metadata.txt` describing the host and settings used.
+
+## Apply at boot
+
+```bash
+sudo install -m 0644 systemd/tuning.service /etc/systemd/system/tuning.service
+sudo systemctl daemon-reload
+sudo systemctl enable --now tuning.service
+```
+
+Edit the unit (or use `systemctl edit` for a drop-in) to select the profile and
+the checkout path.
+
+## Continuous integration
+
+CI runs [shellcheck](https://www.shellcheck.net/) over every `.sh` file and
+compiles the BPF objects with `make check`. It does **not** load or attach the
+programs: that needs privileges and a matching kernel, so it is left to the
+operator.
+
+## License
+
+Files carry an SPDX identifier in their header (`GPL-2.0 OR BSD-3-Clause` for
+eBPF sources, GPL-2.0 for kernel-loadable objects, and the project default for
+scripts and configuration).
+

bench/common.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# common.sh - Shared helpers for the benchmark scripts in this directory.
+#
+# Sourced (not executed) by run-iperf3.sh, run-fio.sh and run-perf-stat.sh.
+# Provides logging, a run-directory factory, and reproducibility metadata.
+# Never reports results itself: scripts only capture raw tool output.
+
+set -euo pipefail
+
+BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RESULTS_ROOT="${BENCH_DIR}/results"
+
+log() { printf '[%s] %s\n' "$(date -u +%H:%M:%SZ)" "$*" >&2; }
+die() { log "error: $*"; exit 1; }
+
+# require_cmd <name...> -> fail early if a required tool is missing.
+require_cmd() {
+	local missing=()
+	local c
+	for c in "$@"; do
+		command -v "${c}" >/dev/null 2>&1 || missing+=("${c}")
+	done
+	if [[ "${#missing[@]}" -gt 0 ]]; then
+		die "missing required command(s): ${missing[*]}"
+	fi
+}
+
+# new_run_dir <label> -> creates and prints results/<UTC timestamp>-<label>.
+new_run_dir() {
+	local label="${1:-run}"
+	local stamp dir
+	stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+	dir="${RESULTS_ROOT}/${stamp}-${label}"
+	mkdir -p -- "${dir}"
+	printf '%s\n' "${dir}"
+}
+
+# write_metadata <dir> <phase> <notes...> -> records environment for the run.
+write_metadata() {
+	local dir="$1" phase="$2"
+	shift 2
+	local file="${dir}/metadata.txt"
+
+	{
+		printf 'timestamp_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+		printf 'phase=%s\n' "${phase}"
+		printf 'hostname=%s\n' "$(uname -n)"
+		printf 'kernel=%s\n' "$(uname -sr)"
+		printf 'arch=%s\n' "$(uname -m)"
+		printf 'cpu_model=%s\n' \
+			"$(awk -F: '/model name/{print $2; exit}' /proc/cpuinfo 2>/dev/null | sed 's/^ //')"
+		printf 'cpu_count=%s\n' "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo unknown)"
+		printf 'congestion_control=%s\n' \
+			"$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo unknown)"
+		printf 'default_qdisc=%s\n' \
+			"$(sysctl -n net.core.default_qdisc 2>/dev/null || echo unknown)"
+		for note in "$@"; do
+			printf 'note=%s\n' "${note}"
+		done
+	} > "${file}"
+
+	log "wrote metadata to ${file}"
+}
+
+# usage_for <script> <extra-lines...>
+usage_for() {
+	local script="$1"
+	shift
+	printf 'Usage: %s [options]\n\n' "$(basename "${script}")"
+	for line in "$@"; do
+		printf '%s\n' "${line}"
+	done
+}
+
+# tee_capture <file> <command...> -> run command, tee stdout+stderr to <file>.
+tee_capture() {
+	local file="$1"
+	shift
+	log "running: $*"
+	# shellcheck disable=SC2068
+	"$@" 2>&1 | tee -- "${file}"
+	return "${PIPESTATUS[0]}"
+}