The Diagnostic Toolkit

# Instructions per cycle plus the main stall counters.
# IPC above 2.0 means the core runs well. Below 0.5 means it stalls.
perf stat -e cycles,instructions,branches,branch-misses,\
cache-references,cache-misses,dTLB-load-misses ./workload

# Top-down analysis sorts every cycle into retiring, bad speculation,
# front-end bound, and back-end bound.
perf stat --topdown -a --per-core sleep 10

# Which source line takes the cache misses.
perf record -e cache-misses:pp -g ./workload
perf report --stdio --sort=srcline

# Which cache line two cores fight over. This is the false sharing tool.
perf c2c record ./workload
perf c2c report --stdio

# Off-CPU time: where a thread blocks rather than where it computes.
perf sched record -- sleep 10
perf sched latency --sort max

# Pressure, not free memory. "some" means at least one task waited.
# "full" means every task waited, and any non-zero value is a stall.
cat /proc/pressure/memory /proc/pressure/io /proc/pressure/cpu

# Per-cgroup memory events. oom_kill and high are the interesting rows.
cat /sys/fs/cgroup/<path>/memory.events
cat /sys/fs/cgroup/<path>/memory.stat

# Page fault rates. High major faults mean reads reaching the device.
perf stat -e page-faults,major-faults,minor-faults -p <pid> sleep 10

# NUMA placement. numa_miss counts allocations off the preferred node.
numastat -c
numactl --hardware

# Transparent huge page state and TLB miss cost.
cat /sys/kernel/mm/transparent_hugepage/enabled
perf stat -e dTLB-load-misses,dtlb_load_misses.walk_active -p <pid> sleep 10

# CPU throttling. nr_throttled rising means the quota is the bottleneck.
cat /sys/fs/cgroup/<path>/cpu.stat

# Involuntary context switches: preemption rather than blocking.
pidstat -w -p <pid> 1

# Run queue latency, per task, in microseconds.
bpftrace -e 'tracepoint:sched:sched_wakeup { @q[args->comm] = hist(nsecs); }'

# Which scheduler class and priority a thread runs at.
chrt -p <tid>
ps -eLo pid,tid,class,rtprio,pri,psr,comm | head -30

# Per-device latency and queue depth. await is the number to read.
iostat -x 1

# Block layer latency histogram, in microseconds.
biolatency-bpfcc -m 10 1

# Which process issues which I/O, with size and latency.
biosnoop-bpfcc

# fsync latency per file, which is commit latency for a database.
ext4slower-bpfcc 10      # or xfsslower, btrfsslower

# Page cache hit ratio.
cachestat-bpfcc 1

# Readahead policy for one file descriptor's device.
cat /sys/block/<dev>/queue/read_ahead_kb

# Every counter that matters. Look for listen queue overflows,
# retransmits, and pruned packets.
netstat -s | grep -iE 'overflow|listen|retrans|prune|collapse'
ss -lti                     # per-socket congestion window, rtt, retrans

# Socket memory and accept queue depth, per listening socket.
ss -ltn

# Round trip time and loss to one peer, without ICMP.
tcping <host> <port>        # or: ss -ti dst <host>

# Which connections use which congestion control.
ss -ti | grep -o 'bbr\|cubic' | sort | uniq -c

# Per-queue interrupt distribution. An uneven spread means one core
# does all the receive work.
cat /proc/interrupts | grep -i eth
mpstat -P ALL 1 | head -20

# Packet drops in the driver and the stack.
ethtool -S <iface> | grep -iE 'drop|err|miss|discard'
nstat -az | grep -i drop

# What a pod actually got, against what it asked for.
kubectl get pod <pod> -o jsonpath='{.spec.containers[*].resources}'
kubectl top pod <pod> --containers

# Throttling and OOM, from inside the node.
crictl inspect <container-id> | jq '.info.runtimeSpec.linux.resources'
cat /sys/fs/cgroup/kubepods.slice/.../cpu.stat

# Why a pod restarted. lastState holds the previous exit reason.
kubectl get pod <pod> -o jsonpath='{.status.containerStatuses[*].lastState}'

# Events, newest last, which is where scheduling and eviction show up.
kubectl get events --sort-by=.lastTimestamp -n <namespace>

# Image pull time, which usually dominates cold start.
kubectl describe pod <pod> | grep -A2 Pulled

# Envoy's own view: cluster health, circuit breaker state, and stats.
kubectl exec <pod> -c istio-proxy -- \
  curl -s localhost:15000/clusters | grep -E 'health_flags|cx_active'
kubectl exec <pod> -c istio-proxy -- \
  curl -s localhost:15000/stats | grep -E 'upstream_rq_pending|overflow'

# The configuration the proxy actually has, not what you applied.
istioctl proxy-config cluster <pod>
istioctl proxy-config route <pod>
istioctl analyze -n <namespace>

# Retry and outlier ejection counters.
kubectl exec <pod> -c istio-proxy -- \
  curl -s localhost:15000/stats | grep -E 'retry|ejections'

-- PostgreSQL: the longest-running transaction, which holds back vacuum.
SELECT pid, now() - xact_start AS age, state, left(query, 80)
  FROM pg_stat_activity
 WHERE xact_start IS NOT NULL
 ORDER BY age DESC LIMIT 10;

-- Table bloat and last autovacuum.
SELECT relname, n_live_tup, n_dead_tup,
       round(100.0 * n_dead_tup / nullif(n_live_tup + n_dead_tup, 0), 1)
         AS dead_pct,
       last_autovacuum
  FROM pg_stat_user_tables
 ORDER BY n_dead_tup DESC LIMIT 20;

-- Buffer cache hit ratio per table. Below 0.99 on a hot table means
-- the working set does not fit.
SELECT relname,
       round(heap_blks_hit::numeric /
             nullif(heap_blks_hit + heap_blks_read, 0), 4) AS hit_ratio
  FROM pg_statio_user_tables
 ORDER BY heap_blks_read DESC LIMIT 20;

-- Slowest statements by total time. Needs pg_stat_statements.
SELECT calls, round(mean_exec_time::numeric, 2) AS mean_ms,
       round(total_exec_time::numeric) AS total_ms, left(query, 70)
  FROM pg_stat_statements
 ORDER BY total_exec_time DESC LIMIT 20;

-- Replication slot lag. A stopped consumer fills the primary's disk.
SELECT slot_name, active,
       pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(),
                                      restart_lsn)) AS retained
  FROM pg_replication_slots;

# Consumer group lag, per partition. Read it in seconds, not records.
kafka-consumer-groups.sh --bootstrap-server <b> --describe --group <g>

# Per-partition rates, to find a skewed key.
kafka-run-class.sh kafka.tools.GetOffsetShell \
  --bootstrap-server <b> --topic <t> --time -1

# Under-replicated and offline partitions. Both are alerts, not metrics.
kafka-topics.sh --bootstrap-server <b> --describe --under-replicated-partitions
kafka-topics.sh --bootstrap-server <b> --describe --unavailable-partitions

# Rebalance rate. A rising count means a storm, not a deployment.
# Metric: kafka.consumer:type=consumer-coordinator-metrics,
#         name=rebalance-rate-per-hour

# The three service level indicators. Utilization is misleading here.
# A bandwidth-bound decode phase shows low SM utilization while saturated.
nvidia-smi dmon -s pucm -d 1
nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu \
           --format=csv -l 1

# Memory bandwidth achieved against peak, which is the real utilization
# for decode.
dcgmi dmon -e 1005,1009,1010 -d 1000

# Server-side metrics that matter, from the vLLM or TensorRT-LLM
# Prometheus endpoint:
#   time_to_first_token_seconds        (prefill plus queueing)
#   time_per_output_token_seconds      (decode, bandwidth bound)
#   num_requests_running, num_requests_waiting
#   gpu_cache_usage_perc               (KV cache pressure)
#   gpu_prefix_cache_hit_rate          (enable this first)
curl -s localhost:8000/metrics | grep -E 'time_to_first|cache_usage|waiting'

# eBPF profiler across every process on a node, about 1 percent CPU.
# Parca agent, or the OpenTelemetry eBPF profiler.
parca-agent --node=$(hostname) --remote-store-address=parca:7070

# One-off flame graph for a single process, no agent needed.
perf record -F 99 -p <pid> -g -- sleep 30
perf script | stackcollapse-perf.pl | flamegraph.pl > out.svg

# Allocation profile, JVM.
jcmd <pid> JFR.start duration=60s filename=/tmp/rec.jfr \
     settings=profile
jfr summary /tmp/rec.jfr

# Allocation and block profile, Go.
curl -o heap.out localhost:6060/debug/pprof/heap
curl -o block.out 'localhost:6060/debug/pprof/block?seconds=30'
go tool pprof -http=: heap.out