test(h20): report an end-to-end run the way it is actually judged

The raw log prints one block per rank, so the two things that decide whether a run passed
-- do all eight ranks agree on the token ids, and does the fallback list match the kernels
we know are unported -- are both buried in repetition. This extracts them: a count per
distinct token sequence (eight means every rank agreed), one continuation per prompt, the
kernels still served by the default backend, and whether a traceback occurred.

Splitting "hard failures" out from the stats logger's output matters on torch 2.11, where
`ProcessGroupNCCL.memory_stats` does not exist and the logger says so without the run being
affected.
This commit is contained in:
loopyt 2026-10-22 12:34:15 +08:00
parent 28015cba81
commit 81adc36599

47
scripts/h20_e2e_report.sh Executable file
View File

@ -0,0 +1,47 @@
#!/bin/bash
# Run one end-to-end pass and report only what the pass is judged on.
#
# A successful run is not just "some text came out": it is every data-parallel rank agreeing
# on the token ids, and the fallback list matching what we already know is unported. Both are
# easy to skim past in the raw log, which prints one block per rank, so they are extracted here.
#
# bash scripts/h20_e2e_report.sh
# EP=8 MAX_TOKENS=64 bash scripts/h20_e2e_report.sh
#
# Env: EP (default 8), MAX_TOKENS (default 32), LOG (default e2e.log). Expects
# DEEPINFER_OPS_BACKEND=triton and DEEPINFER_HF_MODEL_DIR to be set, and the cwd to be the
# repo root with its dependencies importable.
set -uo pipefail
EP="${EP:-8}"
MAX_TOKENS="${MAX_TOKENS:-32}"
LOG="${LOG:-e2e.log}"
python scripts/h20_ask.py --ep "$EP" --max_tokens "$MAX_TOKENS" >"$LOG" 2>&1
echo "exit=$?"
echo
echo "=== token ids: distinct sequence and how many ranks produced it"
# `uniq -c` is the point: a sequence appearing once per rank means every rank agreed.
grep -o "=== token ids ([0-9]*): \[[^]]*\]" "$LOG" | sed 's/=== token ids ([0-9]*): //' | sort | uniq -c
echo
echo "=== continuation, one per distinct prompt"
grep -o "=== continuation: '.*'" "$LOG" | sort -u
echo
echo "=== kernels served by the default backend"
grep -o "\[ops\] kernel '[a-z_0-9]*' used from '[a-z]*'" "$LOG" | sed "s/.*kernel '//;s/' used from.*//" | sort -u
grep -o "backend '[a-z]*' lacks [0-9]* kernel(s)" "$LOG" | sort -u
grep -q "rejection sampling" "$LOG" && echo "fused_sample: rejection-sampling variant (MTP verify)"
echo
echo "=== hard failures"
# Tracebacks specifically: the stats logger mentions "AttributeError" without the run failing.
if grep -q "Traceback" "$LOG"; then
echo "TRACEBACK"
grep -A 5 "Traceback" "$LOG" | head -20
else
echo "none"
fi