mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-10-31 00:36:20 +08:00
test(h20): report an end-to-end run the way it is actually judged
The raw log prints one block per rank, so the two things that decide whether a run passed -- do all eight ranks agree on the token ids, and does the fallback list match the kernels we know are unported -- are both buried in repetition. This extracts them: a count per distinct token sequence (eight means every rank agreed), one continuation per prompt, the kernels still served by the default backend, and whether a traceback occurred. Splitting "hard failures" out from the stats logger's output matters on torch 2.11, where `ProcessGroupNCCL.memory_stats` does not exist and the logger says so without the run being affected.
This commit is contained in:
parent
28015cba81
commit
81adc36599
47
scripts/h20_e2e_report.sh
Executable file
47
scripts/h20_e2e_report.sh
Executable file
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Run one end-to-end pass and report only what the pass is judged on.
|
||||
#
|
||||
# A successful run is not just "some text came out": it is every data-parallel rank agreeing
|
||||
# on the token ids, and the fallback list matching what we already know is unported. Both are
|
||||
# easy to skim past in the raw log, which prints one block per rank, so they are extracted here.
|
||||
#
|
||||
# bash scripts/h20_e2e_report.sh
|
||||
# EP=8 MAX_TOKENS=64 bash scripts/h20_e2e_report.sh
|
||||
#
|
||||
# Env: EP (default 8), MAX_TOKENS (default 32), LOG (default e2e.log). Expects
|
||||
# DEEPINFER_OPS_BACKEND=triton and DEEPINFER_HF_MODEL_DIR to be set, and the cwd to be the
|
||||
# repo root with its dependencies importable.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
EP="${EP:-8}"
|
||||
MAX_TOKENS="${MAX_TOKENS:-32}"
|
||||
LOG="${LOG:-e2e.log}"
|
||||
|
||||
python scripts/h20_ask.py --ep "$EP" --max_tokens "$MAX_TOKENS" >"$LOG" 2>&1
|
||||
echo "exit=$?"
|
||||
|
||||
echo
|
||||
echo "=== token ids: distinct sequence and how many ranks produced it"
|
||||
# `uniq -c` is the point: a sequence appearing once per rank means every rank agreed.
|
||||
grep -o "=== token ids ([0-9]*): \[[^]]*\]" "$LOG" | sed 's/=== token ids ([0-9]*): //' | sort | uniq -c
|
||||
|
||||
echo
|
||||
echo "=== continuation, one per distinct prompt"
|
||||
grep -o "=== continuation: '.*'" "$LOG" | sort -u
|
||||
|
||||
echo
|
||||
echo "=== kernels served by the default backend"
|
||||
grep -o "\[ops\] kernel '[a-z_0-9]*' used from '[a-z]*'" "$LOG" | sed "s/.*kernel '//;s/' used from.*//" | sort -u
|
||||
grep -o "backend '[a-z]*' lacks [0-9]* kernel(s)" "$LOG" | sort -u
|
||||
grep -q "rejection sampling" "$LOG" && echo "fused_sample: rejection-sampling variant (MTP verify)"
|
||||
|
||||
echo
|
||||
echo "=== hard failures"
|
||||
# Tracebacks specifically: the stats logger mentions "AttributeError" without the run failing.
|
||||
if grep -q "Traceback" "$LOG"; then
|
||||
echo "TRACEBACK"
|
||||
grep -A 5 "Traceback" "$LOG" | head -20
|
||||
else
|
||||
echo "none"
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user