From 81adc36599ce334753ba1c9f935705caa494f03d Mon Sep 17 00:00:00 2001
From: loopyt <zhsong@baai.ac.cn>
Date: Thu, 22 Oct 2026 12:34:15 +0800
Subject: [PATCH] test(h20): report an end-to-end run the way it is actually
 judged

The raw log prints one block per rank, so the two things that decide whether a run passed
-- do all eight ranks agree on the token ids, and does the fallback list match the kernels
we know are unported -- are both buried in repetition. This extracts them: a count per
distinct token sequence (eight means every rank agreed), one continuation per prompt, the
kernels still served by the default backend, and whether a traceback occurred.

Splitting "hard failures" out from the stats logger's output matters on torch 2.11, where
`ProcessGroupNCCL.memory_stats` does not exist and the logger says so without the run being
affected.
---
 scripts/h20_e2e_report.sh | 47 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100755 scripts/h20_e2e_report.sh

diff --git a/scripts/h20_e2e_report.sh b/scripts/h20_e2e_report.sh
new file mode 100755
index 0000000..b839bae
--- /dev/null
+++ b/scripts/h20_e2e_report.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Run one end-to-end pass and report only what the pass is judged on.
+#
+# A successful run is not just "some text came out": it is every data-parallel rank agreeing
+# on the token ids, and the fallback list matching what we already know is unported. Both are
+# easy to skim past in the raw log, which prints one block per rank, so they are extracted here.
+#
+#   bash scripts/h20_e2e_report.sh
+#   EP=8 MAX_TOKENS=64 bash scripts/h20_e2e_report.sh
+#
+# Env: EP (default 8), MAX_TOKENS (default 32), LOG (default e2e.log). Expects
+# DEEPINFER_OPS_BACKEND=triton and DEEPINFER_HF_MODEL_DIR to be set, and the cwd to be the
+# repo root with its dependencies importable.
+
+set -uo pipefail
+
+EP="${EP:-8}"
+MAX_TOKENS="${MAX_TOKENS:-32}"
+LOG="${LOG:-e2e.log}"
+
+python scripts/h20_ask.py --ep "$EP" --max_tokens "$MAX_TOKENS" >"$LOG" 2>&1
+echo "exit=$?"
+
+echo
+echo "=== token ids: distinct sequence and how many ranks produced it"
+# `uniq -c` is the point: a sequence appearing once per rank means every rank agreed.
+grep -o "=== token ids ([0-9]*): \[[^]]*\]" "$LOG" | sed 's/=== token ids ([0-9]*): //' | sort | uniq -c
+
+echo
+echo "=== continuation, one per distinct prompt"
+grep -o "=== continuation: '.*'" "$LOG" | sort -u
+
+echo
+echo "=== kernels served by the default backend"
+grep -o "\[ops\] kernel '[a-z_0-9]*' used from '[a-z]*'" "$LOG" | sed "s/.*kernel '//;s/' used from.*//" | sort -u
+grep -o "backend '[a-z]*' lacks [0-9]* kernel(s)" "$LOG" | sort -u
+grep -q "rejection sampling" "$LOG" && echo "fused_sample: rejection-sampling variant (MTP verify)"
+
+echo
+echo "=== hard failures"
+# Tracebacks specifically: the stats logger mentions "AttributeError" without the run failing.
+if grep -q "Traceback" "$LOG"; then
+    echo "TRACEBACK"
+    grep -A 5 "Traceback" "$LOG" | head -20
+else
+    echo "none"
+fi