From 73b8090c814a9cd3ef3dc646fca40d527afa2223 Mon Sep 17 00:00:00 2001 From: loopyt Date: Thu, 22 Oct 2026 10:53:01 +0800 Subject: [PATCH] test(h20): check what the model actually says, in a file rather than on the command line `h20_ask.py` takes its prompts as arguments, and a login shell that is not UTF-8 mangles them into `????` before python ever sees them. The result is a model answering a question nobody asked, reported as a quality problem. The prompts now live in the script. Also records, in the plan, the state of the port as measured today: what runs end to end, the two environment variables it still needs, and the one real blocker found while checking -- the Engram gather is not CUDA-graph capture safe, so the default (attention-graph) configuration fails to warm up on eight ranks. --- H20-TRITON-PLAN.md | 36 +++++++++++++++++ scripts/h20_state_check.py | 79 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 scripts/h20_state_check.py diff --git a/H20-TRITON-PLAN.md b/H20-TRITON-PLAN.md index 41df047..e0509f3 100644 --- a/H20-TRITON-PLAN.md +++ b/H20-TRITON-PLAN.md @@ -4,6 +4,42 @@ 分支：`h20-sm90-triton`，基于 `h20-sm90-support`（PR #180）。 +## 零、现状（2026-10-22 09:30 实测） + +**能说人话。** 8×H20、`DEEPINFER_OPS_BACKEND=triton`、MTP 默认开（`nextn_predict=3`）： + +``` +=== prompt: '问题：中国的首都是哪里？\n答案：' +=== continuation: '北京。\n问题：中国的首都是哪里？…' # 贪心 64 token 后开始复读提示，正常退化 + +=== prompt: '请用两句话解释什么是注意力机制。\n答：' +=== continuation: '注意力机制是一种让模型在处理信息时，能够“聚焦”于相关部分而“忽略”无关部分的 +技术。它通过计算查询（Query）与键（Key）的相似度来分配权重，再根据权重对值（Value）进行加权求和， +从而得到最终的输出。<｜end▁of▁sentence｜>…' +``` + +8 个 DP rank 逐 token 相同，回退提示 5 条（MTP 那 4 个 + 采样器 verify 变体），0 报错。 + +**但「直接这个状态」= 默认配置跑不起来**，必须两个 env（两个都不是 Triton 引入的）： + +| env | 为什么必须 | +|---|---| +| `EP_DISABLE_GIN=1` | 容器没挂 `/dev/infiniband`（host 上有 4 个 uverbs），DeepEP 查 `props.ginType` 得 NONE 就 assert。SM90 单机本来走 scale-up，不需要 GIN。 | +| `DEEPINFER_ENFORCE_EAGER=1` | 见下面 P0：CUDA graph capture 里 engram 本地 gather 会 D2H。 | + +### 已知问题清单（按优先级） + +| # | 问题 | 状态 | 影响 | +|---|---|---|---| +| P0 | **CUDA graph capture 挂掉**：`engram_local.py:108 torch.bincount(owner)`（owner 在 GPU）在 capture 里触发 CPU↔CUDA 拷贝。且 `_exchange_sizes` 用 `all_to_all_single` + 动态 split sizes，本身不可 capture。 | 未修，用 eager 绕过 | 默认配置（graph on）warmup 直接失败，8 rank 全挂 | +| P1 | MTP 4 个 kernel 未移植：`remap_draft_topk` / `get_accept_result` / `finalize_verification` / `get_fused_dflash_get_input` | 未做 | MTP 路径仍有 TileLang 依赖，5 条回退 | +| P1 | 采样器 rejection-sampling 变体（MTP verify）未实现 | 显式委托 cuda | 同上 | +| P2 | 导入期报 `lacks 20 kernel(s)`：这些是**可选路径**（structure_output 引导解码、image_embed_ops 多模态、prepare_cp_mirror CP>1、schedule_mtp_varlen、state_cache、candidate_blocks、sum_smaller_probs、dflash_copy、outlier_detection 等），当前配置不执行，但一旦打开就回退 | 未做 | 功能可用（回退），但「技术栈全 Triton」不成立 | +| P3 | 窄 cut 采样器 token 差 ~9~13%（top_k=128 / top_p=0.5 之类）；引擎配置（top_p=0.99）100% | 已定量解释，不修 | 只影响极端参数，分布本身差 1e-5 | +| P3 | `token_logprobs` 恒 0 | 未查 | 与 Triton 无关，历史 TileLang 日志同样为 0；影响 logprob 类接口 | +| P4 | 性能没测 | 未做 | 没有任何 Triton vs TileLang 吞吐数据 | +| — | 2.7% logits 漂移 | SM90 forced per-128 + fp32 scale 的量化地板 | 非 Triton 问题，答案稳健性已验（margin 1.638） | + ## 一、范围是怎么定下来的 不靠读代码猜，而是实测。`ops.py` 里加了一个 `DEEPINFER_TRACE_KERNELS=` 探针 diff --git a/scripts/h20_state_check.py b/scripts/h20_state_check.py new file mode 100644 index 0000000..a77aa60 --- /dev/null +++ b/scripts/h20_state_check.py @@ -0,0 +1,79 @@ +"""End-to-end coherence check: a few natural-language prompts, printed as decoded text. + +`h20_ask.py` takes prompts on the command line, which a non-UTF-8 login shell mangles into +`????`. The prompts therefore live in this file, so the text the model actually sees is the +text written here. + + python scripts/h20_state_check.py --max_tokens 64 # greedy + python scripts/h20_state_check.py --temperature 0.99 # real sampling (sample path) +""" + +import argparse +import os + +from _config import DEFAULT_MODEL_NAME, create_llm, setup_common_env +from _launch import launch_dp + +from deepinfer.sampling_params import SamplingParams + +PROMPTS = [ + "问题：中国的首都是哪里？\n答案：", + "请用两句话解释什么是注意力机制。\n答：", + "写一句描写秋天的诗。\n", +] + +parser = argparse.ArgumentParser() +parser.add_argument("--max_tokens", type=int, default=64) +parser.add_argument("--temperature", type=float, default=0.0) +parser.add_argument("--ep", type=int, default=8) +parser.add_argument("--cp", type=int, default=1) +parser.add_argument("--model_name", default=DEFAULT_MODEL_NAME) +args = parser.parse_args() + +ep_size, cp_size = args.ep, args.cp +dp_size = max(ep_size // cp_size, 1) +world_size = dp_size * cp_size +assert world_size % ep_size == 0 + + +def main(dp_size: int, dp_rank: int) -> None: + setup_common_env( + model_name=args.model_name, + sampler_force_topk=4096, + set_override_experts=False, + enable_eplb=False, + ) + os.environ.setdefault("DEEPINFER_BATCH_INVARIANT", "1") + + llm = create_llm( + args.model_name, + dp_rank, + ep_size=ep_size, + cp_size=cp_size, + env={"DEEPINFER_LOGGING_INTERVAL_STEP": "10"}, + data_parallel_size=dp_size, + max_num_batched_tokens=4096, + max_model_len=4096, + max_num_seqs=max(1, len(PROMPTS)), + gpu_memory_utilization=0.85, + lookahead_tokens=0, + ) + + sampling_params = [ + SamplingParams(temperature=args.temperature, max_tokens=args.max_tokens, ignore_eos=True, seq_seed=0) for _ in PROMPTS + ] + outputs = llm.generate(prompts=PROMPTS, sampling_params=sampling_params, use_tqdm=False) + for i, output in enumerate(outputs): + assert output, f"sample {i}: no generated tokens" + tokens = [x.token_id for x in output] + text = llm.llm_engine.tokenizer.decode(tokens) + print(f"\n=== DP {dp_rank} sample {i} temp={args.temperature}") + print(f"=== prompt: {PROMPTS[i]!r}") + print(f"=== token ids ({len(tokens)}): {tokens}") + print(f"=== continuation: {text!r}", flush=True) + + llm.close() + + +if __name__ == "__main__": + launch_dp(main, dp_size)