From 74ba468e76b3ab356a022879d5b5624b69a32170 Mon Sep 17 00:00:00 2001 From: loopyt Date: Wed, 21 Oct 2026 22:37:49 +0800 Subject: [PATCH] fix(debug): move debug_dump out of h20_ops so other platforms can import it The dump hooks live in five platform-neutral files -- `sampler.py`, `mlp.py`, `deepseek.py`, `decoder_layer.py`, `model_runner.py` -- and all of them imported `deepinfer.h20_ops.debug_dump`. That is wrong on every platform except SM90: `h20_ops/__init__.py` calls `extend_import_patches(override=True, ...)` at module scope, so importing it from shared code installs the SM90 import shims globally. On an SM100 machine that replaces `flash_mla.decode_mla_mqa_sparse`, `flash_mla.prefill_attn_varlen_sparse` and `deep_select.interface.topk` with the Hopper implementations, and on NPU it would do the same before the Ascend shims get a chance to matter. `debug_dump` itself has nothing platform-specific in it -- `os`, `pathlib`, `numpy`, `torch` -- so it moves to `deepinfer/debug_dump.py` and the five imports follow. Verified that importing it no longer pulls `h20_ops` into `sys.modules`, and re-ran the end-to-end check on H20: same output token for token, `[17607, 2793, 3139, 666, 6127, 666, 320, 1]`. This bug is already in the open PR, so it needs to land before that PR is worth reviewing further. --- H20-DEBUG-PLAN.md | 29 +++++++++++++++++-- deepinfer/{h20_ops => }/debug_dump.py | 0 deepinfer/model_executor/layers/sampler.py | 2 +- .../model_executor/models/decoder_layer.py | 2 +- deepinfer/model_executor/models/deepseek.py | 2 +- .../model_executor/models/v4_modules/mlp.py | 2 +- deepinfer/worker/model_runner.py | 2 +- 7 files changed, 31 insertions(+), 8 deletions(-) rename deepinfer/{h20_ops => }/debug_dump.py (100%) diff --git a/H20-DEBUG-PLAN.md b/H20-DEBUG-PLAN.md index 8492b64..df65624 100644 --- a/H20-DEBUG-PLAN.md +++ b/H20-DEBUG-PLAN.md @@ -366,12 +366,35 @@ prefill 选中了正确的 token，误差只把 margin 压缩了 12 倍。但端 - `scripts/h20_kv_roundtrip.py`：KV cache 闭环 round-trip（不依赖 ref） - `scripts/h20_index_decode_check.py`：`_decode_p_sum_sm90` 的独立数值对照 -## P5 logprob 异常 +## P5 logprob 异常（已排查，未修，非阻塞） 当前 8 个 token 的 `token_logprobs` 全是 `0.0`（概率=1），不正常。 -- [ ] P5.1 确认是采样器 bug 还是 logits 极端（打印 logits 的 max/median） -- [ ] P5.2 检查 `DEEPINFER_SAMPLER_FORCE_TOPK=4096` 与 logprob 回填的关系 +### 已排除（约 20 分钟静态排查 + 单测） + +| 嫌疑 | 结论 | +|---|---| +| `map_lookup` kernel | 无罪。单测 6 passed，`by_key` + float32 + `force_topk=4096` 全覆盖，与 `map_lookup_ref` 逐元素相等 | +| sampler 全部 kernel | 无罪。`sampler.py -m "not benchmark"` → **74 passed / 0 failed** | +| `SampledInfo.from_reqcnt` 的 `torch.zeros` | 死代码，全仓库无调用点；`update(is_final, candidate)` 也无调用点 | +| sampler→engine 字段传递 | 直通，`llm_engine.py:747` 原样传 `output.token_logprobs` | + +**一个反向线索**：`map_lookup` 里 `out = torch.empty`，键未命中时本应写 `-inf` 而**不是 0**。 +所以 0.0 更像「上层根本没把值带下来」，而不是「查表查空了」。 + +### 剩余候选与判定路径 + +- `process_sampling` 返回的 `logprobs` 本身为 0（`fused_sample` 的 logprob 输出路径； + 现有单测只覆盖 kernel 本身，没有覆盖「整个 forward 的 logprob 输出」） +- `build_request_outputs` 里逐 token 对象的属性回填 + +判定只需跑 1 次整机（~5 min）：在 sampler 里打印 `logprobs.max()` 与 `token_logprobs_cur`。 +sampler 层就是 0 → 定位在 `fused_sample`；sampler 层正常 → 定位在 output 回填。 + +### 为什么留着不修 + +不影响推理正确性，只影响 logprob 字段的可用性（AIME 打分、perplexity 这类评估用途）。 +预估典型 1 h、最坏 2 h（若牵涉 MTP 异步 logprob 回填），与本次 SM90 支持的目标无关。 --- diff --git a/deepinfer/h20_ops/debug_dump.py b/deepinfer/debug_dump.py similarity index 100% rename from deepinfer/h20_ops/debug_dump.py rename to deepinfer/debug_dump.py diff --git a/deepinfer/model_executor/layers/sampler.py b/deepinfer/model_executor/layers/sampler.py index d44026f..d81488a 100644 --- a/deepinfer/model_executor/layers/sampler.py +++ b/deepinfer/model_executor/layers/sampler.py @@ -11,8 +11,8 @@ import torch import torch.nn as nn from deep_select.interface import topk +from deepinfer import debug_dump as _debug_dump from deepinfer import envs -from deepinfer.h20_ops import debug_dump as _debug_dump from deepinfer.logger import init_logger from deepinfer.model_executor.input_metadata import InputMetadata from deepinfer.model_executor.layers.mtp_info_pool import MTPInfoPool diff --git a/deepinfer/model_executor/models/decoder_layer.py b/deepinfer/model_executor/models/decoder_layer.py index 6aa27d4..59213c3 100644 --- a/deepinfer/model_executor/models/decoder_layer.py +++ b/deepinfer/model_executor/models/decoder_layer.py @@ -5,7 +5,7 @@ import torch from torch import nn from deepinfer import AttrDict, envs -from deepinfer.h20_ops import debug_dump as _debug_dump +from deepinfer import debug_dump as _debug_dump from deepinfer.logger import init_logger from deepinfer.model_executor.input_metadata import InputMetadata from deepinfer.model_executor.input_metadata.yoco import YocoMetadataMixin diff --git a/deepinfer/model_executor/models/deepseek.py b/deepinfer/model_executor/models/deepseek.py index 36eb614..4e01ac9 100644 --- a/deepinfer/model_executor/models/deepseek.py +++ b/deepinfer/model_executor/models/deepseek.py @@ -9,8 +9,8 @@ import torch from torch import nn from deepinfer import AttrDict, envs +from deepinfer import debug_dump as _debug_dump from deepinfer.engine.engine_type import EngineType -from deepinfer.h20_ops import debug_dump as _debug_dump from deepinfer.logger import init_logger from deepinfer.model_executor import eplb from deepinfer.model_executor.input_metadata import InputMetadata diff --git a/deepinfer/model_executor/models/v4_modules/mlp.py b/deepinfer/model_executor/models/v4_modules/mlp.py index 2edcc4c..3c8b2e4 100644 --- a/deepinfer/model_executor/models/v4_modules/mlp.py +++ b/deepinfer/model_executor/models/v4_modules/mlp.py @@ -21,8 +21,8 @@ from tile_kernels.quant import swiglu_forward_and_per_token_cast from torch import nn from deepinfer import PLATFORM, AttrDict, envs +from deepinfer import debug_dump as _debug_dump from deepinfer.engine.engine_type import EngineType -from deepinfer.h20_ops import debug_dump as _debug_dump from deepinfer.logger import init_logger from deepinfer.model_executor import eplb from deepinfer.model_executor.input_metadata import InputMetadata diff --git a/deepinfer/worker/model_runner.py b/deepinfer/worker/model_runner.py index e08bf82..f97f67b 100644 --- a/deepinfer/worker/model_runner.py +++ b/deepinfer/worker/model_runner.py @@ -11,11 +11,11 @@ import numpy as np import torch import torch.distributed as dist +from deepinfer import debug_dump as _debug_dump from deepinfer import envs from deepinfer._C.engine import prepare_sampler_inputs, update_prompt_logprobs # pyright: ignore[reportMissingModuleSource] from deepinfer.config import CacheConfig, ModelConfig, ParallelConfig, SchedulerConfig from deepinfer.engine.engine_type import EngineType -from deepinfer.h20_ops import debug_dump as _debug_dump from deepinfer.logger import init_logger from deepinfer.model_executor.input_metadata import InputMetadata from deepinfer.model_executor.layers.mtp_info_pool import get_nextn_pad_token_ids