mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-10-31 08:46:21 +08:00
3c15952e76
Narrows the SM90 numerical divergence to layer 0's FFN. The sparse MLA attention kernel is not the problem: at layer 0 its output matches the reference with cos=0.99897 / rel_err=4.7e-2, while the full layer output is already at 12.7% and the error compounds to cos=0.865 by layer 11, which finally flips the top-1 token. Tooling - `h20_ops/debug_dump.py`: opt-in capture (`DEEPINFER_DUMP_LAYERS=<dir>`), last position only, reduced over `hc_mult`, written to `.npz` at exit. `DEEPINFER_DUMP_TOKENS=<n>` gates on the real prompt length, because the engine runs a dummy profile forward with a different token count before it allocates the KV cache and that dump would otherwise be mistaken for the real one. - `scripts/ref_layer_dump.py`: the same anchors from the reference model via forward hooks (block output, attention sub-layer output, embedding), plus the final logits and greedy token. - `scripts/compare_layers.py`: cosine similarity, relative L2 error and norm ratio per layer, reporting the first layer above a tolerance. Two traps this had to fix before the numbers meant anything - `decoder_layer` returns the *sub-layer* output; the residual stream lives in `residual` (`HCLazyPostData`, hc_mult=4). Comparing without merging them through `hc_post` produced a bogus cos of 0.02 at every layer. - `DEEPINFER_DEBUG_NAN` and the dump both need a host copy inside the forward, so both take `allow_cuda_sync()` rather than fighting the engine's sync guard. Also documents the reproduction case and the findings in H20-DEBUG-PLAN.md.
86 lines
3.3 KiB
Python
86 lines
3.3 KiB
Python
"""Diff per-layer hidden states between DeepInfer and the reference implementation.
|
|
|
|
Both dumps keep only the final position of the prompt, reduced over the `hc_mult` copies, so
|
|
the comparison is one vector per layer. Reports cosine similarity, relative L2 error and the
|
|
ratio of norms, which together distinguish "slightly off" from "different direction".
|
|
|
|
python scripts/compare_layers.py --ref ref_layers.npz --mine deepinfer_layers.npz
|
|
"""
|
|
|
|
import argparse
|
|
|
|
import numpy as np
|
|
|
|
|
|
def rel_err(a: np.ndarray, b: np.ndarray) -> float:
|
|
return float(np.linalg.norm(a - b) / (np.linalg.norm(b) + 1e-12))
|
|
|
|
|
|
def cosine(a: np.ndarray, b: np.ndarray) -> float:
|
|
na, nb = np.linalg.norm(a), np.linalg.norm(b)
|
|
if na == 0 or nb == 0:
|
|
return float("nan")
|
|
return float(np.dot(a, b) / (na * nb))
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--ref", required=True)
|
|
parser.add_argument("--mine", required=True)
|
|
parser.add_argument("--tol", type=float, default=1e-2, help="first layer above this rel_err is the culprit")
|
|
args = parser.parse_args()
|
|
|
|
ref = np.load(args.ref)
|
|
mine = np.load(args.mine)
|
|
|
|
layers = sorted(int(k[5:]) for k in ref.files if k.startswith("layer") and k[5:].isdigit())
|
|
mine_layers = {int(k[5:]) for k in mine.files if k.startswith("layer") and k[5:].isdigit()}
|
|
missing = [i for i in layers if i not in mine_layers]
|
|
if missing:
|
|
print(f"WARNING: no dump from DeepInfer for layers {missing}")
|
|
|
|
for key in ("embed", "layer0_attn", "layer1_attn"):
|
|
if key in ref.files and key in mine.files:
|
|
r, m = ref[key].astype(np.float64), mine[key].astype(np.float64)
|
|
if r.shape == m.shape:
|
|
print(
|
|
f"{key:>12} cos={cosine(m, r):9.5f} rel_err={rel_err(m, r):9.2e}"
|
|
f" |mine|/|ref|={np.linalg.norm(m) / (np.linalg.norm(r) + 1e-12):8.4f}"
|
|
)
|
|
print()
|
|
print(f"{'layer':>5} {'cos':>9} {'rel_err':>9} {'|mine|/|ref|':>12}")
|
|
first_bad = None
|
|
for i in layers:
|
|
if i not in mine_layers:
|
|
continue
|
|
r, m = ref[f"layer{i}"].astype(np.float64), mine[f"layer{i}"].astype(np.float64)
|
|
if r.shape != m.shape:
|
|
print(f"{i:>5} shape mismatch ref={r.shape} mine={m.shape}")
|
|
continue
|
|
err, cos = rel_err(m, r), cosine(m, r)
|
|
ratio = float(np.linalg.norm(m) / (np.linalg.norm(r) + 1e-12))
|
|
flag = ""
|
|
if first_bad is None and err > args.tol:
|
|
first_bad = i
|
|
flag = " <-- first > tol"
|
|
print(f"{i:>5} {cos:9.5f} {err:9.2e} {ratio:12.4f}{flag}")
|
|
|
|
ref_tok = int(ref["next_token"][0]) if "next_token" in ref.files else None
|
|
print(f"\nreference next token: {ref_tok}")
|
|
|
|
if "final_row_logits" in ref.files:
|
|
rl = ref["final_row_logits"].astype(np.float64)
|
|
top = np.argsort(-rl)[:5]
|
|
print("reference top-5 tokens:", [int(x) for x in top], "logits:", [round(float(rl[x]), 3) for x in top])
|
|
# DeepInfer's logits are not dumped; derive its choice from the harness output instead.
|
|
|
|
print()
|
|
if first_bad is None:
|
|
print(f"RESULT: every dumped layer is within rel_err <= {args.tol}; divergence is downstream")
|
|
else:
|
|
print(f"RESULT: first layer above rel_err {args.tol} is layer {first_bad}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|