DeepInfer/scripts/h20_topk_check.py

"""Check the torch fallback for `deep_select.interface.topk` against a reference.

Covers the parts of the contract the indexer and sampler actually depend on:
`end` truncation (scores past it are uninitialized garbage), OOB padding when a row has
fewer than `topk` valid entries, `output_idx_offset`, in-place writes into a non-contiguous
view, and descending order for `sorted_index=True`.
"""

import torch

from deepinfer.h20_ops.deep_select_sm90 import topk


def ref_topk(scores, k, end=None, offset=None, fill=-1, val_fill=float("-inf")):
    """Reference: consider only [0, end) per row, take k largest, pad, then shift."""
    rows = scores.shape[0]
    idx_rows, val_rows = [], []
    for r in range(rows):
        n = int(end[r]) if end is not None else scores.shape[1]
        row = scores[r, :n]
        kk = min(k, n)
        order = torch.argsort(row, descending=True)[:kk]
        vals = row[order]
        idxs = order.long()
        if offset is not None:
            idxs = idxs + int(offset[r])
        pad = k - kk
        if pad:
            idxs = torch.cat([idxs, torch.full((pad,), fill, dtype=idxs.dtype, device=scores.device)])
            vals = torch.cat([vals, torch.full((pad,), val_fill, dtype=vals.dtype, device=scores.device)])
        idx_rows.append(idxs)
        val_rows.append(vals)
    return torch.stack(val_rows), torch.stack(idx_rows)


def main() -> None:
    dev = "cuda"
    torch.manual_seed(0)
    rows, width, k = 6, 4096, 128

    scores = torch.randn(rows, width, device=dev, dtype=torch.float32)
    # Garbage past `end` that would win if the implementation ignored the bound.
    end = torch.tensor([4096, 4096, 200, 128, 3, 50], dtype=torch.int32, device=dev)
    for r, n in enumerate(end.tolist()):
        scores[r, n:] = 1e6

    vals, idxs = topk(scores, k, sorted=False, end=end, indices_type=torch.int32, return_value=True)
    # Default `idx_oob_fill_value` is INT_MAX, so the reference must pad with the same value.
    rv, ri = ref_topk(scores, k, end=end, fill=2147483647)
    assert idxs.dtype == torch.int32
    bad_rows = [r for r in range(rows) if not torch.equal(vals[r], rv[r]) or not torch.equal(idxs[r].long(), ri[r])]
    print(f"[1] end truncation + OOB pad: rows mismatched = {bad_rows}")
    assert not bad_rows
    # Rows shorter than topk must be padded with the fill value, not with garbage indices.
    short = (idxs[4] == 2147483647).sum().item()
    print(f"    row 4 (end=3) padded slots = {short} (expect {k - 3})")
    assert short == k - 3

    # Descending order for the valid prefix, which is what `sorted_index=True` promises.
    lens = torch.clamp(end.long(), max=k)
    for r in range(rows):
        n = int(lens[r])
        assert torch.equal(vals[r, :n], torch.sort(vals[r, :n], descending=True).values)
    print("[2] descending order holds")

    # In-place write into a non-contiguous view + per-row offset.
    storage = torch.full((rows, k * 2), -7, dtype=torch.int32, device=dev)
    view = storage[:, :k]
    offset = torch.arange(rows, dtype=torch.int32, device=dev) * 1000
    vals2, idxs2 = topk(
        scores, k, end=end, indices_type=torch.int32, output_idx=view, output_idx_offset=offset,
        idx_oob_fill_value=-1, return_value=False,
    )
    assert idxs2.data_ptr() == view.data_ptr()
    assert vals2 is None
    rv2, ri2 = ref_topk(scores, k, end=end, offset=offset, fill=-1)
    print(f"[3] in-place view: match = {torch.equal(view.long(), ri2)}")
    assert torch.equal(view.long(), ri2)
    # The fill value must not be shifted by the offset.
    assert int(view[4].min()) == -1

    # No `end` at all takes the plain path.
    v3, i3 = topk(scores, k, indices_type=torch.int64)
    rv3, ri3 = ref_topk(scores, k)
    print(f"[4] no end: values match = {torch.allclose(v3, rv3)}, indices match = {torch.equal(i3, ri3)}")
    assert torch.allclose(v3, rv3) and torch.equal(i3, ri3)

    # bf16 rows (the candidate-block path feeds bf16 block scores).
    bs = torch.randn(3, 512, device=dev, dtype=torch.bfloat16)
    vb, ib = topk(bs, 16, end=torch.tensor([512, 40, 8], dtype=torch.int32, device=dev), indices_type=torch.int32, return_value=False)
    padded = int((ib[2] == 2147483647).sum())
    print(f"[5] bf16: rows={tuple(ib.shape)} row2 padded={padded} (expect 8)")
    assert padded == 16 - 8

    print("PASS: deep_select torch fallback matches the documented contract")


if __name__ == "__main__":
    main()