DeepInfer/deepinfer/model_executor/layers/tilelang_ops/neuron/sampler.py

"""Mirrors ``tilelang_ops/asc/sampler.py`` in pure torch.

Implemented (used by plain sampling, including nextn=0 decode): prepare_verification_metadata_tl,
expand_sampling_params_tl, map_lookup_tl.

MTP/DSpark verification only, out of scope on Neuron basic mode (NEURON_PORT.md "Scope"; the factories keep
the asc signatures and raise): remap_draft_topk_tl, get_accept_result_tl, finalize_verification_tl.
"""

from typing import Any, Literal

import torch

from ..common import DRAFT_POS_HASH_MUL
from ._unsupported import unsupported


def prepare_verification_metadata_tl(num_steps: int) -> Any:
    """kernel(ftp, seqlen_q, row_to_req, local_step, is_verify); rows are packed per request from ftp."""

    def _prepare_verification_metadata(
        ftp: torch.Tensor,
        seqlen_q: torch.Tensor,
        row_to_req: torch.Tensor,
        local_step: torch.Tensor,
        is_verify: torch.Tensor,
    ) -> None:
        total_q = row_to_req.shape[0]
        req_cnt = ftp.shape[0]
        device = ftp.device
        # row -> request without searchsorted/host sync: mark each request start, then prefix-count.
        starts = torch.zeros(total_q + 1, dtype=torch.int32, device=device)
        starts.scatter_add_(0, ftp.to(torch.int64).clamp(0, total_q), torch.ones(req_cnt, dtype=torch.int32, device=device))
        req = starts[:total_q].cumsum(0, dtype=torch.int32) - 1
        req_idx = req.clamp(min=0).to(torch.int64)
        step = torch.arange(total_q, dtype=torch.int32, device=device) - ftp.to(torch.int32)[req_idx]
        row_to_req.copy_(req)
        local_step.copy_(step)
        is_verify.copy_(step + 1 < seqlen_q.to(torch.int32)[req_idx])

    return _prepare_verification_metadata


def expand_sampling_params_tl() -> Any:
    """kernel(temperatures, top_ps, top_ks, sample_positions, seq_seeds, row_to_req, local_step,
    temperatures_q, top_ps_q, top_ks_q, sample_positions_q, seq_seeds_q)."""

    def _expand_sampling_params(
        temperatures: torch.Tensor,
        top_ps: torch.Tensor,
        top_ks: torch.Tensor,
        sample_positions: torch.Tensor,
        seq_seeds: torch.Tensor,
        row_to_req: torch.Tensor,
        local_step: torch.Tensor,
        temperatures_q: torch.Tensor,
        top_ps_q: torch.Tensor,
        top_ks_q: torch.Tensor,
        sample_positions_q: torch.Tensor,
        seq_seeds_q: torch.Tensor,
    ) -> None:
        req = row_to_req.to(torch.int64)
        temperatures_q.copy_(temperatures[req])
        top_ps_q.copy_(top_ps[req])
        top_ks_q.copy_(top_ks[req])
        # int32 arithmetic with wraparound, as in the kernel.
        sample_positions_q.copy_(sample_positions.to(torch.int32)[req] + local_step.to(torch.int32) * DRAFT_POS_HASH_MUL)
        seq_seeds_q.view(torch.int64).copy_(seq_seeds.view(torch.int64)[req])

    return _expand_sampling_params


def map_lookup_tl(
    force_topk: int,
    mode: Literal["by_key", "by_index"] = "by_key",
    value_dtype: torch.dtype = torch.float32,
) -> Any:
    """kernel(key, value, query, out) with out/query [rows, 1].

    by_index: ``value[row, query]`` or the out-of-bounds value (-inf float / -1 int).
    by_key: the value whose key equals the query (keys are unique top-k indices; with duplicate keys this
    returns their max), -inf when missing."""
    assert mode in ("by_key", "by_index")
    assert mode != "by_key" or value_dtype.is_floating_point, "by_key requires floating-point values"
    oob_value = float("-inf") if value_dtype.is_floating_point else -1

    def _map_lookup(
        key: torch.Tensor | None,
        value: torch.Tensor,
        query: torch.Tensor,
        out: torch.Tensor,
    ) -> None:
        q = query[:, 0]
        if mode == "by_index":
            q64 = q.to(torch.int64)
            in_bounds = (q64 >= 0) & (q64 < force_topk)
            got = value[:, :force_topk].gather(1, torch.where(in_bounds, q64, torch.zeros_like(q64)).unsqueeze(1)).squeeze(1)
            result = torch.where(in_bounds, got, torch.full_like(got, oob_value))
        else:
            match = key[:, :force_topk] == q.unsqueeze(1)
            v = value[:, :force_topk]
            result = torch.where(match, v, torch.full_like(v, oob_value)).amax(dim=1)
        out[:, 0].copy_(result)

    return _map_lookup


def remap_draft_topk_tl(force_topk: int, vocab_size: int) -> Any:
    unsupported("remap_draft_topk_tl")


def get_accept_result_tl(has_golden: bool) -> Any:
    """The frontend ``get_accept_result_ref`` (static shapes: a cumsum prefix), with the kernel's argument list."""
    from ..sampler import get_accept_result_ref

    def _accept(raw_accept, golden, draft_pool, read_rows, candidates, is_verify, ftp, row_to_req, token_ids, accept, is_final):  # noqa: ANN001
        draft = draft_pool.index_select(0, read_rows.long())
        tokens, eff, final = get_accept_result_ref(
            raw_accept, golden if has_golden else None, draft, candidates, is_verify, ftp, row_to_req
        )
        token_ids.copy_(tokens)
        accept.copy_(eff)
        is_final.copy_(final)

    return _accept


def finalize_verification_tl(
    has_sum_smaller_probs: bool,
    has_confidence: bool,
    num_steps: int,
    num_stop_tokens: int,
) -> Any:
    """The frontend ``finalize_verification_ref`` with the kernel's argument list, its boolean-mask final-row lookup
    replaced by a static one (exactly one final row per request: ``ftp + last_valid_offset``)."""
    from ..sampler import compute_confidence_error_ref

    def _finalize(
        effective_accept, is_final, effective_token_ids, token_logprobs, ssp, top_lp, top_ids, ftp, seqlen_q,
        row_to_req, is_verify, accept_probs, confidence_pool, read_rows, stop_token_ids,
        last_valid_offset, num_generated_tokens, num_verified_tokens, batch_confidence_error, emitted_stop_token,
        sampled_token_ids, sampled_token_logprobs, sampled_ssp, sampled_top_lp, sampled_top_ids,
        verified_token_ids, verified_token_logprobs, verified_ssp, verified_top_lp, verified_top_ids,
    ):  # noqa: ANN001
        q = effective_accept.shape[0]
        req = row_to_req.long()
        rows = torch.arange(q, dtype=torch.int64, device=ftp.device)
        lvo = torch.zeros(ftp.shape[0], dtype=torch.int64, device=ftp.device)
        lvo.scatter_add_(0, req, effective_accept.to(torch.int64))
        last_valid_offset.copy_(lvo.to(last_valid_offset.dtype))
        final_index = ftp.long() + lvo
        local_step = rows - ftp.long()[req]
        step_index = local_step.clamp(max=num_steps - 1)
        gen = torch.zeros(num_steps, dtype=torch.int32, device=ftp.device)
        ver = torch.zeros(num_steps, dtype=torch.int32, device=ftp.device)
        gen.scatter_add_(0, step_index, effective_accept.to(torch.int32))
        ver.scatter_add_(0, step_index, is_verify.to(torch.int32))
        num_generated_tokens.copy_(gen)
        num_verified_tokens.copy_(ver)
        if has_confidence:
            rates = confidence_pool.index_select(0, read_rows.long())
            batch_confidence_error.copy_(compute_confidence_error_ref(accept_probs, rates, is_verify).reshape(1))
            row_stops = stop_token_ids.index_select(0, req)
            hits = ((effective_token_ids.unsqueeze(1) == row_stops).any(dim=1)) & (effective_accept | is_final)
            counts = torch.zeros(ftp.shape[0], dtype=torch.int32, device=ftp.device)
            counts.scatter_add_(0, req, hits.to(torch.int32))
            emitted_stop_token.copy_(counts > 0)
        else:
            batch_confidence_error.zero_()
        sampled_token_ids.copy_(effective_token_ids.index_select(0, final_index))
        sampled_token_logprobs.copy_(token_logprobs.index_select(0, final_index))
        if has_sum_smaller_probs:
            sampled_ssp.copy_(ssp.index_select(0, final_index))
            verified_ssp.copy_(torch.where(effective_accept, ssp, 0.0))
        if top_lp.shape[-1]:
            sampled_top_lp.copy_(top_lp.index_select(0, final_index))
            sampled_top_ids.copy_(top_ids.index_select(0, final_index))
            verified_top_lp.copy_(torch.where(effective_accept.unsqueeze(-1), top_lp, 0.0))
            verified_top_ids.copy_(torch.where(effective_accept.unsqueeze(-1), top_ids, 0).to(torch.int32))
        verified_token_ids.copy_(torch.where(effective_accept, effective_token_ids, -1).to(torch.int32))
        verified_token_logprobs.copy_(torch.where(effective_accept, token_logprobs, 0.0))

    return _finalize