diff --git a/deepinfer/neuron_ops/nki_attention.py b/deepinfer/neuron_ops/nki_attention.py index 98221af..1467008 100644 --- a/deepinfer/neuron_ops/nki_attention.py +++ b/deepinfer/neuron_ops/nki_attention.py @@ -255,6 +255,8 @@ def flash_mla_decode( if mask is not None: invalid = invalid | mask scopes.append((cache, torch.where(invalid, torch.zeros_like(idx), idx), invalid)) + # Prompt chunks only: DSpark verification steps (6 rows) on the table path lost acceptance (3.85 -> 1.27 tokens + # a step) and ran slower (bench21), so they stay on the per-row path. if rows == PREFILL_BUCKET and sum(c.shape[0] * c.shape[1] for c, _, _ in scopes) <= _TABLE_MAX_SLOTS: out = _table_attention(q.reshape(rows, h_q, d), scopes, softmax_scale, attn_sink) elif rows == PREFILL_BUCKET and any(idx.shape[1] >= _SHARED_SCOPE_MIN for _, idx, _ in scopes): diff --git a/deepinfer/neuron_ops/regions.py b/deepinfer/neuron_ops/regions.py index d71e424..72c226c 100644 --- a/deepinfer/neuron_ops/regions.py +++ b/deepinfer/neuron_ops/regions.py @@ -119,6 +119,9 @@ def _boundary() -> None: #: Rows of the step being run (set by NeuronGraphRunner.replay): a prefill chunk has more than one. _STEP_ROWS = [1] +#: Steps up to this many rows keep whole-layer graphs (DEEPINFER_NEURON_SPLIT_MIN_ROWS). Whole 6-row layers (DSpark +#: verification) crash neuronx-cc on the full model, so they are split like prompt chunks by default. +_SPLIT_MIN_ROWS = int(os.environ.get("DEEPINFER_NEURON_SPLIT_MIN_ROWS", "1")) def set_step_rows(rows: int) -> None: @@ -131,5 +134,5 @@ def split() -> None: layer, issue #26), while decode's 1-row graphs are faster whole.""" if not torch.compiler.is_compiling(): return - if envs.DEEPINFER_NEURON_REGION_SPLIT or (envs.DEEPINFER_NEURON_REGION_SPLIT_PREFILL and _STEP_ROWS[0] > 1): + if envs.DEEPINFER_NEURON_REGION_SPLIT or (envs.DEEPINFER_NEURON_REGION_SPLIT_PREFILL and _STEP_ROWS[0] > _SPLIT_MIN_ROWS): _boundary() diff --git a/examples/neuron/bench_neuron.py b/examples/neuron/bench_neuron.py index 1dff11a..8a750c8 100644 --- a/examples/neuron/bench_neuron.py +++ b/examples/neuron/bench_neuron.py @@ -82,7 +82,7 @@ def _run_one(llm, prompt: list[int] | None, params) -> tuple[float, float, int] unfinished = engine.has_unfinished_requests() outputs = engine.step() if unfinished else engine.flush_inflight_reqs() now = time.perf_counter() - steps += bool(outputs) + steps += len(outputs) > 0 for out in outputs: tokens += 1 first = first if first is not None else now