From 7d897aedb2cc9272d17b6a4e249bd0a77daa516f Mon Sep 17 00:00:00 2001 From: whn09 Date: Fri, 23 Oct 2026 12:42:25 +0800 Subject: [PATCH] Neuron: skip the unused candidate-block source when the filter is vacuous (#26) With runtime_max_model_len <= candidate_topk_blocks * candidate_block_size every consumer already skips the candidate filter, so the source layer's block top-k feeds nothing; below 16384 positions it is not even defined (fewer blocks than candidate_topk_blocks, assert in _topk_candidate_block). Co-Authored-By: Claude Opus 5.5 --- deepinfer/model_executor/models/v4_modules/index_attn.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deepinfer/model_executor/models/v4_modules/index_attn.py b/deepinfer/model_executor/models/v4_modules/index_attn.py index 345b12c..9d4f955 100644 --- a/deepinfer/model_executor/models/v4_modules/index_attn.py +++ b/deepinfer/model_executor/models/v4_modules/index_attn.py @@ -98,6 +98,15 @@ class MQA(nn.Module): candidate_cfg = shared_topology.candidate_source_cfg[candidate_source] if candidate_source is not None else None self.is_candidate_topk_source = shared_topology is not None and shared_topology.is_candidate_source_layer(self.layer_idx) self.use_candidate_topk = candidate_source is not None and not self.is_candidate_topk_source + if ( + PLATFORM.is_neuron + and self.is_candidate_topk_source + and candidate_cfg is not None + and cfg.runtime_max_model_len <= candidate_cfg.candidate_topk_blocks * candidate_cfg.candidate_block_size + ): + # Neuron: every consumer skips the vacuous filter (below), so nothing reads the candidate blocks; and with + # fewer blocks than candidate_topk_blocks the block top-k would not be defined. + self.is_candidate_topk_source = False if ( PLATFORM.is_neuron and self.use_candidate_topk