From 279334f330e2201cae0ed7789dcf14c6ec306468 Mon Sep 17 00:00:00 2001 From: whn09 Date: Fri, 23 Oct 2026 17:48:39 +0800 Subject: [PATCH] Neuron: capacity MoE off by default (measured slower) (#26) A 128-token chunk at EP=64: forward_capacity 108.5 ms a layer against 58.1 ms for the dense MoE, and with capacity 1024 every layer overflowed and reran densely. The chunk's attention (122 ms a layer) is the larger prefill cost. Co-Authored-By: Claude Opus 5.5 --- deepinfer/environ.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deepinfer/environ.py b/deepinfer/environ.py index 30f352a..3b71376 100644 --- a/deepinfer/environ.py +++ b/deepinfer/environ.py @@ -123,8 +123,9 @@ class envs: # DEEPINFER_MAX_BACKTRACKING_TOKENS >= bucket (the SWA / compressor rings must hold a whole chunk). DEEPINFER_NEURON_PREFILL_BUCKET = EnvVar.int(default=0) # Neuron only: routed rows per local expert in a prefill chunk's MoE (0 = dense over all gathered rows). An expert - # with more routed rows reruns the layer's MoE densely. - DEEPINFER_NEURON_MOE_CAPACITY = EnvVar.int(default=1024) + # with more routed rows reruns the layer's MoE densely. Off by default: measured slower (108 vs 58 ms a layer for + # a 128-token chunk at EP=64, and every layer overflowed at 1024), issue #26. + DEEPINFER_NEURON_MOE_CAPACITY = EnvVar.int(default=0) # Neuron only: run the sampler's process_sampling as one compiled graph instead of ~1900 eager ops a step. DEEPINFER_NEURON_COMPILE_SAMPLER = EnvVar.bool(default=True) # Neuron only: compile the decode step with fullgraph=True (raise on the first graph break).