diff --git a/deepinfer/environ.py b/deepinfer/environ.py index 30f352a..3b71376 100644 --- a/deepinfer/environ.py +++ b/deepinfer/environ.py @@ -123,8 +123,9 @@ class envs: # DEEPINFER_MAX_BACKTRACKING_TOKENS >= bucket (the SWA / compressor rings must hold a whole chunk). DEEPINFER_NEURON_PREFILL_BUCKET = EnvVar.int(default=0) # Neuron only: routed rows per local expert in a prefill chunk's MoE (0 = dense over all gathered rows). An expert - # with more routed rows reruns the layer's MoE densely. - DEEPINFER_NEURON_MOE_CAPACITY = EnvVar.int(default=1024) + # with more routed rows reruns the layer's MoE densely. Off by default: measured slower (108 vs 58 ms a layer for + # a 128-token chunk at EP=64, and every layer overflowed at 1024), issue #26. + DEEPINFER_NEURON_MOE_CAPACITY = EnvVar.int(default=0) # Neuron only: run the sampler's process_sampling as one compiled graph instead of ~1900 eager ops a step. DEEPINFER_NEURON_COMPILE_SAMPLER = EnvVar.bool(default=True) # Neuron only: compile the decode step with fullgraph=True (raise on the first graph break).