From 83be1fa44c34231911cfc223b9212d6364254462 Mon Sep 17 00:00:00 2001 From: Jiejing <113606+kzjeef@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:35:59 +0000 Subject: [PATCH] serve: drop the 32 redundant expert slots when EPLB is off Without rearrangement they are fixed copies of experts 0-31 and only add expert weight per rank. Decode SPS +4.8% / +3.9% at 4 / 16 requests per rank on GB300 (examples/benchmark.py, 128K context), matching whn09's B200 numbers; 6 GiB less model memory per GPU. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YNgkNrJL5oR1pZqVYzZMHW --- serve/README.md | 4 +++- serve/openai_server.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/serve/README.md b/serve/README.md index 20b2f1f..3a7a33a 100644 --- a/serve/README.md +++ b/serve/README.md @@ -32,7 +32,9 @@ per rank). The cache requires decoder bounded replay (`--yoco-keep-tokens > 0`, EPLB expert rearrangement is off by default (`--eplb-interval 0`). Every rearrangement stalls all ranks for about 0.3 s, and with `--eplb-after-prefill 1` it fires after nearly every step under multi-turn traffic; enable it with -`--eplb-interval 30` only for decode-heavy workloads whose expert load is skewed. +`--eplb-interval 30` only for decode-heavy workloads whose expert load is skewed. With EPLB off the server also drops +the 32 redundant expert slots (`DEEPINFER_OVERRIDE_NUM_EXTRA_EXPERTS=0`): without rearrangement they are fixed copies +of experts 0-31, and removing them speeds decode up by 4-5%. ## Prompt tokenization diff --git a/serve/openai_server.py b/serve/openai_server.py index 772390d..ca6c557 100644 --- a/serve/openai_server.py +++ b/serve/openai_server.py @@ -123,6 +123,10 @@ def worker_main(args: argparse.Namespace, dp_rank: int, in_q: mp.Queue, out_q: m # 128-boundaries behind the end of a prompt (see HostPrefixCache.state_lookback_blocks). os.environ.setdefault("DEEPINFER_PREFILL_STORE_EXTRA_SIZE", "512") os.environ.setdefault("DEEPINFER_PREFILL_ANY_ATTEMPT_INTERVAL", str(args.prefill_any_interval)) + if args.eplb_interval == 0: + # The 32 redundant expert slots only pay off with EPLB rearrangement; without it they are fixed copies of + # experts 0-31 and only add expert weight to read per rank (decode +4-5% without them on B200 and GB300). + os.environ.setdefault("DEEPINFER_OVERRIDE_NUM_EXTRA_EXPERTS", "0") setup_common_env(model_name=args.model, sampler_force_topk=1024, set_override_experts=True, enable_eplb=False) if args.eplb_interval > 0: os.environ["DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL"] = str(args.eplb_interval)