From 8b1ed3f26aefbd956178f5a3577931b34874d8fe Mon Sep 17 00:00:00 2001 From: Jiejing <113606+kzjeef@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:37:45 +0000 Subject: [PATCH] serve: document the --prefill-any-interval TTFT / decode-speed trade-off Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YNgkNrJL5oR1pZqVYzZMHW --- serve/README.md | 5 +++++ serve/openai_server.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/serve/README.md b/serve/README.md index 3a7a33a..44ce939 100644 --- a/serve/README.md +++ b/serve/README.md @@ -36,6 +36,11 @@ EPLB expert rearrangement is off by default (`--eplb-interval 0`). Every rearran the 32 redundant expert slots (`DEEPINFER_OVERRIDE_NUM_EXTRA_EXPERTS=0`): without rearrangement they are fixed copies of experts 0-31, and removing them speeds decode up by 4-5%. +`--prefill-any-interval` (default 2) trades TTFT for decode speed. A prompt step pauses decode on every DP rank, and +with 2 almost every arriving request triggers one. That keeps TTFT low; the engine default of 100 adds a ~1.4 s +floor. At high concurrency, 30-100 gives faster decode and more throughput at the cost of TTFT: on B200 AgentX c256, +interval 100 gave 255K vs 211K tok/s/GPU and interactivity p50 119 vs 55, with TTFT p50 2.5 vs 0.7 s. + ## Prompt tokenization Agentic clients resend the whole conversation every turn, and `tokenizer.encode` needs ~1.7 ms per 1K tokens on a diff --git a/serve/openai_server.py b/serve/openai_server.py index ca6c557..b11f94e 100644 --- a/serve/openai_server.py +++ b/serve/openai_server.py @@ -67,7 +67,8 @@ def parse_args() -> argparse.Namespace: type=int, default=2, help="DEEPINFER_PREFILL_ANY_ATTEMPT_INTERVAL: a rank with a waiting prompt gets a prefill step at most this " - "many decode steps later even if other ranks are idle (engine default 100 => ~1.4 s TTFT floor)", + "many decode steps later even if other ranks are idle (engine default 100 => ~1.4 s TTFT floor). A prompt step " + "pauses decode on every rank, so at high concurrency a larger value (30-100) trades TTFT for decode speed", ) p.add_argument( "--eplb-interval",