From 0e80d7c835ab1f022b1e3949414aebfa5ee1dc14 Mon Sep 17 00:00:00 2001 From: uncle <3097014910@qq.com> Date: Wed, 21 Oct 2026 16:58:12 +0800 Subject: [PATCH] fix(examples): honor DEEPINFER_USE_NEXTN_DECODE draft length offline_inference.py and benchmark.py wrap the env value in min(1, ...), which caps lookahead_tokens at 1 for any configured value above zero. The README documents DEEPINFER_USE_NEXTN_DECODE as controlling the maximum draft length (default: 5), so pass the configured value through. Co-Authored-By: Claude Code --- examples/benchmark.py | 2 +- examples/offline_inference.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmark.py b/examples/benchmark.py index 331b1b0..3883849 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -181,7 +181,7 @@ def setup_llm(dp_rank: int) -> LLM: max_num_batched_tokens=args.prefill_batch_size, max_model_len=max_model_len, gpu_memory_utilization=0.95, - lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))), + lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")), ) return llm diff --git a/examples/offline_inference.py b/examples/offline_inference.py index 698ccea..ca64a91 100644 --- a/examples/offline_inference.py +++ b/examples/offline_inference.py @@ -53,7 +53,7 @@ def main(dp_size: int, dp_rank: int) -> None: max_model_len=32768, max_num_seqs=4, gpu_memory_utilization=0.85, - lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))), + lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")), ) from _prompts import OFFLINE_INPUT_LEN, OFFLINE_OUTPUT_LEN, lengthen_offline_prompts, prepare_offline_prompts