diff --git a/examples/benchmark.py b/examples/benchmark.py index 331b1b0..3883849 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -181,7 +181,7 @@ def setup_llm(dp_rank: int) -> LLM: max_num_batched_tokens=args.prefill_batch_size, max_model_len=max_model_len, gpu_memory_utilization=0.95, - lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))), + lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")), ) return llm diff --git a/examples/offline_inference.py b/examples/offline_inference.py index 698ccea..ca64a91 100644 --- a/examples/offline_inference.py +++ b/examples/offline_inference.py @@ -53,7 +53,7 @@ def main(dp_size: int, dp_rank: int) -> None: max_model_len=32768, max_num_seqs=4, gpu_memory_utilization=0.85, - lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))), + lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")), ) from _prompts import OFFLINE_INPUT_LEN, OFFLINE_OUTPUT_LEN, lengthen_offline_prompts, prepare_offline_prompts