coffee/DeepInfer
1
0
Fork 0
mirror of https://github.com/deepseek-ai/DeepInfer.git synced 2026-10-29 23:37:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge 0e80d7c835ab1f022b1e3949414aebfa5ee1dc14 into 553741557a57df44a46fb3ab72b6aafeaf945337

Browse Source
This commit is contained in:
akauncle 2026-10-26 10:06:07 -07:00 committed by GitHub
commit d854ba4c92
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/benchmark.py
View File

@ -181,7 +181,7 @@ def setup_llm(dp_rank: int) -> LLM:
max_num_batched_tokens=args.prefill_batch_size,
max_model_len=max_model_len,
gpu_memory_utilization=0.95,
lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))),
lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")),
)
return llm

2
examples/offline_inference.py
View File

@ -53,7 +53,7 @@ def main(dp_size: int, dp_rank: int) -> None:
max_model_len=32768,
max_num_seqs=4,
gpu_memory_utilization=0.85,
lookahead_tokens=min(1, int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5"))),
lookahead_tokens=int(os.getenv("DEEPINFER_USE_NEXTN_DECODE", "5")),
)
from _prompts import OFFLINE_INPUT_LEN, OFFLINE_OUTPUT_LEN, lengthen_offline_prompts, prepare_offline_prompts