diff --git a/examples/neuron/bench_neuron.py b/examples/neuron/bench_neuron.py index 5bcc6ec..96ef1d5 100644 --- a/examples/neuron/bench_neuron.py +++ b/examples/neuron/bench_neuron.py @@ -122,6 +122,8 @@ def main(dp_size: int, dp_rank: int) -> None: _run_one(llm, _random_prompt(lens[0], 10_000 + w + 7919 * dp_rank, vocab) if busy else None, params) results = [] for length in lens: + # One untimed request a cell: a cell's first request pays for its own context bucket's lazy compile. + _run_one(llm, _random_prompt(length, 20_000 + length + 7919 * dp_rank, vocab) if busy else None, params) cell = [] for r in range(args.repeats): got = _run_one(llm, _random_prompt(length, 1000 * length + r + 7919 * dp_rank, vocab) if busy else None, params)