From 5a026d3aad67a937bac5440aa65990647693e8f6 Mon Sep 17 00:00:00 2001 From: whn09 Date: Tue, 27 Oct 2026 04:56:47 +0800 Subject: [PATCH] Map media-free requests inline instead of through the multimodal worker (DEEPINFER_MULTI_MODAL_INLINE_TEXT=1) A text request to the VL model went through the multimodal worker's queue and became schedulable only at a later engine step's fetch(): on Neuron two ~0.36 s idle placeholder steps before every text prefill. vllm-sonnet 512 (prefill bucket 512): TTFT 2.42 -> 1.67 s, tokens unchanged. bench_neuron: BENCH_TRACE_STEPS. Co-Authored-By: Claude Opus 5.5 --- deepinfer/environ.py | 3 +++ deepinfer/ve/multimodal_processor.py | 13 +++++++++++++ examples/neuron/bench_neuron.py | 2 ++ 3 files changed, 18 insertions(+) diff --git a/deepinfer/environ.py b/deepinfer/environ.py index 54b68bf..383334f 100644 --- a/deepinfer/environ.py +++ b/deepinfer/environ.py @@ -258,6 +258,9 @@ class envs: DEEPINFER_USE_MULTIMODAL_POOL = EnvVar.bool() DEEPINFER_MULTIMODAL_POOL_SIZE_GB = EnvVar.int(default=8) DEEPINFER_MULTI_MODAL_BUFFER_SIZE = EnvVar.int(default=512) + # Map requests without media inline at add time instead of through the multimodal worker (no IPC round trip, + # schedulable in the same step). + DEEPINFER_MULTI_MODAL_INLINE_TEXT = EnvVar.bool(default=True) DEEPINFER_MULTI_MODAL_WORKER_TIMEOUT = EnvVar.float(default=1800.0) diff --git a/deepinfer/ve/multimodal_processor.py b/deepinfer/ve/multimodal_processor.py index 05e0077..a0b1e0c 100644 --- a/deepinfer/ve/multimodal_processor.py +++ b/deepinfer/ve/multimodal_processor.py @@ -254,6 +254,19 @@ class MultimodalProcessor: # 计算缓存图片集合，剥离 binary，减少 IPC 传输量 self._compute_and_strip_cached_images(multi_modal_data, num_cached_tokens) + if envs.DEEPINFER_MULTI_MODAL_INLINE_TEXT and not any( + isinstance(v, (list, tuple)) and len(v) for k, v in multi_modal_data.items() if not k.startswith("_") + ): + # No media (a text request to the VL model): the mapping is trivial, so do it here and make the request + # schedulable now. Through the worker it waits for a queue round trip and the next engine step's fetch(): + # on Neuron two ~0.36 s idle steps before every text prefill (issue #29). + # In place, as fetch() does: the Sequence holds this dict object. + mapped = self.multi_modal_mapper(dict(multi_modal_data)) + multi_modal_data.clear() + multi_modal_data.update(mapped) + self.multi_modal_dict[id] = multi_modal_data + self.scheduler.set_schedulable(id) + return self.multi_modal_dict[id] = multi_modal_data if not self._pending: # Starting a fresh batch after idle: reset the progress clock diff --git a/examples/neuron/bench_neuron.py b/examples/neuron/bench_neuron.py index d426336..b874675 100644 --- a/examples/neuron/bench_neuron.py +++ b/examples/neuron/bench_neuron.py @@ -137,6 +137,8 @@ def _run_one(llm, prompt: list[int] | None, params) -> tuple[float, float, int] unfinished = engine.has_unfinished_requests() outputs = engine.step() if unfinished else engine.flush_inflight_reqs() now = time.perf_counter() + if os.environ.get("BENCH_TRACE_STEPS") == "1" and prompt is not None and tokens < 8: + print(f"BENCH_STEP t={1e3 * (now - start):8.1f} ms outputs={len(outputs)}", flush=True) steps += len(outputs) > 0 for out in outputs: tokens += 1