diff --git a/deepinfer/environ.py b/deepinfer/environ.py index 54b68bf..383334f 100644 --- a/deepinfer/environ.py +++ b/deepinfer/environ.py @@ -258,6 +258,9 @@ class envs: DEEPINFER_USE_MULTIMODAL_POOL = EnvVar.bool() DEEPINFER_MULTIMODAL_POOL_SIZE_GB = EnvVar.int(default=8) DEEPINFER_MULTI_MODAL_BUFFER_SIZE = EnvVar.int(default=512) + # Map requests without media inline at add time instead of through the multimodal worker (no IPC round trip, + # schedulable in the same step). + DEEPINFER_MULTI_MODAL_INLINE_TEXT = EnvVar.bool(default=True) DEEPINFER_MULTI_MODAL_WORKER_TIMEOUT = EnvVar.float(default=1800.0) diff --git a/deepinfer/ve/multimodal_processor.py b/deepinfer/ve/multimodal_processor.py index 05e0077..a0b1e0c 100644 --- a/deepinfer/ve/multimodal_processor.py +++ b/deepinfer/ve/multimodal_processor.py @@ -254,6 +254,19 @@ class MultimodalProcessor: # 计算缓存图片集合,剥离 binary,减少 IPC 传输量 self._compute_and_strip_cached_images(multi_modal_data, num_cached_tokens) + if envs.DEEPINFER_MULTI_MODAL_INLINE_TEXT and not any( + isinstance(v, (list, tuple)) and len(v) for k, v in multi_modal_data.items() if not k.startswith("_") + ): + # No media (a text request to the VL model): the mapping is trivial, so do it here and make the request + # schedulable now. Through the worker it waits for a queue round trip and the next engine step's fetch(): + # on Neuron two ~0.36 s idle steps before every text prefill (issue #29). + # In place, as fetch() does: the Sequence holds this dict object. + mapped = self.multi_modal_mapper(dict(multi_modal_data)) + multi_modal_data.clear() + multi_modal_data.update(mapped) + self.multi_modal_dict[id] = multi_modal_data + self.scheduler.set_schedulable(id) + return self.multi_modal_dict[id] = multi_modal_data if not self._pending: # Starting a fresh batch after idle: reset the progress clock diff --git a/examples/neuron/bench_neuron.py b/examples/neuron/bench_neuron.py index d426336..b874675 100644 --- a/examples/neuron/bench_neuron.py +++ b/examples/neuron/bench_neuron.py @@ -137,6 +137,8 @@ def _run_one(llm, prompt: list[int] | None, params) -> tuple[float, float, int] unfinished = engine.has_unfinished_requests() outputs = engine.step() if unfinished else engine.flush_inflight_reqs() now = time.perf_counter() + if os.environ.get("BENCH_TRACE_STEPS") == "1" and prompt is not None and tokens < 8: + print(f"BENCH_STEP t={1e3 * (now - start):8.1f} ms outputs={len(outputs)}", flush=True) steps += len(outputs) > 0 for out in outputs: tokens += 1