Map media-free requests inline instead of through the multimodal worker (DEEPINFER_MULTI_MODAL_INLINE_TEXT=1)

A text request to the VL model went through the multimodal worker's queue and became schedulable only at a
later engine step's fetch(): on Neuron two ~0.36 s idle placeholder steps before every text prefill.
vllm-sonnet 512 (prefill bucket 512): TTFT 2.42 -> 1.67 s, tokens unchanged. bench_neuron: BENCH_TRACE_STEPS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

deepinfer/environ.py

@@ -258,6 +258,9 @@ class envs:
     DEEPINFER_USE_MULTIMODAL_POOL = EnvVar.bool()
     DEEPINFER_MULTIMODAL_POOL_SIZE_GB = EnvVar.int(default=8)
     DEEPINFER_MULTI_MODAL_BUFFER_SIZE = EnvVar.int(default=512)
+    # Map requests without media inline at add time instead of through the multimodal worker (no IPC round trip,
+    # schedulable in the same step).
+    DEEPINFER_MULTI_MODAL_INLINE_TEXT = EnvVar.bool(default=True)
     DEEPINFER_MULTI_MODAL_WORKER_TIMEOUT = EnvVar.float(default=1800.0)
 
 

deepinfer/ve/multimodal_processor.py

@@ -254,6 +254,19 @@ class MultimodalProcessor:
         # 计算缓存图片集合，剥离 binary，减少 IPC 传输量
         self._compute_and_strip_cached_images(multi_modal_data, num_cached_tokens)
 
+        if envs.DEEPINFER_MULTI_MODAL_INLINE_TEXT and not any(
+            isinstance(v, (list, tuple)) and len(v) for k, v in multi_modal_data.items() if not k.startswith("_")
+        ):
+            # No media (a text request to the VL model): the mapping is trivial, so do it here and make the request
+            # schedulable now. Through the worker it waits for a queue round trip and the next engine step's fetch():
+            # on Neuron two ~0.36 s idle steps before every text prefill (issue #29).
+            # In place, as fetch() does: the Sequence holds this dict object.
+            mapped = self.multi_modal_mapper(dict(multi_modal_data))
+            multi_modal_data.clear()
+            multi_modal_data.update(mapped)
+            self.multi_modal_dict[id] = multi_modal_data
+            self.scheduler.set_schedulable(id)
+            return
         self.multi_modal_dict[id] = multi_modal_data
         if not self._pending:
             # Starting a fresh batch after idle: reset the progress clock

examples/neuron/bench_neuron.py

@@ -137,6 +137,8 @@ def _run_one(llm, prompt: list[int] | None, params) -> tuple[float, float, int]
         unfinished = engine.has_unfinished_requests()
         outputs = engine.step() if unfinished else engine.flush_inflight_reqs()
         now = time.perf_counter()
+        if os.environ.get("BENCH_TRACE_STEPS") == "1" and prompt is not None and tokens < 8:
+            print(f"BENCH_STEP t={1e3 * (now - start):8.1f} ms outputs={len(outputs)}", flush=True)
         steps += len(outputs) > 0
         for out in outputs:
             tokens += 1