Merge f03f3b0899206fbe34458494d86b880a417c7e05 into 553741557a57df44a46fb3ab72b6aafeaf945337

deepinfer/entrypoints/llm.py

@@ -157,13 +157,23 @@ class LLM:
         prompt_token_ids = np.asarray(prompt_token_ids, dtype=np.int32)
         if multi_modal_data is None and self.llm_engine.multi_modal_processor is not None:
             multi_modal_data = dict(image_bytes=[], image_sizes=[])
-        return self.llm_engine.add_request(
+        sequence = self.llm_engine.add_request(
             user_id="0",  # HACK 随便加一个 user_id
             request_id=request_id,
             sampling_params=sampling_params,
             prompt_token_ids=prompt_token_ids,
             multi_modal_data=multi_modal_data,
         )
+        # add_request returns None when it rejects the request (e.g. out-of-range
+        # prompt token ids, see the engine log for details). Silently dropping it
+        # would make generate() return fewer results than the input prompts,
+        # misaligning results[i] with prompts[i], so fail loudly instead.
+        if sequence is None:
+            raise ValueError(
+                f"Request {request_id} was rejected by the engine and returned None "
+                "(see engine log for the reason, e.g. out-of-range prompt token ids)"
+            )
+        return sequence
 
     def _run_engine(self, use_tqdm: bool) -> list[list[RequestOutput] | str]:
         # Initialize tqdm.

tests/entrypoints/test_llm_add_request.py

@@ -0,0 +1,38 @@
+import itertools
+from types import SimpleNamespace
+
+import pytest
+
+from deepinfer.entrypoints.llm import LLM
+from deepinfer.sampling_params import SamplingParams
+
+SENTINEL_SEQUENCE = object()
+
+
+def _llm(add_request_result: object) -> LLM:
+    """A bare LLM whose engine add_request always returns the given result."""
+    llm = object.__new__(LLM)
+    llm.request_counter = itertools.count()
+    llm.llm_engine = SimpleNamespace(
+        multi_modal_processor=None,
+        add_request=lambda **kwargs: add_request_result,
+    )
+    return llm
+
+
+def test_add_request_returns_engine_sequence() -> None:
+    llm = _llm(SENTINEL_SEQUENCE)
+
+    sequence = llm._add_request(prompt=None, sampling_params=SamplingParams(), prompt_token_ids=[1, 2, 3])
+
+    assert sequence is SENTINEL_SEQUENCE
+
+
+def test_add_request_raises_when_engine_rejects_request() -> None:
+    # add_request returns None when the engine drops the request (e.g. out-of-range
+    # prompt token ids). generate() must fail loudly instead of silently returning
+    # fewer outputs than input prompts, which would misalign results with prompts.
+    llm = _llm(None)
+
+    with pytest.raises(ValueError, match="rejected by the engine"):
+        llm._add_request(prompt=None, sampling_params=SamplingParams(), prompt_token_ids=[1, 2, 3])