fix(stream): match stop strings in special-token output

deepseek-recipe-python/tests/test_bindings.py

@@ -1016,3 +1016,41 @@ def test_sse_framing_of_the_three_responses():
         assert json.loads(response.to_json())["id"] == "sse"
         with pytest.raises(RuntimeError, match="consumed"):
             response_type.sse_event_name(outputs[0])
+
+
+@pytest.mark.parametrize(
+    "request_type,response_type,stop_key",
+    [
+        (ChatCompletionRequest, ChatCompletionResponse, "stop"),
+        (MessagesRequest, MessagesResponse, "stop_sequences"),
+    ],
+)
+def test_stop_strings_match_special_tokens(request_type, response_type, stop_key):
+    marker = "<｜ref｜>"
+    converted = request_type({
+        **REQUEST, "thinking": {"type": "disabled"}, stop_key: [marker],
+    }).convert(ConversionOptions())
+    tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
+    transcript = f"answer{marker}not emitted"
+    for tokens in (False, True):
+        generator = request_type.chunk_generator(converted, "stop-test", "m")
+        chunk = (
+            InferenceChunk.tokens(tokenizer.encode(transcript)) if tokens
+            else InferenceChunk.text(transcript)
+        )
+        outputs = run_processor(
+            generator, converted.parsing_options,
+            [InferenceChunk.ready(), chunk],
+            tokenizer if tokens else None,
+        )
+        response = response_type("stop-test", "m", 0, 0, 0)
+        for output in outputs:
+            response.accumulate(output)
+        body = json.loads(response.to_json())
+        if request_type is ChatCompletionRequest:
+            assert body["choices"][0]["message"]["content"] == "answer"
+            assert body["choices"][0]["finish_reason"] == "stop"
+        else:
+            assert body["content"] == [{"type": "text", "text": "answer"}]
+            assert body["stop_reason"] == "stop_sequence"
+            assert body["stop_sequence"] == marker

deepseek-recipe/src/stream/output_parser.rs

@@ -51,6 +51,7 @@ pub struct ParsingOptions {
     /// tokens.
     pub thinking_initial_stage: Option<ThinkingInitialStage>,
     /// Match stop sequences in answer content and detected JSON output.
+    /// Matching uses the decoded spelling of both plain and special tokens.
     /// Reasoning content, tool-call markup, and the text around JSON output are
     /// excluded. The parser discards empty sequences during construction.
     pub stop_sequences: Vec<String>,
@@ -611,7 +612,14 @@ impl State {
             self.buffered_len += 1;
             let mut matched_branch = None;
             for branch in self.match_branches.0.iter_mut() {
-                if branch.state.feed(byte, kind) {
+                // Stop strings constrain decoded text regardless of token kind;
+                // structural markers still require their original token kinds.
+                let match_kind = if branch.action_on_matched == OutputAction::StopSequence {
+                    ChunkKind::Text
+                } else {
+                    kind
+                };
+                if branch.state.feed(byte, match_kind) {
                     matched_branch = Some((
                         branch.state.matching_len(),
                         branch.next_stage,

deepseek-recipe/src/stream/processor.rs

@@ -1142,4 +1142,131 @@ mod tests {
         assert_eq!(content(&output), "");
         assert_eq!(finish_chunk(&output).2.completion_tokens, 2);
     }
+
+    #[tokio::test]
+    async fn stop_sequences_match_special_token_output() {
+        let tokenizer = tokenizer();
+        let marker = "<｜ref｜>";
+        let options = || ParsingOptions {
+            stop_sequences: vec![marker.to_owned()],
+            ..ParsingOptions::default()
+        };
+        let transcript = format!("answer{marker}not emitted");
+        let text_output = parse_texts(&[&transcript], options()).await;
+        assert_eq!(content(&text_output), "answer");
+        let ids = encode(&tokenizer, &transcript);
+        assert!(ids.contains(&special_token_id(&tokenizer, marker)));
+        let token_output = parse(
+            processor(options()).with_tokenizer(tokenizer),
+            vec![
+                InferenceChunk::Tokens {
+                    token_ids: ids,
+                    logprobs: None,
+                },
+                finish(InferenceFinishReason::Stop),
+            ],
+        )
+        .await;
+        assert_eq!(content(&token_output), "answer");
+        assert_eq!(finish_chunk(&token_output).0, FinishReason::StopSequence);
+        assert_eq!(finish_chunk(&token_output).1, Some(marker));
+    }
+
+    #[tokio::test]
+    async fn stop_sequences_span_plain_and_special_tokens_at_every_token_split() {
+        let tokenizer = tokenizer();
+        let stop = "ab<｜ref｜>cd";
+        let ids = encode(&tokenizer, "aab<｜ref｜>cdd");
+        for split in 0..=ids.len() {
+            let options = ParsingOptions {
+                stop_sequences: vec![stop.to_owned()],
+                ..ParsingOptions::default()
+            };
+            let output = parse(
+                processor(options).with_tokenizer(tokenizer.clone()),
+                vec![
+                    InferenceChunk::Tokens {
+                        token_ids: ids[..split].to_vec(),
+                        logprobs: None,
+                    },
+                    InferenceChunk::Tokens {
+                        token_ids: ids[split..].to_vec(),
+                        logprobs: None,
+                    },
+                    finish(InferenceFinishReason::Stop),
+                ],
+            )
+            .await;
+            assert_eq!(content(&output), "a", "split {split}");
+            assert_eq!(finish_chunk(&output).0, FinishReason::StopSequence);
+            assert_eq!(finish_chunk(&output).1, Some(stop));
+        }
+    }
+
+    #[tokio::test]
+    async fn special_token_stops_stay_excluded_from_reasoning_and_tool_arguments() {
+        let tokenizer = tokenizer();
+        let marker = "<｜ref｜>";
+        let options = || ParsingOptions {
+            stop_sequences: vec![marker.to_owned()],
+            ..reasoning_options()
+        };
+        let transcript = format!("hidden{marker}value</think>answer{marker}ignored");
+        let output = parse(
+            processor(options()).with_tokenizer(tokenizer.clone()),
+            vec![
+                InferenceChunk::Tokens {
+                    token_ids: encode(&tokenizer, &transcript),
+                    logprobs: None,
+                },
+                finish(InferenceFinishReason::Stop),
+            ],
+        )
+        .await;
+        assert_eq!(reasoning(&output), format!("hidden{marker}value"));
+        assert_eq!(content(&output), "answer");
+        assert_eq!(finish_chunk(&output).0, FinishReason::StopSequence);
+
+        let tool_calls = format!(
+            r#"<｜DSML｜tool_calls><｜DSML｜invoke name="f"><｜DSML｜parameter name="x" string="true">a{marker}b</｜DSML｜parameter></｜DSML｜invoke></｜DSML｜tool_calls>"#
+        );
+        let output = parse(
+            processor(options()).with_tokenizer(tokenizer.clone()),
+            vec![
+                InferenceChunk::Tokens {
+                    token_ids: encode(&tokenizer, &tool_calls),
+                    logprobs: None,
+                },
+                finish(InferenceFinishReason::Stop),
+            ],
+        )
+        .await;
+        assert_tool_calls(&output, &[("f", r#"{"x": "a<｜ref｜>b"}"#)]);
+        assert_eq!(finish_chunk(&output).0, FinishReason::ToolCalls);
+    }
+
+    #[tokio::test]
+    async fn special_token_stops_also_match_in_json_output() {
+        let tokenizer = tokenizer();
+        let marker = "<｜ref｜>";
+        let options = ParsingOptions {
+            stop_sequences: vec![marker.to_owned()],
+            parse_json_output: true,
+            ..ParsingOptions::default()
+        };
+        let output = parse(
+            processor(options).with_tokenizer(tokenizer.clone()),
+            vec![
+                InferenceChunk::Tokens {
+                    token_ids: encode(&tokenizer, r#"{"x":"a<｜ref｜>b"}"#),
+                    logprobs: None,
+                },
+                finish(InferenceFinishReason::Stop),
+            ],
+        )
+        .await;
+        assert_eq!(content(&output), r#"{"x":"a"#);
+        assert_eq!(finish_chunk(&output).0, FinishReason::StopSequence);
+        assert_eq!(finish_chunk(&output).1, Some(marker));
+    }
 }

docs/streaming.md

@@ -8,6 +8,12 @@ Completions chunks. The application supplies the backend and HTTP transport.
 In thinking mode, the processor counts `reasoning_tokens` for usage only when a
 tokenizer is attached; see [Use with tokenizer](tokenizer.md#decoding).
 
+Stop strings match decoded answer content and detected JSON output for both
+text and token-ID input, including special-token spellings. They do not match
+reasoning content or tool-call markup and arguments. Recognition of structural
+markers such as the thinking terminator and DSML still respects token identity
+for token-ID input.
+
 ## Rust
 
 Add the dependencies to your application:

docs/streaming.zh.md

@@ -4,6 +4,8 @@
 
 `StreamProcessor` 把后端输出转换为 Messages、Chat Completions 或 Responses 事件。以下示例使用 mock 推理并打印 Chat Completions chunk。后端与 HTTP 传输由应用提供。思考模式下，只有为 processor 附加 tokenizer 后，它才会统计 usage 中的 `reasoning_tokens`，见[使用 tokenizer](tokenizer.zh.md#解码)。
 
+stop 字符串会匹配解码后的正文和检测到的 JSON 输出，包括 special token 的拼写；文本输入和 token-ID 输入遵循同一规则。它不匹配思考内容、工具调用标记或参数。对于 token-ID 输入，思考结束标记和 DSML 等结构标记的识别仍依据 token 身份。
+
 ## Rust
 
 为你的应用添加依赖：