fix(encoding): preserve prompts with configured tokenizers

deepseek-recipe-encoding/src/tokenizer.rs

@@ -36,7 +36,9 @@ pub trait TokenizerEncoder: Send + Sync {
 ///
 /// It holds a copy of the tokenizer without its added vocabulary, which
 /// encodes plain text, and the added vocabulary, which supplies special token
-/// IDs.
+/// IDs. Padding and truncation are disabled in the copy: plain regions are
+/// fragments of one prompt, not independent sequences. The caller's tokenizer
+/// is unchanged.
 #[derive(Clone)]
 pub struct PromptTokenizer {
     plain_tokenizer: Tokenizer,
@@ -54,6 +56,10 @@ impl PromptTokenizer {
             .collect();
         let mut plain_tokenizer = tokenizer.clone();
         plain_tokenizer.with_added_vocabulary(AddedVocabulary::new());
+        plain_tokenizer.with_padding(None);
+        plain_tokenizer
+            .with_truncation(None)
+            .expect("disabling truncation is valid");
         Self {
             plain_tokenizer,
             added_vocabulary,
@@ -74,7 +80,7 @@ impl TokenizerEncoder for PromptTokenizer {
     /// token ID of its spelling. The batch runs in parallel unless the
     /// `TOKENIZERS_PARALLELISM` environment variable disables it. The
     /// tokenizer adds no tokens, such as a BOS token, beyond those that the
-    /// text encodes to, provided that it has no padding configured.
+    /// text encodes to, and does not truncate the plain regions.
     ///
     /// # Errors
     ///
@@ -122,3 +128,124 @@ impl<T: TokenizerEncoder + ?Sized> TokenizerEncoder for Arc<T> {
         (**self).encode_ids(text, special_ranges)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use tokenizers::{PaddingParams, PaddingStrategy, TruncationParams};
+
+    use super::*;
+
+    fn tokenizer() -> Tokenizer {
+        Tokenizer::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../static/tokenizers/v41/tokenizer.json"
+        ))
+        .expect("the bundled V4.1 tokenizer loads")
+    }
+
+    fn assert_prompt_regions(padding: Option<PaddingStrategy>, truncation: bool) {
+        let original = tokenizer();
+        let plain = "one two three four five";
+        let bos = "<｜begin▁of▁sentence｜>";
+        let eos = "<｜end▁of▁sentence｜>";
+        let text = format!("{bos}{plain}{eos}");
+        let ranges = [0..bos.len(), bos.len() + plain.len()..text.len()];
+        let mut expected = vec![original.token_to_id(bos).unwrap()];
+        let plain_ids = original.encode(plain, false).unwrap().get_ids().to_vec();
+        assert!(plain_ids.len() > 2, "the fixture must exercise truncation");
+        expected.extend_from_slice(&plain_ids);
+        expected.push(original.token_to_id(eos).unwrap());
+
+        let mut configured = original.clone();
+        configured.with_padding(padding.clone().map(|strategy| PaddingParams {
+            strategy,
+            pad_id: 0,
+            ..PaddingParams::default()
+        }));
+        configured
+            .with_truncation(truncation.then_some(TruncationParams {
+                max_length: 2,
+                ..TruncationParams::default()
+            }))
+            .unwrap();
+        let before = configured.to_string(false).unwrap();
+        let encoder = PromptTokenizer::new(&configured);
+        assert_eq!(
+            encoder.encode_ids(&text, &ranges).unwrap(),
+            expected,
+            "padding={padding:?}, truncation={truncation}"
+        );
+        assert_eq!(configured.to_string(false).unwrap(), before);
+        assert_eq!(encoder.encode_ids(plain, &[]).unwrap(), plain_ids);
+        assert!(encoder.encode_ids("", &[]).unwrap().is_empty());
+    }
+
+    #[test]
+    fn prompt_regions_are_not_padded() {
+        assert_prompt_regions(Some(PaddingStrategy::Fixed(32)), false);
+        assert_prompt_regions(Some(PaddingStrategy::BatchLongest), false);
+    }
+
+    #[test]
+    fn prompt_regions_are_not_truncated() {
+        assert_prompt_regions(None, true);
+    }
+
+    #[test]
+    fn configured_tokenizers_preserve_both_model_prompts() {
+        use deepseek_recipe_core::conversation::Conversation;
+        use deepseek_recipe_core::messages::InputMessage;
+        use deepseek_recipe_core::prompt::Prompt;
+
+        use crate::PromptEncoding;
+        use crate::v4::{dsv4::DeepseekV4Encoding, dsv41::DeepseekV41Encoding};
+
+        let original = tokenizer();
+        let mut configured = original.clone();
+        configured.with_padding(Some(PaddingParams {
+            strategy: PaddingStrategy::BatchLongest,
+            pad_to_multiple_of: Some(8),
+            ..PaddingParams::default()
+        }));
+        configured
+            .with_truncation(Some(TruncationParams {
+                max_length: 2,
+                ..TruncationParams::default()
+            }))
+            .unwrap();
+        // Saved tokenizer.json files retain these settings, including when
+        // loaded through the Python bindings.
+        let configured =
+            Tokenizer::from_bytes(configured.to_string(false).unwrap().as_bytes()).unwrap();
+        let conversation = Conversation {
+            messages: vec![InputMessage::User {
+                content: Prompt::plain("Please preserve every word in this message."),
+                image_sources: Vec::new(),
+            }],
+            thinking_mode: false,
+            ..Conversation::default()
+        };
+        let reference = PromptTokenizer::new(&original);
+        let actual = PromptTokenizer::new(&configured);
+        assert_eq!(
+            DeepseekV4Encoding::new()
+                .with_tokenizer(actual.clone())
+                .encode(&conversation)
+                .unwrap(),
+            DeepseekV4Encoding::new()
+                .with_tokenizer(reference.clone())
+                .encode(&conversation)
+                .unwrap()
+        );
+        assert_eq!(
+            DeepseekV41Encoding::new()
+                .with_tokenizer(actual)
+                .encode(&conversation)
+                .unwrap(),
+            DeepseekV41Encoding::new()
+                .with_tokenizer(reference)
+                .encode(&conversation)
+                .unwrap()
+        );
+    }
+}

deepseek-recipe-python/tests/test_bindings.py

@@ -149,6 +149,37 @@ def test_encoding_with_tokenizer():
     assert prompt_ids == tokenizer.encode(encoding.render_conversation(converted.conversation).prompt)
 
 
+
+@pytest.mark.parametrize("encoding_class", [DeepseekV4Encoding, DeepseekV41Encoding])
+def test_prompt_encoding_preserves_input_with_configured_tokenizer(encoding_class):
+    text = "Please preserve every word in this message."
+    conversation = Conversation(messages=[UserMessage(text)], thinking_mode=False)
+    reference = Tokenizer.from_file(str(TOKENIZER_PATH))
+    config = json.loads(TOKENIZER_PATH.read_text())
+    config["padding"] = {
+        "strategy": "BatchLongest",
+        "direction": "Right",
+        "pad_to_multiple_of": 8,
+        "pad_id": 0,
+        "pad_type_id": 0,
+        "pad_token": "[PAD]",
+    }
+    config["truncation"] = {
+        "direction": "Right",
+        "max_length": 2,
+        "strategy": "LongestFirst",
+        "stride": 0,
+    }
+    configured = Tokenizer.from_str(json.dumps(config))
+    direct_ids = configured.encode(text)
+    assert len(direct_ids) == 8
+    assert encoding_class().with_tokenizer(configured).encode(conversation) == (
+        encoding_class().with_tokenizer(reference).encode(conversation)
+    )
+    # Preparing the prompt encoder must not change direct tokenizer usage.
+    assert configured.encode(text) == direct_ids
+
+
 def test_prompt_encoding_differs_from_tokenizer_on_special_token_spellings():
     tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))
     encoding = DeepseekV41Encoding().with_tokenizer(tokenizer)

docs/tokenizer.md

@@ -71,6 +71,11 @@ ranges is plain text, encoded without the added vocabulary. `PromptTokenizer`
 encodes the plain regions as one batch, which the `tokenizers` crate can encode
 in parallel, depending on its `TOKENIZERS_PARALLELISM` setting.
 
+`PromptTokenizer` disables padding and truncation in its private copy of the
+tokenizer, so that individual plain regions are neither padded nor shortened.
+The original tokenizer and its direct encoding behavior are unchanged. Apply
+context limits to the complete encoded prompt, as described below.
+
 A special-token spelling typed into a user message is plain text, so it cannot
 act as a control token. A caller that tokenizes `RenderedPrompt::prompt` itself
 must apply `special_ranges` the same way.

docs/tokenizer.zh.md

@@ -45,6 +45,8 @@ fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
 
 `render_conversation` 只执行渲染这一步：它返回 prompt 文本、其中显式 special token 的字节区间（`special_ranges`）以及图片来源。渲染把模板的控制 token、消息图片来源对应的占位符，以及 assistant 内容和 reasoning 内容中的 grounding token 作为显式 special token 插入。分词时，每个 special 区间编码为其拼写对应的 token ID，区间之间的文本是 plain text，不使用 added vocabulary 编码；`PromptTokenizer` 把各 plain 区域作为一个 batch 编码；`tokenizers` crate 可以并行编码该 batch，取决于其 `TOKENIZERS_PARALLELISM` 设置。
 
+`PromptTokenizer` 会在私有 tokenizer 副本中禁用 padding 和 truncation，避免对单个 plain 区域填充或截断。原 tokenizer 及其直接编码行为保持不变。上下文限制应在完整 prompt 编码后处理，见下文。
+
 因此，用户消息里写出的 special token 拼写是 plain text，不会成为控制 token。调用方自行对 `RenderedPrompt::prompt` 分词时必须同样处理 `special_ranges`。
 
 编码之后，调用方用 `ConversationRequest::resolve_max_tokens(prompt_tokens, context_limit)` 解析要下发的 `max_tokens`。`prompt_tokens` 是编码得到的 token ID 数加上图片超出占位符的 token 数（`MultiModalData::image_token_adjustment`），`context_limit` 是模型上下文可容纳的 token 数。请求指定的 `max_tokens` 保持不变；未指定时，非思考模式取 8192，思考模式且 reasoning effort 为 `max` 时取 131072，其余思考模式取 65536，并降到 prompt 之后剩余的上下文以内。prompt 与请求的 `max_tokens` 之和超过 `context_limit` 时，该方法返回 `ConversionError`。