diff --git a/deepseek-recipe-encoding/src/tokenizer.rs b/deepseek-recipe-encoding/src/tokenizer.rs index 10e5c61..1eda0d6 100644 --- a/deepseek-recipe-encoding/src/tokenizer.rs +++ b/deepseek-recipe-encoding/src/tokenizer.rs @@ -36,7 +36,9 @@ pub trait TokenizerEncoder: Send + Sync { /// /// It holds a copy of the tokenizer without its added vocabulary, which /// encodes plain text, and the added vocabulary, which supplies special token -/// IDs. +/// IDs. Padding and truncation are disabled in the copy: plain regions are +/// fragments of one prompt, not independent sequences. The caller's tokenizer +/// is unchanged. #[derive(Clone)] pub struct PromptTokenizer { plain_tokenizer: Tokenizer, @@ -54,6 +56,10 @@ impl PromptTokenizer { .collect(); let mut plain_tokenizer = tokenizer.clone(); plain_tokenizer.with_added_vocabulary(AddedVocabulary::new()); + plain_tokenizer.with_padding(None); + plain_tokenizer + .with_truncation(None) + .expect("disabling truncation is valid"); Self { plain_tokenizer, added_vocabulary, @@ -74,7 +80,7 @@ impl TokenizerEncoder for PromptTokenizer { /// token ID of its spelling. The batch runs in parallel unless the /// `TOKENIZERS_PARALLELISM` environment variable disables it. The /// tokenizer adds no tokens, such as a BOS token, beyond those that the - /// text encodes to, provided that it has no padding configured. + /// text encodes to, and does not truncate the plain regions. /// /// # Errors /// @@ -122,3 +128,124 @@ impl TokenizerEncoder for Arc { (**self).encode_ids(text, special_ranges) } } + +#[cfg(test)] +mod tests { + use tokenizers::{PaddingParams, PaddingStrategy, TruncationParams}; + + use super::*; + + fn tokenizer() -> Tokenizer { + Tokenizer::from_file(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../static/tokenizers/v41/tokenizer.json" + )) + .expect("the bundled V4.1 tokenizer loads") + } + + fn assert_prompt_regions(padding: Option, truncation: bool) { + let original = tokenizer(); + let plain = "one two three four five"; + let bos = "<|begin▁of▁sentence|>"; + let eos = "<|end▁of▁sentence|>"; + let text = format!("{bos}{plain}{eos}"); + let ranges = [0..bos.len(), bos.len() + plain.len()..text.len()]; + let mut expected = vec![original.token_to_id(bos).unwrap()]; + let plain_ids = original.encode(plain, false).unwrap().get_ids().to_vec(); + assert!(plain_ids.len() > 2, "the fixture must exercise truncation"); + expected.extend_from_slice(&plain_ids); + expected.push(original.token_to_id(eos).unwrap()); + + let mut configured = original.clone(); + configured.with_padding(padding.clone().map(|strategy| PaddingParams { + strategy, + pad_id: 0, + ..PaddingParams::default() + })); + configured + .with_truncation(truncation.then_some(TruncationParams { + max_length: 2, + ..TruncationParams::default() + })) + .unwrap(); + let before = configured.to_string(false).unwrap(); + let encoder = PromptTokenizer::new(&configured); + assert_eq!( + encoder.encode_ids(&text, &ranges).unwrap(), + expected, + "padding={padding:?}, truncation={truncation}" + ); + assert_eq!(configured.to_string(false).unwrap(), before); + assert_eq!(encoder.encode_ids(plain, &[]).unwrap(), plain_ids); + assert!(encoder.encode_ids("", &[]).unwrap().is_empty()); + } + + #[test] + fn prompt_regions_are_not_padded() { + assert_prompt_regions(Some(PaddingStrategy::Fixed(32)), false); + assert_prompt_regions(Some(PaddingStrategy::BatchLongest), false); + } + + #[test] + fn prompt_regions_are_not_truncated() { + assert_prompt_regions(None, true); + } + + #[test] + fn configured_tokenizers_preserve_both_model_prompts() { + use deepseek_recipe_core::conversation::Conversation; + use deepseek_recipe_core::messages::InputMessage; + use deepseek_recipe_core::prompt::Prompt; + + use crate::PromptEncoding; + use crate::v4::{dsv4::DeepseekV4Encoding, dsv41::DeepseekV41Encoding}; + + let original = tokenizer(); + let mut configured = original.clone(); + configured.with_padding(Some(PaddingParams { + strategy: PaddingStrategy::BatchLongest, + pad_to_multiple_of: Some(8), + ..PaddingParams::default() + })); + configured + .with_truncation(Some(TruncationParams { + max_length: 2, + ..TruncationParams::default() + })) + .unwrap(); + // Saved tokenizer.json files retain these settings, including when + // loaded through the Python bindings. + let configured = + Tokenizer::from_bytes(configured.to_string(false).unwrap().as_bytes()).unwrap(); + let conversation = Conversation { + messages: vec![InputMessage::User { + content: Prompt::plain("Please preserve every word in this message."), + image_sources: Vec::new(), + }], + thinking_mode: false, + ..Conversation::default() + }; + let reference = PromptTokenizer::new(&original); + let actual = PromptTokenizer::new(&configured); + assert_eq!( + DeepseekV4Encoding::new() + .with_tokenizer(actual.clone()) + .encode(&conversation) + .unwrap(), + DeepseekV4Encoding::new() + .with_tokenizer(reference.clone()) + .encode(&conversation) + .unwrap() + ); + assert_eq!( + DeepseekV41Encoding::new() + .with_tokenizer(actual) + .encode(&conversation) + .unwrap(), + DeepseekV41Encoding::new() + .with_tokenizer(reference) + .encode(&conversation) + .unwrap() + ); + } +} diff --git a/deepseek-recipe-python/tests/test_bindings.py b/deepseek-recipe-python/tests/test_bindings.py index 78dd659..deb27de 100644 --- a/deepseek-recipe-python/tests/test_bindings.py +++ b/deepseek-recipe-python/tests/test_bindings.py @@ -149,6 +149,37 @@ def test_encoding_with_tokenizer(): assert prompt_ids == tokenizer.encode(encoding.render_conversation(converted.conversation).prompt) + +@pytest.mark.parametrize("encoding_class", [DeepseekV4Encoding, DeepseekV41Encoding]) +def test_prompt_encoding_preserves_input_with_configured_tokenizer(encoding_class): + text = "Please preserve every word in this message." + conversation = Conversation(messages=[UserMessage(text)], thinking_mode=False) + reference = Tokenizer.from_file(str(TOKENIZER_PATH)) + config = json.loads(TOKENIZER_PATH.read_text()) + config["padding"] = { + "strategy": "BatchLongest", + "direction": "Right", + "pad_to_multiple_of": 8, + "pad_id": 0, + "pad_type_id": 0, + "pad_token": "[PAD]", + } + config["truncation"] = { + "direction": "Right", + "max_length": 2, + "strategy": "LongestFirst", + "stride": 0, + } + configured = Tokenizer.from_str(json.dumps(config)) + direct_ids = configured.encode(text) + assert len(direct_ids) == 8 + assert encoding_class().with_tokenizer(configured).encode(conversation) == ( + encoding_class().with_tokenizer(reference).encode(conversation) + ) + # Preparing the prompt encoder must not change direct tokenizer usage. + assert configured.encode(text) == direct_ids + + def test_prompt_encoding_differs_from_tokenizer_on_special_token_spellings(): tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH)) encoding = DeepseekV41Encoding().with_tokenizer(tokenizer) diff --git a/docs/tokenizer.md b/docs/tokenizer.md index 8ee41a5..efad379 100644 --- a/docs/tokenizer.md +++ b/docs/tokenizer.md @@ -71,6 +71,11 @@ ranges is plain text, encoded without the added vocabulary. `PromptTokenizer` encodes the plain regions as one batch, which the `tokenizers` crate can encode in parallel, depending on its `TOKENIZERS_PARALLELISM` setting. +`PromptTokenizer` disables padding and truncation in its private copy of the +tokenizer, so that individual plain regions are neither padded nor shortened. +The original tokenizer and its direct encoding behavior are unchanged. Apply +context limits to the complete encoded prompt, as described below. + A special-token spelling typed into a user message is plain text, so it cannot act as a control token. A caller that tokenizes `RenderedPrompt::prompt` itself must apply `special_ranges` the same way. diff --git a/docs/tokenizer.zh.md b/docs/tokenizer.zh.md index e94a39a..918a7ca 100644 --- a/docs/tokenizer.zh.md +++ b/docs/tokenizer.zh.md @@ -45,6 +45,8 @@ fn main() -> Result<(), Box> { `render_conversation` 只执行渲染这一步:它返回 prompt 文本、其中显式 special token 的字节区间(`special_ranges`)以及图片来源。渲染把模板的控制 token、消息图片来源对应的占位符,以及 assistant 内容和 reasoning 内容中的 grounding token 作为显式 special token 插入。分词时,每个 special 区间编码为其拼写对应的 token ID,区间之间的文本是 plain text,不使用 added vocabulary 编码;`PromptTokenizer` 把各 plain 区域作为一个 batch 编码;`tokenizers` crate 可以并行编码该 batch,取决于其 `TOKENIZERS_PARALLELISM` 设置。 +`PromptTokenizer` 会在私有 tokenizer 副本中禁用 padding 和 truncation,避免对单个 plain 区域填充或截断。原 tokenizer 及其直接编码行为保持不变。上下文限制应在完整 prompt 编码后处理,见下文。 + 因此,用户消息里写出的 special token 拼写是 plain text,不会成为控制 token。调用方自行对 `RenderedPrompt::prompt` 分词时必须同样处理 `special_ranges`。 编码之后,调用方用 `ConversationRequest::resolve_max_tokens(prompt_tokens, context_limit)` 解析要下发的 `max_tokens`。`prompt_tokens` 是编码得到的 token ID 数加上图片超出占位符的 token 数(`MultiModalData::image_token_adjustment`),`context_limit` 是模型上下文可容纳的 token 数。请求指定的 `max_tokens` 保持不变;未指定时,非思考模式取 8192,思考模式且 reasoning effort 为 `max` 时取 131072,其余思考模式取 65536,并降到 prompt 之后剩余的上下文以内。prompt 与请求的 `max_tokens` 之和超过 `context_limit` 时,该方法返回 `ConversionError`。