fix: preserve spaces in DSML tool parameter names

I made the output parser read a DSML parameter name up to its closing
quote instead of its first space, so a name like "start date" no longer
breaks the tool call arguments JSON. The quotes around the name now come
from the argument labels, so the output for other names is unchanged.

Assisted-by: Claude
Assisted-by: GPT-6 Astra
Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
This commit is contained in:
mottopanikeiku 2026-10-22 13:11:06 -07:00
parent 935817c66f
commit 63f742bc74
2 changed files with 47 additions and 9 deletions

View File

@ -161,11 +161,11 @@ pub(super) enum InvalidTokenKind {
/// two distinct actions to produce both of its quotes.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) enum ArgumentsLabel {
/// `{` before the first argument.
/// `{"` before the first argument name.
ObjectOpen,
/// `, ` before each later argument.
/// `, "` before each later argument name.
ArgumentSeparator,
/// `: ` between an argument name and its value.
/// `": ` between an argument name and its value.
KeyValueSeparator,
/// `"` before a string value.
StringOpen,
@ -176,9 +176,9 @@ pub(super) enum ArgumentsLabel {
impl ArgumentsLabel {
pub(super) fn as_str(self) -> &'static str {
match self {
Self::ObjectOpen => "{",
Self::ArgumentSeparator => ", ",
Self::KeyValueSeparator => ": ",
Self::ObjectOpen => "{\"",
Self::ArgumentSeparator => ", \"",
Self::KeyValueSeparator => "\": ",
Self::StringOpen | Self::StringClose => "\"",
}
}
@ -247,7 +247,7 @@ enum Stage {
/// Inside an `invoke` tag, before a `parameter` tag or the closing tag.
/// `is_first_argument` holds until the first argument is written.
ToolCallArguments { is_first_argument: bool },
/// The quoted argument name of a `parameter` tag.
/// The argument name of a `parameter` tag, up to its closing quote.
ToolCallArgumentName,
/// The `string="true|false">` attribute of a `parameter` tag.
ToolCallArgumentStringAttribute,
@ -504,7 +504,7 @@ impl State {
}
Stage::ToolCallArguments { is_first_argument } => {
match_branches.add_literal_branch(
"parameter name=",
"parameter name=\"",
Stage::ToolCallArgumentName,
if is_first_argument {
ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::ObjectOpen)
@ -525,7 +525,7 @@ impl State {
}
Stage::ToolCallArgumentName => {
match_branches.add_literal_branch(
" ",
"\"",
Stage::ToolCallArgumentStringAttribute,
ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::KeyValueSeparator),
);

View File

@ -748,6 +748,44 @@ mod tests {
assert_eq!(finish_chunk(&chunks).0, FinishReason::ToolCalls);
}
#[tokio::test]
async fn argument_names_keep_their_spaces() {
// Names with and without spaces, as first and later arguments, with
// string and raw values; an empty name ends the second call.
let block = r#"<|DSML|tool_calls>
<|DSML|invoke name="book_trip">
<|DSML|parameter name="start date" string="true">2026-10-22</|DSML|parameter>
<|DSML|parameter name="city" string="true">Tokyo</|DSML|parameter>
<|DSML|parameter name="party size" string="false">2</|DSML|parameter>
<|DSML|parameter name="nights" string="false">3</|DSML|parameter>
</|DSML|invoke>
<|DSML|invoke name="book_trip">
<|DSML|parameter name="room count" string="false">1</|DSML|parameter>
<|DSML|parameter name="guest name" string="true">Ada Lovelace</|DSML|parameter>
<|DSML|parameter name="" string="true">empty</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls>"#;
let expected = [
(
"book_trip",
r#"{"start date": "2026-10-22", "city": "Tokyo", "party size": 2, "nights": 3}"#,
),
(
"book_trip",
r#"{"room count": 1, "guest name": "Ada Lovelace", "": "empty"}"#,
),
];
let output = format!("{PREAMBLE}\n\n{block}");
for chunks in [
parse_characters(&output, ParsingOptions::default()).await,
parse_texts(&[&output], ParsingOptions::default()).await,
] {
assert_eq!(content(&chunks), PREAMBLE);
assert_tool_calls(&chunks, &expected);
assert_eq!(finish_chunk(&chunks).0, FinishReason::ToolCalls);
}
}
#[tokio::test]
async fn json_text_in_a_string_value_is_escaped() {
let output = format!("{PREAMBLE}\n\n{TOOL_CALLS}");