Merge 63f742bc74e892c59012e63a20eefe68be740995 into 935817c66f5d6e64838f419692545f06787e485f

This commit is contained in:
alp 2026-10-22 13:12:37 -07:00 committed by GitHub
commit 74ea8b2e3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 47 additions and 9 deletions

View File

@ -161,11 +161,11 @@ pub(super) enum InvalidTokenKind {
/// two distinct actions to produce both of its quotes. /// two distinct actions to produce both of its quotes.
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
pub(super) enum ArgumentsLabel { pub(super) enum ArgumentsLabel {
/// `{` before the first argument. /// `{"` before the first argument name.
ObjectOpen, ObjectOpen,
/// `, ` before each later argument. /// `, "` before each later argument name.
ArgumentSeparator, ArgumentSeparator,
/// `: ` between an argument name and its value. /// `": ` between an argument name and its value.
KeyValueSeparator, KeyValueSeparator,
/// `"` before a string value. /// `"` before a string value.
StringOpen, StringOpen,
@ -176,9 +176,9 @@ pub(super) enum ArgumentsLabel {
impl ArgumentsLabel { impl ArgumentsLabel {
pub(super) fn as_str(self) -> &'static str { pub(super) fn as_str(self) -> &'static str {
match self { match self {
Self::ObjectOpen => "{", Self::ObjectOpen => "{\"",
Self::ArgumentSeparator => ", ", Self::ArgumentSeparator => ", \"",
Self::KeyValueSeparator => ": ", Self::KeyValueSeparator => "\": ",
Self::StringOpen | Self::StringClose => "\"", Self::StringOpen | Self::StringClose => "\"",
} }
} }
@ -247,7 +247,7 @@ enum Stage {
/// Inside an `invoke` tag, before a `parameter` tag or the closing tag. /// Inside an `invoke` tag, before a `parameter` tag or the closing tag.
/// `is_first_argument` holds until the first argument is written. /// `is_first_argument` holds until the first argument is written.
ToolCallArguments { is_first_argument: bool }, ToolCallArguments { is_first_argument: bool },
/// The quoted argument name of a `parameter` tag. /// The argument name of a `parameter` tag, up to its closing quote.
ToolCallArgumentName, ToolCallArgumentName,
/// The `string="true|false">` attribute of a `parameter` tag. /// The `string="true|false">` attribute of a `parameter` tag.
ToolCallArgumentStringAttribute, ToolCallArgumentStringAttribute,
@ -504,7 +504,7 @@ impl State {
} }
Stage::ToolCallArguments { is_first_argument } => { Stage::ToolCallArguments { is_first_argument } => {
match_branches.add_literal_branch( match_branches.add_literal_branch(
"parameter name=", "parameter name=\"",
Stage::ToolCallArgumentName, Stage::ToolCallArgumentName,
if is_first_argument { if is_first_argument {
ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::ObjectOpen) ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::ObjectOpen)
@ -525,7 +525,7 @@ impl State {
} }
Stage::ToolCallArgumentName => { Stage::ToolCallArgumentName => {
match_branches.add_literal_branch( match_branches.add_literal_branch(
" ", "\"",
Stage::ToolCallArgumentStringAttribute, Stage::ToolCallArgumentStringAttribute,
ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::KeyValueSeparator), ActionOnMatched::LabelToolCallArguments(ArgumentsLabel::KeyValueSeparator),
); );

View File

@ -748,6 +748,44 @@ mod tests {
assert_eq!(finish_chunk(&chunks).0, FinishReason::ToolCalls); assert_eq!(finish_chunk(&chunks).0, FinishReason::ToolCalls);
} }
#[tokio::test]
async fn argument_names_keep_their_spaces() {
// Names with and without spaces, as first and later arguments, with
// string and raw values; an empty name ends the second call.
let block = r#"<|DSML|tool_calls>
<|DSML|invoke name="book_trip">
<|DSML|parameter name="start date" string="true">2026-10-22</|DSML|parameter>
<|DSML|parameter name="city" string="true">Tokyo</|DSML|parameter>
<|DSML|parameter name="party size" string="false">2</|DSML|parameter>
<|DSML|parameter name="nights" string="false">3</|DSML|parameter>
</|DSML|invoke>
<|DSML|invoke name="book_trip">
<|DSML|parameter name="room count" string="false">1</|DSML|parameter>
<|DSML|parameter name="guest name" string="true">Ada Lovelace</|DSML|parameter>
<|DSML|parameter name="" string="true">empty</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls>"#;
let expected = [
(
"book_trip",
r#"{"start date": "2026-10-22", "city": "Tokyo", "party size": 2, "nights": 3}"#,
),
(
"book_trip",
r#"{"room count": 1, "guest name": "Ada Lovelace", "": "empty"}"#,
),
];
let output = format!("{PREAMBLE}\n\n{block}");
for chunks in [
parse_characters(&output, ParsingOptions::default()).await,
parse_texts(&[&output], ParsingOptions::default()).await,
] {
assert_eq!(content(&chunks), PREAMBLE);
assert_tool_calls(&chunks, &expected);
assert_eq!(finish_chunk(&chunks).0, FinishReason::ToolCalls);
}
}
#[tokio::test] #[tokio::test]
async fn json_text_in_a_string_value_is_escaped() { async fn json_text_in_a_string_value_is_escaped() {
let output = format!("{PREAMBLE}\n\n{TOOL_CALLS}"); let output = format!("{PREAMBLE}\n\n{TOOL_CALLS}");