Merge 5dbea6ef5474578b5b7825d1475f13bdc5d3a81b into 553741557a57df44a46fb3ab72b6aafeaf945337

eval/aime/bench.py

@@ -89,7 +89,7 @@ def extract_boxed(text: str) -> str | None:
     return None
 
 
-_NUMBER_RE = re.compile(r"(?<![\w.])(\d{1,3})(?![\w.])")
+_NUMBER_RE = re.compile(r"(?<![\w.])(-?\d{1,3})(?![\w.])")
 
 
 def normalize_answer(ans: str | None) -> str | None:
@@ -120,7 +120,8 @@ def extract_answer_with_source(text: str) -> tuple[str | None, str]:
 
     1. `boxed`: the content of the last \boxed{...}. This is the format the prompt requests, so it
        is the most reliable path;
-    2. `tail_number`: the last 0-999 integer after `</think>` (or within the final 400 characters).
+    2. `tail_number`: the last integer of up to three digits, preserving its sign, after `</think>`
+       (or within the final 400 characters).
        A fallback for malformed output, **higher risk**; score.py reports its share separately;
     3. `none`: nothing found.
     """

tests/eval/test_aime_signed_answers.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import pytest
+
+from eval.aime.bench import extract_answer_with_source, score_rollout
+
+pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("The answer is -1", "-1"),
+        ("The answer is -42", "-42"),
+        ("The answer is -042", "-42"),
+        ("The answer is -999", "-999"),
+        ("The answer is $-42$", "-42"),
+        ("The answer is (-42)", "-42"),
+        ("I considered 42, but the final answer is -42", "-42"),
+        ("<think>The answer might be 42</think>The answer is -42", "-42"),
+        ("The answer is 42", "42"),
+        ("The answer is +42", "42"),
+        ("The answer is 042", "42"),
+        ("The answer is 0", "0"),
+    ],
+)
+def test_signed_tail_number_scoring(text: str, expected: str) -> None:
+    assert extract_answer_with_source(text) == (expected, "tail_number")
+    row = score_rollout({"raw_text": text}, {"answer": expected.lstrip("-")})
+    assert row["correct"] is (not expected.startswith("-"))
+    assert row["raw_text"] == text
+
+
+@pytest.mark.parametrize("text", ["The answer is -42.5", "The answer is -0.042", "The answer is -1234"])
+def test_signed_tail_rejects_decimals_and_long_numbers(text: str) -> None:
+    assert extract_answer_with_source(text) == (None, "none")
+
+
+@pytest.mark.parametrize("answer", ["-42", "42"])
+def test_boxed_answer_still_takes_priority(answer: str) -> None:
+    text = rf"The answer is \boxed{{{answer}}}. Another number is -17"
+    assert extract_answer_with_source(text) == (answer, "boxed")