Merge 17cdd0d852455dedc8546ee7b5ab444877b3f135 into 553741557a57df44a46fb3ab72b6aafeaf945337

deepinfer/environ.py

@@ -7,10 +7,17 @@ from datetime import timedelta
 from typing import Any
 
 DEFAULT_PREFILL_MAX_SUM_SEQLEN_K = 2**30
+_FALSE = {"0", "", "false"}
+_TRUE = {"1", "true"}
 
 
 def _bool(v: str | None, default: str = "0") -> bool:
-    return (v if v is not None else default) not in ("0", "", "false", "False")
+    value = (v if v is not None else default).lower()
+    if value in _FALSE:
+        return False
+    if value in _TRUE:
+        return True
+    raise ValueError("expected one of '0', '1', 'false', 'true' (case-insensitive)")
 
 
 def _int(v: str | None, default: int = 0) -> int:

eval/aime/bench.py

@@ -89,7 +89,8 @@ def extract_boxed(text: str) -> str | None:
     return None
 
 
-_NUMBER_RE = re.compile(r"(?<![\w.])(\d{1,3})(?![\w.])")
+# A period followed by a digit is a decimal separator; a sentence-ending period is allowed.
+_NUMBER_RE = re.compile(r"(?<![\w.])(\d{1,3})(?!\w|\.\d)")
 
 
 def normalize_answer(ans: str | None) -> str | None:

tests/eval/test_aime.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import pytest
+
+from eval.aime.bench import extract_answer_with_source, score_rollout
+
+pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "The answer is 42",
+        "The answer is 42.",
+        "The answer is 42. This is the final answer.",
+        "The answer is 42.\n",
+        "The answer is 42!",
+        "The answer is 42...",
+        "The answer is 042.",
+        "<think>I first tried 17.</think>The answer is 42.",
+    ],
+)
+def test_tail_number_with_sentence_punctuation(text: str) -> None:
+    assert extract_answer_with_source(text) == ("42", "tail_number")
+    row = score_rollout({"raw_text": text}, {"answer": "42"})
+    assert row["correct"] is True
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "The answer is 42.5",
+        "The answer is 42.5.",
+        "The answer is .42",
+        "The answer is 0.042.",
+        "The answer is 1234.",
+        "The answer is x42.",
+        "The answer is 42x.",
+        "There is no answer.",
+    ],
+)
+def test_tail_number_rejects_non_integer_tokens(text: str) -> None:
+    assert extract_answer_with_source(text) == (None, "none")
+
+
+def test_boxed_answer_takes_priority_over_tail_number() -> None:
+    assert extract_answer_with_source(r"The answer is \boxed{17}. I rejected 42.") == ("17", "boxed")