diff --git a/eval/gpqa/bench.py b/eval/gpqa/bench.py index 0ba4ae8..67789b4 100644 --- a/eval/gpqa/bench.py +++ b/eval/gpqa/bench.py @@ -109,7 +109,7 @@ def build_messages(problem: Problem) -> list[dict[str, Any]]: # --------------------------------------------------------------------------- # _ANSWER_LINE_RE = re.compile(r"(?im)^\s*ANSWER\s*[::]\s*\(?([A-D])\b") -_BOXED_RE = re.compile(r"\\boxed\{\s*\(?([A-D])\)?\s*\}") +_BOXED_RE = re.compile(r"\\boxed\{\s*\(?([A-D])\)?\s*\}", re.I) _STANDALONE_RE = re.compile(r"(? ModuleType: + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "eval")) + return importlib.import_module("gpqa.bench") + + +@pytest.mark.parametrize("text", [r"\boxed{d}", r"\boxed{D}"]) +def test_extract_answer_normalizes_boxed_case(gpqa: ModuleType, text: str) -> None: + assert gpqa.extract_answer_with_source(text) == ("D", "boxed") + + +def test_extract_answer_uses_last_boxed(gpqa: ModuleType) -> None: + assert gpqa.extract_answer_with_source(r"\boxed{a} then \boxed{c}") == ("C", "boxed") + + +def test_extract_answer_preserves_answer_line_behavior(gpqa: ModuleType) -> None: + assert gpqa.extract_answer_with_source("answer: d") == ("D", "answer_line") + + +@pytest.mark.parametrize("text", ["a", ""]) +def test_extract_answer_does_not_widen_lowercase_fallback(gpqa: ModuleType, text: str) -> None: + assert gpqa.extract_answer_with_source(text) == (None, "none") + + +def test_score_rollout_accepts_lowercase_boxed(gpqa: ModuleType) -> None: + rollout = {"task_id": "gpqa-test", "sample_idx": 0, "raw_text": r"\boxed{d}"} + + row = gpqa.score_rollout(rollout, {"answer": "D"}) + + assert row["pred"] == "D" + assert row["extract_source"] == "boxed" + assert row["correct"] is True