Merge 4bda76dda5b9158380f2b89eecad68fa4d1614e4 into 553741557a57df44a46fb3ab72b6aafeaf945337

eval/gpqa/bench.py

@@ -109,7 +109,7 @@ def build_messages(problem: Problem) -> list[dict[str, Any]]:
 # --------------------------------------------------------------------------- #
 
 _ANSWER_LINE_RE = re.compile(r"(?im)^\s*ANSWER\s*[:：]\s*\(?([A-D])\b")
-_BOXED_RE = re.compile(r"\\boxed\{\s*\(?([A-D])\)?\s*\}")
+_BOXED_RE = re.compile(r"\\boxed\{\s*\(?([A-D])\)?\s*\}", re.I)
 _STANDALONE_RE = re.compile(r"(?<![A-Za-z])([A-D])(?![A-Za-z])")
 
 

tests/eval/test_gpqa_boxed_case.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
+
+
+@pytest.fixture
+def gpqa(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+    monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "eval"))
+    return importlib.import_module("gpqa.bench")
+
+
+@pytest.mark.parametrize("text", [r"\boxed{d}", r"\boxed{D}"])
+def test_extract_answer_normalizes_boxed_case(gpqa: ModuleType, text: str) -> None:
+    assert gpqa.extract_answer_with_source(text) == ("D", "boxed")
+
+
+def test_extract_answer_uses_last_boxed(gpqa: ModuleType) -> None:
+    assert gpqa.extract_answer_with_source(r"\boxed{a} then \boxed{c}") == ("C", "boxed")
+
+
+def test_extract_answer_preserves_answer_line_behavior(gpqa: ModuleType) -> None:
+    assert gpqa.extract_answer_with_source("answer: d") == ("D", "answer_line")
+
+
+@pytest.mark.parametrize("text", ["a", ""])
+def test_extract_answer_does_not_widen_lowercase_fallback(gpqa: ModuleType, text: str) -> None:
+    assert gpqa.extract_answer_with_source(text) == (None, "none")
+
+
+def test_score_rollout_accepts_lowercase_boxed(gpqa: ModuleType) -> None:
+    rollout = {"task_id": "gpqa-test", "sample_idx": 0, "raw_text": r"\boxed{d}"}
+
+    row = gpqa.score_rollout(rollout, {"answer": "D"})
+
+    assert row["pred"] == "D"
+    assert row["extract_source"] == "boxed"
+    assert row["correct"] is True