Merge 38200b9e1f69d1c27430a1ad38a14cfd7768386b into 553741557a57df44a46fb3ab72b6aafeaf945337

eval/gpqa/bench.py

@@ -117,13 +117,14 @@ def extract_answer_with_source(text: str) -> tuple[str | None, str]:
     """Extract the answer letter by priority and report which path matched."""
     if not text:
         return None, "none"
-    matches = _ANSWER_LINE_RE.findall(text)
+    final_text = text.rsplit("</think>", 1)[-1]
+    matches = _ANSWER_LINE_RE.findall(final_text)
     if matches:
         return matches[-1].upper(), "answer_line"
-    matches = _BOXED_RE.findall(text)
+    matches = _BOXED_RE.findall(final_text)
     if matches:
         return matches[-1].upper(), "boxed"
-    tail = text.rsplit("</think>", 1)[-1] if "</think>" in text else text[-400:]
+    tail = final_text if "</think>" in text else text[-400:]
     matches = _STANDALONE_RE.findall(tail)
     if matches:
         return matches[-1].upper(), "tail_letter"

tests/eval/test_gpqa_answer_extraction.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
+
+
+@pytest.fixture
+def gpqa(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
+    monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "eval"))
+    return importlib.import_module("gpqa.bench")
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("<think>\nANSWER: A\n</think>\n\\boxed{B}", ("B", "boxed")),
+        ("<think>\nANSWER: A\n</think>\nB", ("B", "tail_letter")),
+        ("<think>\\boxed{A}</think>\nB", ("B", "tail_letter")),
+        ("<think>\nANSWER: A\n</think>\nI cannot answer this question.", (None, "none")),
+        ("<think>\\boxed{A}</think>", (None, "none")),
+        ("<think>\nANSWER: A\n</think>\nANSWER: C", ("C", "answer_line")),
+        ("<think>\nANSWER: A\n</think>\nANSWER: B\n</think>\n\\boxed{D}", ("D", "boxed")),
+        ("ANSWER: A\nANSWER: B", ("B", "answer_line")),
+        ("\\boxed{C}", ("C", "boxed")),
+        ("The final choice is D.", ("D", "tail_letter")),
+        ("ANSWER: B\n\\boxed{C}", ("B", "answer_line")),
+        ("", (None, "none")),
+    ],
+)
+def test_extract_answer_ignores_thinking(gpqa: ModuleType, text: str, expected: tuple[str | None, str]) -> None:
+    assert gpqa.extract_answer_with_source(text) == expected
+
+
+@pytest.mark.parametrize(("gold", "correct"), [("A", False), ("B", True)])
+def test_score_rollout_uses_final_answer(gpqa: ModuleType, gold: str, correct: bool) -> None:
+    rollout = {"task_id": "gpqa-test", "sample_idx": 0, "raw_text": "<think>\nANSWER: A\n</think>\n\\boxed{B}"}
+    row = gpqa.score_rollout(rollout, {"answer": gold})
+    assert row["pred"] == "B"
+    assert row["extract_source"] == "boxed"
+    assert row["correct"] is correct
+    assert row["raw_text"] == rollout["raw_text"]