mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-10-30 07:47:10 +08:00
47 lines
1.8 KiB
Python
47 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
|
|
pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
|
|
|
|
|
|
@pytest.fixture
|
|
def gpqa(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
|
|
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "eval"))
|
|
return importlib.import_module("gpqa.bench")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "expected"),
|
|
[
|
|
("<think>\nANSWER: A\n</think>\n\\boxed{B}", ("B", "boxed")),
|
|
("<think>\nANSWER: A\n</think>\nB", ("B", "tail_letter")),
|
|
("<think>\\boxed{A}</think>\nB", ("B", "tail_letter")),
|
|
("<think>\nANSWER: A\n</think>\nI cannot answer this question.", (None, "none")),
|
|
("<think>\\boxed{A}</think>", (None, "none")),
|
|
("<think>\nANSWER: A\n</think>\nANSWER: C", ("C", "answer_line")),
|
|
("<think>\nANSWER: A\n</think>\nANSWER: B\n</think>\n\\boxed{D}", ("D", "boxed")),
|
|
("ANSWER: A\nANSWER: B", ("B", "answer_line")),
|
|
("\\boxed{C}", ("C", "boxed")),
|
|
("The final choice is D.", ("D", "tail_letter")),
|
|
("ANSWER: B\n\\boxed{C}", ("B", "answer_line")),
|
|
("", (None, "none")),
|
|
],
|
|
)
|
|
def test_extract_answer_ignores_thinking(gpqa: ModuleType, text: str, expected: tuple[str | None, str]) -> None:
|
|
assert gpqa.extract_answer_with_source(text) == expected
|
|
|
|
|
|
@pytest.mark.parametrize(("gold", "correct"), [("A", False), ("B", True)])
|
|
def test_score_rollout_uses_final_answer(gpqa: ModuleType, gold: str, correct: bool) -> None:
|
|
rollout = {"task_id": "gpqa-test", "sample_idx": 0, "raw_text": "<think>\nANSWER: A\n</think>\n\\boxed{B}"}
|
|
row = gpqa.score_rollout(rollout, {"answer": gold})
|
|
assert row["pred"] == "B"
|
|
assert row["extract_source"] == "boxed"
|
|
assert row["correct"] is correct
|
|
assert row["raw_text"] == rollout["raw_text"]
|