diff --git a/tests/eval/test_bench_loader.py b/tests/eval/test_bench_loader.py index e0fc2a8..c91daa6 100644 --- a/tests/eval/test_bench_loader.py +++ b/tests/eval/test_bench_loader.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib +import json import sys from collections.abc import Generator from pathlib import Path @@ -91,3 +92,29 @@ def test_unsupported_benchmark_is_rejected(loader: ModuleType) -> None: def test_missing_benchmark_file_is_rejected(loader: ModuleType) -> None: with pytest.raises(SystemExit, match="benchmark implementation not found"): loader.load_bench("gpqa") + + +def test_scoring_can_retry_after_benchmark_import_failure(loader: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + score = importlib.import_module("score") + dependency = "_deepinfer_score_retry_dependency" + monkeypatch.delitem(sys.modules, dependency, raising=False) + write_bench(loader, "gpqa", f"import {dependency}\nfrom gpqa.bench import *\n") + data = tmp_path / "questions.jsonl" + data.write_text(json.dumps({"task_id": "q1", "question": "Choose B.", "answer": "B"}) + "\n", encoding="utf-8") + output = tmp_path / "run" + output.mkdir() + (output / "meta.json").write_text(json.dumps({"bench": "gpqa", "data_file": str(data), "n_sample": 1}), encoding="utf-8") + (output / "raw_rank0.jsonl").write_text( + json.dumps({"task_id": "q1", "sample_idx": 0, "raw_text": "ANSWER: B"}) + "\n", encoding="utf-8" + ) + with pytest.raises(ModuleNotFoundError, match=dependency): + score.score_run(str(output), judge="rule", verbose=False) + assert not (output / "summary.json").exists() + monkeypatch.setitem(sys.modules, dependency, ModuleType(dependency)) + assert score.score_run(str(output), judge="rule", verbose=False) == 0 + row = json.loads((output / "scored.jsonl").read_text(encoding="utf-8")) + summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) + assert row["pred"] == "B" + assert row["correct"] is True + assert summary["coverage_complete"] is True + assert summary["num_rollouts"] == 1