From 6eab83a26f47374d9cc643f6b2c92909b3033eee Mon Sep 17 00:00:00 2001 From: determine123 Date: Tue, 20 Oct 2026 22:11:49 +0800 Subject: [PATCH 1/2] fix(eval): discard failed benchmark module imports --- eval/common/loader.py | 6 ++- tests/eval/test_bench_loader.py | 93 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 tests/eval/test_bench_loader.py diff --git a/eval/common/loader.py b/eval/common/loader.py index 300c20d..b4dd592 100644 --- a/eval/common/loader.py +++ b/eval/common/loader.py @@ -21,5 +21,9 @@ def load_bench(name: str) -> ModuleType: spec = importlib.util.spec_from_file_location(mod_name, path) module = importlib.util.module_from_spec(spec) sys.modules[mod_name] = module - spec.loader.exec_module(module) + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(mod_name, None) + raise return module diff --git a/tests/eval/test_bench_loader.py b/tests/eval/test_bench_loader.py new file mode 100644 index 0000000..e0fc2a8 --- /dev/null +++ b/tests/eval/test_bench_loader.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import importlib +import sys +from collections.abc import Generator +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = [pytest.mark.ci, pytest.mark.ci_npu] + + +@pytest.fixture +def loader(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[ModuleType]: + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "eval")) + module = importlib.import_module("common.loader") + monkeypatch.setattr(module, "EVAL_DIR", tmp_path) + saved = {f"eval_bench_{name}": sys.modules.pop(f"eval_bench_{name}", None) for name in module.SUPPORTED} + try: + yield module + finally: + for name, previous in saved.items(): + sys.modules.pop(name, None) + if previous is not None: + sys.modules[name] = previous + + +def write_bench(loader: ModuleType, name: str, source: str) -> None: + folder = loader.EVAL_DIR / name + folder.mkdir() + (folder / "bench.py").write_text(source, encoding="utf-8") + + +@pytest.mark.parametrize("error_type", [RuntimeError, SystemExit, KeyboardInterrupt]) +def test_failed_load_can_be_retried(loader: ModuleType, monkeypatch: pytest.MonkeyPatch, error_type: type[BaseException]) -> None: + write_bench( + loader, + "gpqa", + "import os\n" + "if os.environ.get('DEEPINFER_TEST_BENCH_READY') != '1':\n" + f" raise {error_type.__name__}('initialization failed')\n" + "READY = True\n", + ) + monkeypatch.delenv("DEEPINFER_TEST_BENCH_READY", raising=False) + with pytest.raises(error_type, match="initialization failed"): + loader.load_bench("gpqa") + assert "eval_bench_gpqa" not in sys.modules + monkeypatch.setenv("DEEPINFER_TEST_BENCH_READY", "1") + bench = loader.load_bench("gpqa") + assert bench.READY is True + assert loader.load_bench("gpqa") is bench + + +def test_retry_after_dependency_becomes_available(loader: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + dependency_name = "_deepinfer_test_bench_dependency" + monkeypatch.delitem(sys.modules, dependency_name, raising=False) + write_bench(loader, "hle", f"import {dependency_name}\nREADY = True\n") + with pytest.raises(ModuleNotFoundError, match=dependency_name): + loader.load_bench("hle") + monkeypatch.setitem(sys.modules, dependency_name, ModuleType(dependency_name)) + assert loader.load_bench("hle").READY is True + + +def test_successful_load_preserves_dataclass_registration_and_cache(loader: ModuleType) -> None: + write_bench( + loader, + "aime", + "from __future__ import annotations\nfrom dataclasses import dataclass\n@dataclass\nclass Problem:\n answer: str = '42'\n", + ) + bench = loader.load_bench("aime") + assert bench.Problem().answer == "42" + assert sys.modules[bench.Problem.__module__] is bench + assert loader.load_bench("aime") is bench + + +def test_failed_benchmark_does_not_remove_another_cached_module(loader: ModuleType) -> None: + write_bench(loader, "aime", "READY = True\n") + write_bench(loader, "gpqa", "raise RuntimeError('broken')\n") + aime = loader.load_bench("aime") + with pytest.raises(RuntimeError, match="broken"): + loader.load_bench("gpqa") + assert loader.load_bench("aime") is aime + + +def test_unsupported_benchmark_is_rejected(loader: ModuleType) -> None: + with pytest.raises(SystemExit, match="benchmark not wired up"): + loader.load_bench("unsupported") + + +def test_missing_benchmark_file_is_rejected(loader: ModuleType) -> None: + with pytest.raises(SystemExit, match="benchmark implementation not found"): + loader.load_bench("gpqa") From aad408280e0ec5f9b77e86c9502189dacb8eadcd Mon Sep 17 00:00:00 2001 From: determine123 Date: Tue, 27 Oct 2026 11:11:02 +0800 Subject: [PATCH 2/2] test(eval): verify scoring recovers after benchmark import failure --- tests/eval/test_bench_loader.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/eval/test_bench_loader.py b/tests/eval/test_bench_loader.py index e0fc2a8..c91daa6 100644 --- a/tests/eval/test_bench_loader.py +++ b/tests/eval/test_bench_loader.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib +import json import sys from collections.abc import Generator from pathlib import Path @@ -91,3 +92,29 @@ def test_unsupported_benchmark_is_rejected(loader: ModuleType) -> None: def test_missing_benchmark_file_is_rejected(loader: ModuleType) -> None: with pytest.raises(SystemExit, match="benchmark implementation not found"): loader.load_bench("gpqa") + + +def test_scoring_can_retry_after_benchmark_import_failure(loader: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + score = importlib.import_module("score") + dependency = "_deepinfer_score_retry_dependency" + monkeypatch.delitem(sys.modules, dependency, raising=False) + write_bench(loader, "gpqa", f"import {dependency}\nfrom gpqa.bench import *\n") + data = tmp_path / "questions.jsonl" + data.write_text(json.dumps({"task_id": "q1", "question": "Choose B.", "answer": "B"}) + "\n", encoding="utf-8") + output = tmp_path / "run" + output.mkdir() + (output / "meta.json").write_text(json.dumps({"bench": "gpqa", "data_file": str(data), "n_sample": 1}), encoding="utf-8") + (output / "raw_rank0.jsonl").write_text( + json.dumps({"task_id": "q1", "sample_idx": 0, "raw_text": "ANSWER: B"}) + "\n", encoding="utf-8" + ) + with pytest.raises(ModuleNotFoundError, match=dependency): + score.score_run(str(output), judge="rule", verbose=False) + assert not (output / "summary.json").exists() + monkeypatch.setitem(sys.modules, dependency, ModuleType(dependency)) + assert score.score_run(str(output), judge="rule", verbose=False) == 0 + row = json.loads((output / "scored.jsonl").read_text(encoding="utf-8")) + summary = json.loads((output / "summary.json").read_text(encoding="utf-8")) + assert row["pred"] == "B" + assert row["correct"] is True + assert summary["coverage_complete"] is True + assert summary["num_rollouts"] == 1