mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-10-31 00:36:20 +08:00
80 lines
2.5 KiB
Python
80 lines
2.5 KiB
Python
"""`--limit` must be a positive integer: 0 and negative values used to evaluate one problem."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
|
|
import pytest
|
|
|
|
pytestmark = [pytest.mark.ci, pytest.mark.ci_npu]
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
EVAL_DIR = REPO / "eval"
|
|
RUN_EVAL = EVAL_DIR / "run_eval.py"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _drop_run_eval_module() -> Iterator[None]:
|
|
"""eval/run_eval.py parses argv at import time; never let a parsed module leak between tests."""
|
|
sys.modules.pop("run_eval", None)
|
|
yield
|
|
sys.modules.pop("run_eval", None)
|
|
|
|
|
|
def _import_with_argv(monkeypatch: pytest.MonkeyPatch, *argv: str) -> ModuleType:
|
|
# Drop the cached module so the import below re-executes the module body, including parse_args().
|
|
sys.modules.pop("run_eval", None)
|
|
monkeypatch.setattr(sys, "argv", ["run_eval.py", *argv])
|
|
monkeypatch.syspath_prepend(str(EVAL_DIR))
|
|
import run_eval # noqa: PLC0415
|
|
|
|
return run_eval
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["0", "-1", "-42"])
|
|
def test_non_positive_limit_is_rejected(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
value: str,
|
|
) -> None:
|
|
with pytest.raises(SystemExit) as excinfo:
|
|
_import_with_argv(monkeypatch, "--data-file", "unused.jsonl", "--limit", value)
|
|
assert excinfo.value.code == 2
|
|
stderr = capsys.readouterr().err
|
|
assert "argument --limit" in stderr
|
|
assert "expected a positive integer" in stderr
|
|
|
|
|
|
def test_positive_limit_and_default_are_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
assert _import_with_argv(monkeypatch, "--data-file", "unused.jsonl", "--limit", "3").args.limit == 3
|
|
assert _import_with_argv(monkeypatch, "--data-file", "unused.jsonl").args.limit is None
|
|
|
|
|
|
def test_rejection_happens_before_any_run_artifact(tmp_path: Path) -> None:
|
|
"""The CLI exits while parsing, before main() creates the output dir or launches the engine."""
|
|
out_dir = tmp_path / "out"
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(RUN_EVAL),
|
|
"--bench",
|
|
"aime",
|
|
"--data-file",
|
|
str(tmp_path / "unused.jsonl"),
|
|
"--out-dir",
|
|
str(out_dir),
|
|
"--limit",
|
|
"0",
|
|
],
|
|
cwd=str(REPO),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert proc.returncode == 2
|
|
assert "argument --limit: expected a positive integer" in proc.stderr
|
|
assert not out_dir.exists()
|