Merge a9baf6803c96284322e2ab95a11a91d269138b99 into 553741557a57df44a46fb3ab72b6aafeaf945337

README.md

@@ -65,8 +65,12 @@ export DEEPINFER_ACCELERATOR=nvidia
 ### Offline inference
 
 The default checkpoint, [`deepseek-ai/DeepSeek-V4.1-Flash-1017`](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017), is
-downloaded into the Hugging Face cache on the first run; set `DEEPINFER_HF_MODEL_DIR=/path/to/DeepSeek-V4.1-Flash-1017` to
-run from a local copy instead.
+downloaded into the Hugging Face or ModelScope cache on the first run; set `DEEPINFER_MODEL_DIR=/path/to/DeepSeek-V4.1-Flash-1017` to
+run from a local copy instead. `DEEPINFER_HF_MODEL_DIR` is renamed to `DEEPINFER_MODEL_DIR` for more general usage
+(the legacy name still works; the new one wins when both are set), and ModelScope is supported as an alternative
+model download hub: set `DEEPINFER_USE_MODELSCOPE=1` to download models from
+[ModelScope](https://modelscope.cn/models/deepseek-ai/DeepSeek-V4.1-Flash-1017)
+(requires `pip install modelscope`).
 
 ``` bash
 python examples/offline_inference.py

examples/README.md

@@ -8,7 +8,7 @@ python examples/offline_inference.py --ep 2 --dp 8
 python examples/offline_inference.py --ep 2 --cp 8
 ```
 
-The default model is `deepseek-ai/DeepSeek-V4.1-Flash-1017`: when `--model_name` is a model id, it is downloaded to the local HF cache with `huggingface_hub.snapshot_download` and used from there; when it is a local HF directory (`config.json` plus safetensors), it is used directly with no network access. `DEEPINFER_HF_MODEL_DIR` points the default model at a local copy and does not override an explicitly passed `--model_name`. The prompts for the VL offline samples live in `_prompts.py`, the images come from `images/` in the repository, and no external VL cache is needed. See `python examples/offline_inference.py --help` for the arguments.
+The default model is `deepseek-ai/DeepSeek-V4.1-Flash-1017`: when `--model_name` is a model id, it is downloaded to the local HF cache with `huggingface_hub.snapshot_download`, or from ModelScope with `modelscope.snapshot_download` when `DEEPINFER_USE_MODELSCOPE=1` — and used from there; when it is a local HF directory (`config.json` plus safetensors), it is used directly with no network access. `DEEPINFER_HF_MODEL_DIR` is renamed to `DEEPINFER_MODEL_DIR` for more general usage (the legacy name still works, with the new one winning when both are set) and points the default model at a local copy without overriding an explicitly passed `--model_name`. The prompts for the VL offline samples live in `_prompts.py`, the images come from `images/` in the repository, and no external VL cache is needed. See `python examples/offline_inference.py --help` for the arguments.
 
 With `--check_structured_output`, offline inference reuses the same LLM instance after the sample outputs and `decode_ppl` to run the regex, JSON, and EBNF structured-output checks and to verify a normal EOS termination; any failure fails the run. `tests/models/test_offline_inference.py` enables it explicitly, validating PPL and structured output in a single run; regular offline runs, the long-PPL-only test, and baseline refreshes leave it off by default.
 

examples/_config.py

@@ -28,24 +28,36 @@ from deep_cluster.hardware import accelerator_kind, get_actual_num_gpus  # noqa:
 if TYPE_CHECKING:
     from deepinfer.entrypoints.llm import LLM
 
-# 公开发布的 HF 模型 id；本地目录用 --model_name 或 DEEPINFER_HF_MODEL_DIR 指定。
+# 公开发的 HF / MS 模型 id；本地目录用 --model_name、DEEPINFER_MODEL_DIR（或使用DEEPINFER_HF_MODEL_DIR，两者都设时DEEPINFER_MODEL_DIR优先）指定。
 DEFAULT_MODEL_NAME = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
 
 
 def resolve_model_path(model_name: str) -> str:
-    """Resolve an HF model id or explicit local HF directory, without importing deepinfer.
+    """Resolve a model id or explicit local cache directory, without importing deepinfer.
 
     A local directory must contain config.json; the runtime HF loader validates weights and
-    architecture. Anything else is an HF model id, downloaded to the HF cache, and
-    DEEPINFER_HF_MODEL_DIR points the default model at a local copy of it.
+    architecture. Anything else is a model id, downloaded from ModelScope when
+    DEEPINFER_USE_MODELSCOPE is set and from the HF cache otherwise; DEEPINFER_MODEL_DIR
+    (DEEPINFER_HF_MODEL_DIR still works, the new name wins when both are set) points
+    the default model at a local copy of it.
     """
-    if model_name == DEFAULT_MODEL_NAME and (override := os.environ.get("DEEPINFER_HF_MODEL_DIR")):
+    if model_name == DEFAULT_MODEL_NAME and (override := os.environ.get("DEEPINFER_MODEL_DIR") or os.environ.get("DEEPINFER_HF_MODEL_DIR")):
         model_name = override
     path = Path(model_name).expanduser()
     if not path.is_dir():
-        from huggingface_hub import snapshot_download
+        if os.environ.get("DEEPINFER_USE_MODELSCOPE", "false").strip().lower() in ("1", "true", "yes", "on"):
+            try:
+                from modelscope import snapshot_download
+            except ImportError as exc:
+                raise ImportError(
+                    "DEEPINFER_USE_MODELSCOPE is set but the modelscope package is not installed; "
+                    "install it with `pip install modelscope` or unset the variable to use the HF Hub"
+                ) from exc
+            print(f"[deepinfer] resolving model {model_name} from ModelScope", flush=True)
+        else:
+            from huggingface_hub import snapshot_download
 
-        print(f"[deepinfer] resolving HF model {model_name} from the HF Hub", flush=True)
+            print(f"[deepinfer] resolving HF model {model_name} from the HF Hub", flush=True)
         path = Path(snapshot_download(model_name))
     if not (path / "config.json").is_file():
         raise FileNotFoundError(f"HF checkpoint requires config.json: {path}")