[Feature] Models api

docs/parameters.md

@@ -36,7 +36,7 @@ When using FastDeploy to deploy models (including offline inference and service
 | ```static_decode_blocks``` | `int` | During inference, each request is forced to allocate corresponding number of blocks from Prefill's KVCache for Decode use, default: 2 |
 | ```reasoning_parser``` | `str` | Specify the reasoning parser to extract reasoning content from model output |
 | ```use_cudagraph``` | `bool` | Whether to use cuda graph, default: False |
-｜```graph_optimization_config```    | `str`       | Parameters related to graph optimization can be configured, with default values of'{"use_cudagraph":false, "graph_opt_level":0, "cudagraph_capture_sizes": null }' |
+|```graph_optimization_config```    | `str`       | Parameters related to graph optimization can be configured, with default values of'{"use_cudagraph":false, "graph_opt_level":0, "cudagraph_capture_sizes": null }' |
 | ```enable_custom_all_reduce``` | `bool` | Enable Custom all-reduce, default: False |
 | ```splitwise_role``` | `str` | Whether to enable splitwise inference, default value: mixed, supported parameters: ["mixed", "decode", "prefill"] |
 | ```innode_prefill_ports``` | `str` | Internal engine startup ports for prefill instances (only required for single-machine PD separation), default: None |
@@ -46,7 +46,8 @@ When using FastDeploy to deploy models (including offline inference and service
 | ```dynamic_load_weight``` | `int` | Whether to enable dynamic weight loading, default: 0 |
 | ```enable_expert_parallel``` | `bool` | Whether to enable expert parallel |
 | ```enable_logprob``` | `bool` | Whether to enable return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message.If logrpob is not used, this parameter can be omitted when starting |
-
+| ```served_model_name```| `str`| The model name used in the API. If not specified, the model name will be the same as the --model argument |
+| ```revision``` | `str` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. |
 ## 1. Relationship between KVCache allocation, ```num_gpu_blocks_override``` and ```block_size```?
 
 During FastDeploy inference, GPU memory is occupied by ```model weights```, ```preallocated KVCache blocks``` and ```model computation intermediate activation values```. The preallocated KVCache blocks are determined by ```num_gpu_blocks_override```, with ```block_size``` (default: 64) as its unit, meaning one block can store KVCache for 64 Tokens.

docs/zh/parameters.md

@@ -34,7 +34,7 @@
 | ```static_decode_blocks```         | `int`       | 推理过程中，每条请求强制从Prefill的KVCache分配对应块数给Decode使用，默认2|
 | ```reasoning_parser```             | `str`       | 指定要使用的推理解析器，以便从模型输出中提取推理内容 |
 | ```use_cudagraph```                | `bool`      | 是否使用cuda graph，默认False |
-｜```graph_optimization_config```    | `str`       | 可以配置计算图优化相关的参数，默认值为'{"use_cudagraph":false, "graph_opt_level":0, "cudagraph_capture_sizes": null }' |
+|```graph_optimization_config```    | `str`       | 可以配置计算图优化相关的参数，默认值为'{"use_cudagraph":false, "graph_opt_level":0, "cudagraph_capture_sizes": null }' |
 | ```enable_custom_all_reduce```     | `bool`      | 开启Custom all-reduce，默认False |
 | ```splitwise_role```               | `str`       | 是否开启splitwise推理，默认值mixed， 支持参数为["mixed", "decode", "prefill"] |
 | ```innode_prefill_ports```         | `str`       | prefill 实例内部引擎启动端口 （仅单机PD分离需要），默认值None |
@@ -44,6 +44,8 @@
 | ```dynamic_load_weight```          | `int`       | 是否动态加载权重，默认0 |
 | ```enable_expert_parallel```       | `bool`      | 是否启用专家并行 |
 | ```enable_logprob```       | `bool`      | 是否启用输出token返回logprob。如果未使用 logrpob，则在启动时可以省略此参数。 |
+| ```served_model_name```       | `str`      | API 中使用的模型名称，如果未指定，模型名称将与--model参数相同 |
+| ```revision```       | `str`      | 自动下载模型时，用于指定模型的Git版本，分支名或tag |
 
 ## 1. KVCache分配与```num_gpu_blocks_override```、```block_size```的关系？
 

fastdeploy/engine/args_utils.py

@@ -50,6 +50,10 @@ class EngineArgs:
     """
     The name or path of the model to be used.
     """
+    served_model_name: Optional[str] = None
+    """
+    The name of the model being served.
+    """
     revision: Optional[str] = "master"
     """
     The revision for downloading models.
@@ -367,6 +371,12 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
             default=EngineArgs.model,
             help="Model name or path to be used.",
         )
+        model_group.add_argument(
+            "--served-model-name",
+            type=nullable_str,
+            default=EngineArgs.served_model_name,
+            help="Served model name",
+        )
         model_group.add_argument(
             "--revision",
             type=nullable_str,

fastdeploy/entrypoints/openai/api_server.py

@@ -38,9 +38,11 @@
     CompletionResponse,
     ControlSchedulerRequest,
     ErrorResponse,
+    ModelList,
 )
 from fastdeploy.entrypoints.openai.serving_chat import OpenAIServingChat
 from fastdeploy.entrypoints.openai.serving_completion import OpenAIServingCompletion
+from fastdeploy.entrypoints.openai.serving_models import ModelPath, OpenAIServingModels
 from fastdeploy.metrics.metrics import (
     EXCLUDE_LABELS,
     cleanup_prometheus_files,
@@ -121,6 +123,15 @@ async def lifespan(app: FastAPI):
     else:
         pid = os.getpid()
     api_server_logger.info(f"{pid}")
+
+    if args.served_model_name is not None:
+        served_model_names = args.served_model_name
+        verification = True
+    else:
+        served_model_names = args.model
+        verification = False
+    model_paths = [ModelPath(name=served_model_names, model_path=args.model, verification=verification)]
+
     engine_client = EngineClient(
         args.model,
         args.tokenizer,
@@ -136,8 +147,26 @@ async def lifespan(app: FastAPI):
         args.workers,
     )
     app.state.dynamic_load_weight = args.dynamic_load_weight
-    chat_handler = OpenAIServingChat(engine_client, pid, args.ips, args.max_waiting_time)
-    completion_handler = OpenAIServingCompletion(engine_client, pid, args.ips, args.max_waiting_time)
+    model_handler = OpenAIServingModels(
+        model_paths,
+        args.max_model_len,
+        args.ips,
+    )
+    app.state.model_handler = model_handler
+    chat_handler = OpenAIServingChat(
+        engine_client,
+        app.state.model_handler,
+        pid,
+        args.ips,
+        args.max_waiting_time,
+    )
+    completion_handler = OpenAIServingCompletion(
+        engine_client,
+        app.state.model_handler,
+        pid,
+        args.ips,
+        args.max_waiting_time,
+    )
     engine_client.create_zmq_client(model=pid, mode=zmq.PUSH)
     engine_client.pid = pid
     app.state.engine_client = engine_client
@@ -296,6 +325,23 @@ async def create_completion(request: CompletionRequest):
         return JSONResponse(status_code=e.status_code, content={"detail": e.detail})
 
 
+@app.get("/v1/models")
+async def list_models() -> Response:
+    """
+    List all available models.
+    """
+    if app.state.dynamic_load_weight:
+        status, msg = app.state.engine_client.is_workers_alive()
+        if not status:
+            return JSONResponse(content={"error": "Worker Service Not Healthy"}, status_code=304)
+
+    models = await app.state.model_handler.list_models()
+    if isinstance(models, ErrorResponse):
+        return JSONResponse(content=models.model_dump(), status_code=models.code)
+    elif isinstance(models, ModelList):
+        return JSONResponse(content=models.model_dump())
+
+
 @app.get("/update_model_weight")
 def update_model_weight(request: Request) -> Response:
     """

fastdeploy/entrypoints/openai/protocol.py

@@ -18,6 +18,7 @@
 
 import json
 import time
+import uuid
 from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, Field, model_validator
@@ -55,6 +56,37 @@ class UsageInfo(BaseModel):
     prompt_tokens_details: Optional[PromptTokenUsageInfo] = None
 
 
+class ModelPermission(BaseModel):
+    id: str = Field(default_factory=lambda: f"modelperm-{str(uuid.uuid4().hex)}")
+    object: str = "model_permission"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    allow_create_engine: bool = False
+    allow_sampling: bool = True
+    allow_logprobs: bool = True
+    allow_search_indices: bool = False
+    allow_view: bool = True
+    allow_fine_tuning: bool = False
+    organization: str = "*"
+    group: Optional[str] = None
+    is_blocking: bool = False
+
+
+class ModelInfo(BaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "FastDeploy"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    max_model_len: Optional[int] = None
+    permission: list[ModelPermission] = Field(default_factory=list)
+
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: list[ModelInfo] = Field(default_factory=list)
+
+
 class FunctionCall(BaseModel):
     """
     Function call.

fastdeploy/entrypoints/openai/serving_chat.py

@@ -49,8 +49,9 @@ class OpenAIServingChat:
     OpenAI-style chat completions serving
     """
 
-    def __init__(self, engine_client, pid, ips, max_waiting_time):
+    def __init__(self, engine_client, models, pid, ips, max_waiting_time):
         self.engine_client = engine_client
+        self.models = models
         self.pid = pid
         self.master_ip = ips
         self.max_waiting_time = max_waiting_time
@@ -77,7 +78,11 @@ async def create_chat_completion(self, request: ChatCompletionRequest):
             err_msg = f"Only master node can accept completion request, please send request to master node: {self.pod_ips[0]}"
             api_server_logger.error(err_msg)
             return ErrorResponse(message=err_msg, code=400)
-
+        is_supported, request.model = self.models.is_supported_model(request.model)
+        if not is_supported:
+            err_msg = f"Unsupported model: {request.model}, support {', '.join([x.name for x in self.models.model_paths])} or default"
+            api_server_logger.error(err_msg)
+            return ErrorResponse(message=err_msg, code=400)
         if request.user is not None:
             request_id = f"chatcmpl-{request.user}-{uuid.uuid4()}"
         else:

fastdeploy/entrypoints/openai/serving_completion.py

@@ -40,8 +40,9 @@
 
 
 class OpenAIServingCompletion:
-    def __init__(self, engine_client, pid, ips, max_waiting_time):
+    def __init__(self, engine_client, models, pid, ips, max_waiting_time):
         self.engine_client = engine_client
+        self.models = models
         self.pid = pid
         self.master_ip = ips
         self.host_ip = get_host_ip()
@@ -67,6 +68,11 @@ async def create_completion(self, request: CompletionRequest):
             err_msg = f"Only master node can accept completion request, please send request to master node: {self.pod_ips[0]}"
             api_server_logger.error(err_msg)
             return ErrorResponse(message=err_msg, code=400)
+        is_supported, request.model = self.models.is_supported_model(request.model)
+        if not is_supported:
+            err_msg = f"Unsupported model: {request.model}, support {', '.join([x.name for x in self.models.model_paths])} or default"
+            api_server_logger.error(err_msg)
+            return ErrorResponse(message=err_msg, code=400)
         created_time = int(time.time())
         if request.user is not None:
             request_id = f"cmpl-{request.user}-{uuid.uuid4()}"

fastdeploy/entrypoints/openai/serving_models.py

@@ -0,0 +1,96 @@
+"""
+# Copyright (c) 2025  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+from dataclasses import dataclass
+from typing import List, Union
+
+from fastdeploy.entrypoints.openai.protocol import (
+    ErrorResponse,
+    ModelInfo,
+    ModelList,
+    ModelPermission,
+)
+from fastdeploy.utils import api_server_logger, get_host_ip
+
+
+@dataclass
+class ModelPath:
+    name: str
+    model_path: str
+    verification: bool = False
+
+
+class OpenAIServingModels:
+    """
+    OpenAI-style models serving
+    """
+
+    def __init__(
+        self,
+        model_paths: list[ModelPath],
+        max_model_len: int,
+        ips: Union[List[str], str],
+    ):
+        self.model_paths = model_paths
+        self.max_model_len = max_model_len
+        self.master_ip = ips
+        self.host_ip = get_host_ip()
+        if self.master_ip is not None:
+            if isinstance(self.master_ip, list):
+                self.master_ip = self.master_ip[0]
+            else:
+                self.master_ip = self.master_ip.split(",")[0]
+
+    def _check_master(self):
+        if self.master_ip is None:
+            return True
+        if self.host_ip == self.master_ip:
+            return True
+        return False
+
+    def is_supported_model(self, model_name) -> tuple[bool, str]:
+        """
+        Check whether the specified model is supported.
+        """
+        if self.model_paths[0].verification is False:
+            return True, self.model_name()
+        if model_name == "default":
+            return True, self.model_name()
+        return any(model.name == model_name for model in self.model_paths), model_name
+
+    def model_name(self) -> str:
+        """
+        Returns the current model name.
+        """
+        return self.model_paths[0].name
+
+    async def list_models(self) -> ModelList:
+        """
+        Show available models.
+        """
+        if not self._check_master():
+            err_msg = (
+                f"Only master node can accept models request, please send request to master node: {self.pod_ips[0]}"
+            )
+            api_server_logger.error(err_msg)
+            return ErrorResponse(message=err_msg, code=400)
+        model_infos = [
+            ModelInfo(
+                id=model.name, max_model_len=self.max_model_len, root=model.model_path, permission=[ModelPermission()]
+            )
+            for model in self.model_paths
+        ]
+        return ModelList(data=model_infos)

test/entrypoints/openai/test_serving_models.py

@@ -0,0 +1,51 @@
+import asyncio
+import unittest
+
+from fastdeploy.entrypoints.openai.protocol import ModelInfo, ModelList
+from fastdeploy.entrypoints.openai.serving_models import ModelPath, OpenAIServingModels
+from fastdeploy.utils import get_host_ip
+
+MODEL_NAME = "baidu/ERNIE-4.5-0.3B-PT"
+MODEL_PATHS = [ModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
+MAX_MODEL_LEN = 2048
+
+
+async def _async_serving_models_init() -> OpenAIServingModels:
+    """异步初始化 OpenAIServingModels 实例"""
+    return OpenAIServingModels(
+        model_paths=MODEL_PATHS,
+        max_model_len=MAX_MODEL_LEN,
+        ips=get_host_ip(),
+    )
+
+
+class TestOpenAIServingModels(unittest.TestCase):
+    """测试 OpenAIServingModels 的 unittest 版本"""
+
+    def test_serving_model_name(self):
+        """测试模型名称获取"""
+        # 通过 asyncio.run() 执行异步初始化
+        serving_models = asyncio.run(_async_serving_models_init())
+        self.assertEqual(serving_models.model_name(), MODEL_NAME)
+
+    def test_list_models(self):
+        """测试模型列表功能"""
+        serving_models = asyncio.run(_async_serving_models_init())
+
+        # 通过 asyncio.run() 执行异步方法
+        result = asyncio.run(serving_models.list_models())
+
+        # 验证返回类型和内容
+        self.assertIsInstance(result, ModelList)
+        self.assertEqual(len(result.data), 1)
+
+        model_info = result.data[0]
+        self.assertIsInstance(model_info, ModelInfo)
+        self.assertEqual(model_info.id, MODEL_NAME)
+        self.assertEqual(model_info.max_model_len, MAX_MODEL_LEN)
+        self.assertEqual(model_info.root, MODEL_PATHS[0].model_path)
+        self.assertEqual(result.object, "list")
+
+
+if __name__ == "__main__":
+    unittest.main()