add qwen_vl_processor unittest

Author: lddfym

fastdeploy/input/qwen_mm_processor/process.py

@@ -101,6 +101,21 @@ def __init__(
         }
 
     def _pack_outputs(self, outputs):
+        """
+        Pack and convert all output data into numpy arrays with appropriate types.
+
+        Args:
+            outputs (dict): Dictionary containing model outputs with keys:
+                - images: List of visual features
+                - grid_thw: List of spatial dimensions
+                - image_type_ids: List of content type indicators
+                - input_ids: List of token IDs
+                - token_type_ids: List of type identifiers
+                - position_ids: List of position embeddings
+
+        Returns:
+            dict: Processed outputs with all values converted to numpy arrays
+        """
         # Process visual outputs - stack if exists or set to None if empty
         if not outputs["images"]:
             outputs["images"] = None  # No images case
@@ -188,6 +203,21 @@ def text2ids(self, text, images=None, videos=None):
 
         return self._pack_outputs(outputs)
 
+    def _parse_chat_messages(self, request):
+        """
+        Parse chat messages from request into structured format.
+
+        Args:
+            request (dict): Input request containing chat messages
+
+        Returns:
+            list: Parsed list of message dictionaries with:
+                - role (str): Message role (user/assistant)
+                - content (str): Message text content
+                - images (list, optional): List of image data if present
+        """
+        return parse_chat_messages(request.get("messages"))
+
     def request2ids(
         self, request: Dict[str, Any], tgts: List[str] = None
     ) -> Dict[str, Union[np.ndarray, List[np.ndarray], None]]:
@@ -218,7 +248,7 @@ def request2ids(
         }
 
         # Parse and validate chat messages
-        messages = parse_chat_messages(request.get("messages"))
+        messages = self._parse_chat_messages(request)
         image_message_list = []  # Store visual content messages
 
         for msg in messages:
@@ -234,11 +264,14 @@ def request2ids(
             for item in content_items:
                 if isinstance(item, dict) and item.get("type") in ["image", "video"]:
                     image_message_list.append(item)
+
+        raw_messages = request["messages"]
         request["messages"] = messages
 
         prompt_token_ids = self.apply_chat_template(request)
         if len(prompt_token_ids) == 0:
             raise ValueError("Invalid input: prompt_token_ids must be a non-empty sequence of token IDs")
+        request["messages"] = raw_messages
 
         vision_start_index = 0
         vision_message_index = 0
@@ -376,17 +409,17 @@ def _compute_vision_positions(
         self, start_pos: int, t: int, h: int, w: int, second_per_grid_t: float
     ) -> np.ndarray:
         """
-        Generate 3D positional embeddings for visual content.
+        Generate 3D position IDs for visual inputs.
 
         Args:
-            start_pos: Starting position index
-            t: Temporal dimension (frames)
+            start_pos: Base position in sequence
+            t: Temporal patches (1 for images)
             h: Height in patches
             w: Width in patches
-            second_per_grid_t: Seconds per temporal grid
+            second_per_grid_t: Time per temporal patch
 
         Returns:
-            numpy.ndarray: 3D position IDs shaped (3, t*h*w)
+            np.ndarray: Position IDs for [t,h,w] dimensions
         """
         h //= self.spatial_conv_size
         w //= self.spatial_conv_size
@@ -478,6 +511,7 @@ def apply_chat_template(self, request):
             add_generation_prompt=request.get("add_generation_prompt", True),
         )
         prompt_token_str = raw_prompt.replace(self.image_token, "").replace(self.video_token, "")
+        request["text_after_process"] = raw_prompt
 
         tokens = self.tokenizer.tokenize(prompt_token_str)
         token_ids = self.tokenizer.convert_tokens_to_ids(tokens)

fastdeploy/input/qwen_vl_processor.py

@@ -24,19 +24,18 @@
 
 class QwenVLProcessor(TextProcessor):
     """
-    Processor for Qwen Vision-Language models that handles multimodal inputs.
+    Qwen Vision-Language processor for handling multimodal inputs.
 
-    Inherits from ErnieProcessor and extends functionality for:
+    This processor extends TextProcessor to support:
     - Image and video processing
-    - Multimodal request handling
-    - Generation configuration
+    - Multimodal feature extraction
+    - Tokenization and position encoding
+    - Request processing and model input generation
 
     Attributes:
-        processor: Underlying DataProcessor instance
-        tokenizer: Text tokenizer
-        generation_config: Model generation configuration
-        eos_token_ids: End-of-sequence token IDs
-        limit_mm_per_prompt: Limits for multimodal inputs
+        processor (DataProcessor): Underlying data processor instance
+        tokenizer: Text tokenizer instance
+        limit_mm_per_prompt (dict): Limits for multimodal inputs per prompt
     """
 
     def __init__(
@@ -49,14 +48,15 @@ def __init__(
         tool_parser_obj=None,
     ):
         """
-        Initialize QwenVLProcessor.
+        Initialize QwenVLProcessor instance.
 
         Args:
-            config: Model configuration
-            model_name_or_path: Path to pretrained model
-            limit_mm_per_prompt: Limits for multimodal inputs per prompt
-            mm_processor_kwargs: Additional kwargs for multimodal processor
-            reasoning_parser_obj: Optional reasoning parser
+            config: Model configuration object
+            model_name_or_path (str): Pretrained model name or path
+            limit_mm_per_prompt (dict, optional): Limits for multimodal inputs
+            mm_processor_kwargs (dict, optional): Multimodal processor arguments
+            reasoning_parser_obj: Reasoning parser instance
+            tool_parser_obj: Tool parser instance
         """
         super().__init__(model_name_or_path, reasoning_parser_obj, tool_parser_obj)
 
@@ -73,12 +73,12 @@ def __init__(
 
     def process_request(self, request, max_model_len=None, **kwargs):
         """
-        Process incoming request into model inputs.
+        Process incoming request and generate model inputs.
 
         Args:
             request: Input request object
-            max_model_len: Maximum model context length
-            **kwargs: Additional processing arguments
+            max_model_len (int, optional): Maximum context length
+            **kwargs: Additional processing parameters
 
         Returns:
             Request: Processed request with model inputs
@@ -92,16 +92,16 @@ def process_request(self, request, max_model_len=None, **kwargs):
 
     def _parse_processor_kwargs(self, kwargs):
         """
-        Parse and validate multimodal processor kwargs.
+        Parse and validate multimodal processor arguments.
 
         Args:
-            kwargs: Input kwargs dictionary
+            kwargs (dict): Processor configuration arguments
 
         Returns:
-            dict: Validated processor kwargs
+            dict: Validated processor arguments
 
         Raises:
-            ValueError: If kwargs format is invalid
+            ValueError: If arguments format is invalid
         """
         if not kwargs:
             return {}
@@ -134,7 +134,7 @@ def _parse_limits(self, limits):
         Parse and validate multimodal input limits.
 
         Args:
-            limits: Input limits dictionary
+            limits (dict): Input limits configuration
 
         Returns:
             dict: Validated limits with defaults
@@ -161,7 +161,7 @@ def _check_mm_limits(self, item):
         Validate multimodal inputs against configured limits.
 
         Args:
-            item: Input request item to check
+            item: Input request item to validate
 
         Raises:
             ValueError: If input exceeds configured limits
@@ -176,9 +176,9 @@ def _check_mm_limits(self, item):
             for message in item:
                 if isinstance(message.get("content"), list):
                     for part in message["content"]:
-                        if part.get("type") == "image":
+                        if part.get("type") in ["image_url", "image"]:
                             mm_data["image"].append(part)
-                        elif part.get("type") == "video":
+                        elif part.get("type") in ["video_url", "video"]:
                             mm_data["video"].append(part)
 
         for modality, data in mm_data.items():
@@ -192,8 +192,8 @@ def process_request_dict(self, request, max_model_len=None):
         Process request dictionary into model inputs.
 
         Args:
-            request: Input request dictionary
-            max_model_len: Maximum model context length
+            request (dict): Input request dictionary
+            max_model_len (int, optional): Maximum context length
 
         Returns:
             dict: Processed request with model inputs
@@ -253,6 +253,13 @@ def process_request_dict(self, request, max_model_len=None):
         return request
 
     def append_generated_tokens(self, outputs, generated_token_ids):
+        """
+        Append generated tokens to existing outputs.
+
+        Args:
+            outputs: Current model outputs
+            generated_token_ids: Generated tokens to append
+        """
         out = {"input_ids": [], "token_type_ids": [], "position_ids": [], "cur_position": outputs["cur_position"]}
         self.processor._add_text(generated_token_ids, out)
 
@@ -263,11 +270,20 @@ def append_generated_tokens(self, outputs, generated_token_ids):
             [outputs["token_type_ids"], np.array(out["token_type_ids"], dtype=np.int64)], axis=0
         )
         outputs["position_ids"] = np.concatenate(
-            [outputs["position_ids"], out["position_ids"]], axis=1, dtype=np.int64
+            [outputs["position_ids"], out["position_ids"][0]], axis=1, dtype=np.int64
         )
         outputs["cur_position"] = out["cur_position"]
 
     def pack_outputs(self, outputs):
+        """
+        Prepare final output dictionary for model.
+
+        Args:
+            outputs: Intermediate processing outputs
+
+        Returns:
+            dict: Packed output dictionary with all required fields
+        """
         outputs["image_patch_id"] = self.processor.image_token_id
         outputs["video_patch_id"] = self.processor.video_token_id
         outputs["position_ids"] = outputs["position_ids"].transpose(1, 0)