ci: simplify multi-modality tests by using mixins (sgl-project#9006)

Author: mickqian

python/sglang/srt/multimodal/processors/base_processor.py

@@ -217,9 +217,9 @@ def process_mm_data(
         if videos:
             kwargs["videos"] = videos
         if audios:
-            if self.arch in {
-                "Gemma3nForConditionalGeneration",
-                "Qwen2AudioForConditionalGeneration",
+            if self._processor.__class__.__name__ in {
+                "Gemma3nProcessor",
+                "Qwen2AudioProcessor",
             }:
                 # Note(Xinyuan): for gemma3n, ref: https://github.com/huggingface/transformers/blob/ccf2ca162e33f381e454cdb74bf4b41a51ab976d/src/transformers/models/gemma3n/processing_gemma3n.py#L107
                 kwargs["audio"] = audios

python/sglang/srt/multimodal/processors/llava.py

@@ -18,7 +18,7 @@
 from sglang.srt.models.mistral import Mistral3ForConditionalGeneration
 from sglang.srt.multimodal.mm_utils import expand2square, process_anyres_image
 from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
-from sglang.srt.utils import load_image, logger
+from sglang.srt.utils import ImageData, load_image, logger
 from sglang.utils import get_exception_traceback
 
 
@@ -35,7 +35,7 @@ def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
 
     @staticmethod
     def _process_single_image_task(
-        image_data: Union[str, bytes],
+        image_data: Union[str, bytes, ImageData],
         image_aspect_ratio: Optional[str] = None,
         image_grid_pinpoints: Optional[str] = None,
         processor=None,
@@ -44,18 +44,19 @@ def _process_single_image_task(
         image_processor = processor.image_processor
 
         try:
-            image, image_size = load_image(image_data)
+            url = image_data.url if isinstance(image_data, ImageData) else image_data
+            image, image_size = load_image(url)
             if image_size is not None:
                 # It is a video with multiple images
-                image_hash = hash(image_data)
+                image_hash = hash(url)
                 pixel_values = image_processor(image)["pixel_values"]
                 for _ in range(len(pixel_values)):
                     pixel_values[_] = pixel_values[_].astype(np.float16)
                 pixel_values = np.stack(pixel_values, axis=0)
                 return pixel_values, image_hash, image_size
             else:
                 # It is an image
-                image_hash = hash(image_data)
+                image_hash = hash(url)
                 if image_aspect_ratio == "pad":
                     image = expand2square(
                         image,
@@ -82,7 +83,10 @@ def _process_single_image_task(
             logger.error("Exception in TokenizerManager:\n" + get_exception_traceback())
 
     async def _process_single_image(
-        self, image_data: Union[bytes, str], aspect_ratio: str, grid_pinpoints: str
+        self,
+        image_data: Union[bytes, str, ImageData],
+        aspect_ratio: str,
+        grid_pinpoints: str,
     ):
         if self.cpu_executor is not None:
             loop = asyncio.get_event_loop()
@@ -104,7 +108,7 @@ async def _process_single_image(
 
     async def process_mm_data_async(
         self,
-        image_data: List[Union[str, bytes]],
+        image_data: List[Union[str, bytes, ImageData]],
         input_text,
         request_obj,
         *args,

test/srt/run_suite.py

@@ -110,8 +110,8 @@ class TestFile:
         TestFile("test_utils_update_weights.py", 48),
         TestFile("test_vision_chunked_prefill.py", 175),
         TestFile("test_vlm_input_format.py", 300),
-        TestFile("test_vision_openai_server_a.py", 989),
-        TestFile("test_vision_openai_server_b.py", 620),
+        TestFile("test_vision_openai_server_a.py", 403),
+        TestFile("test_vision_openai_server_b.py", 446),
     ],
     "per-commit-2-gpu": [
         TestFile("lora/test_lora_tp.py", 116),

test/srt/test_vision_openai_server_a.py

@@ -8,16 +8,28 @@
 
 from test_vision_openai_server_common import *
 
-from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
     popen_launch_server,
 )
 
 
-class TestQwen2VLServer(TestOpenAIVisionServer):
+class TestLlava(ImageOpenAITestMixin):
+    @classmethod
+    def setUpClass(cls):
+        cls.model = "lmms-lab/llava-onevision-qwen2-0.5b-ov"
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.api_key = "sk-123456"
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            api_key=cls.api_key,
+        )
+        cls.base_url += "/v1"
+
+
+class TestQwen2VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "Qwen/Qwen2-VL-7B-Instruct"
@@ -37,11 +49,8 @@ def setUpClass(cls):
         )
         cls.base_url += "/v1"
 
-    def test_video_chat_completion(self):
-        self._test_video_chat_completion()
-
 
-class TestQwen2_5_VLServer(TestOpenAIVisionServer):
+class TestQwen2_5_VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "Qwen/Qwen2.5-VL-7B-Instruct"
@@ -61,9 +70,6 @@ def setUpClass(cls):
         )
         cls.base_url += "/v1"
 
-    def test_video_chat_completion(self):
-        self._test_video_chat_completion()
-
 
 class TestVLMContextLengthIssue(CustomTestCase):
     @classmethod
@@ -137,11 +143,8 @@ def test_single_image_chat_completion(self):
 #         )
 #         cls.base_url += "/v1"
 
-#     def test_video_chat_completion(self):
-#         pass
 
-
-class TestMinicpmvServer(TestOpenAIVisionServer):
+class TestMinicpmvServer(ImageOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "openbmb/MiniCPM-V-2_6"
@@ -162,7 +165,7 @@ def setUpClass(cls):
         cls.base_url += "/v1"
 
 
-class TestInternVL2_5Server(TestOpenAIVisionServer):
+class TestInternVL2_5Server(ImageOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "OpenGVLab/InternVL2_5-2B"
@@ -181,7 +184,7 @@ def setUpClass(cls):
         cls.base_url += "/v1"
 
 
-class TestMinicpmoServer(TestOpenAIVisionServer):
+class TestMinicpmoServer(ImageOpenAITestMixin, AudioOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "openbmb/MiniCPM-o-2_6"
@@ -201,12 +204,8 @@ def setUpClass(cls):
         )
         cls.base_url += "/v1"
 
-    def test_audio_chat_completion(self):
-        self._test_audio_speech_completion()
-        self._test_audio_ambient_completion()
-
 
-class TestMimoVLServer(TestOpenAIVisionServer):
+class TestMimoVLServer(ImageOpenAITestMixin):
     @classmethod
     def setUpClass(cls):
         cls.model = "XiaomiMiMo/MiMo-VL-7B-RL"
@@ -228,6 +227,95 @@ def setUpClass(cls):
         cls.base_url += "/v1"
 
 
+class TestVILAServer(ImageOpenAITestMixin):
+    @classmethod
+    def setUpClass(cls):
+        cls.model = "Efficient-Large-Model/NVILA-Lite-2B-hf-0626"
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.api_key = "sk-123456"
+        cls.revision = "6bde1de5964b40e61c802b375fff419edc867506"
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            api_key=cls.api_key,
+            other_args=[
+                "--trust-remote-code",
+                "--context-length=65536",
+                f"--revision={cls.revision}",
+                "--cuda-graph-max-bs",
+                "4",
+            ],
+        )
+        cls.base_url += "/v1"
+
+
+class TestPhi4MMServer(ImageOpenAITestMixin, AudioOpenAITestMixin):
+    @classmethod
+    def setUpClass(cls):
+        # Manually download LoRA adapter_config.json as it's not downloaded by the model loader by default.
+        from huggingface_hub import constants, snapshot_download
+
+        snapshot_download(
+            "microsoft/Phi-4-multimodal-instruct",
+            allow_patterns=["**/adapter_config.json"],
+        )
+
+        cls.model = "microsoft/Phi-4-multimodal-instruct"
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.api_key = "sk-123456"
+
+        revision = "33e62acdd07cd7d6635badd529aa0a3467bb9c6a"
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=[
+                "--trust-remote-code",
+                "--mem-fraction-static",
+                "0.70",
+                "--disable-radix-cache",
+                "--max-loras-per-batch",
+                "2",
+                "--revision",
+                revision,
+                "--lora-paths",
+                f"vision={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/vision-lora",
+                f"speech={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/speech-lora",
+                "--cuda-graph-max-bs",
+                "4",
+            ],
+        )
+        cls.base_url += "/v1"
+
+    def get_vision_request_kwargs(self):
+        return {
+            "extra_body": {
+                "lora_path": "vision",
+                "top_k": 1,
+                "top_p": 1.0,
+            }
+        }
+
+    def get_audio_request_kwargs(self):
+        return {
+            "extra_body": {
+                "lora_path": "speech",
+                "top_k": 1,
+                "top_p": 1.0,
+            }
+        }
+
+    # This _test_audio_ambient_completion test is way too complicated to pass for a small LLM
+    def test_audio_ambient_completion(self):
+        pass
+
+
 if __name__ == "__main__":
-    del TestOpenAIVisionServer
+    del (
+        TestOpenAIOmniServerBase,
+        ImageOpenAITestMixin,
+        VideoOpenAITestMixin,
+        AudioOpenAITestMixin,
+    )
     unittest.main()