[ViPPET][SmartNVR] Update SmartNVR Pipeline (open-edge-platform#455)

Co-authored-by: Paweł Żak <pawel.zak@intel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

tools/visual-pipeline-and-platform-evaluation-tool/app.py

@@ -667,13 +667,13 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
 
     try:
         download_file(
-            "https://github.com/intel-iot-devkit/sample-videos/raw/master/person-bicycle-car-detection.mp4",
-            "/tmp/person-bicycle-car-detection.mp4",
+            "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/people.mp4",
+            "/tmp/people.mp4",
         )
         input_video_player = gr.Video(
             label="Input Video",
             interactive=True,
-            value="/tmp/person-bicycle-car-detection.mp4",
+            value="/tmp/people.mp4",
             sources="upload",
             elem_id="input_video_player",
         )
@@ -716,7 +716,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     inferencing_channels = gr.Slider(
         minimum=0,
         maximum=30,
-        value=11,
+        value=8,
         step=1,
         label="Number of Recording + Inferencing channels",
         interactive=True,
@@ -727,7 +727,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     recording_channels = gr.Slider(
         minimum=0,
         maximum=30,
-        value=3,
+        value=8,
         step=1,
         label="Number of Recording only channels",
         interactive=True,
@@ -806,8 +806,8 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     # Object detection inference interval
     object_detection_inference_interval = gr.Slider(
         minimum=1,
-        maximum=5,
-        value=1,
+        maximum=6,
+        value=3,
         step=1,
         label="Object Detection Inference Interval",
         interactive=True,
@@ -830,6 +830,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     object_classification_model = gr.Dropdown(
         label="Object Classification Model",
         choices=[
+            "Disabled",
             "EfficientNet B0 (INT8)" ,
             "MobileNet V2 PyTorch (FP16)",
             "ResNet-50 TF (INT8)",
@@ -841,7 +842,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     # Object classification device
     object_classification_device = gr.Dropdown(
         label="Object Classification Device",
-        choices=device_choices,
+        choices=device_choices + ["Disabled"],
         value=preferred_device,
         elem_id="object_classification_device",
     )
@@ -860,8 +861,8 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     # Object classification inference interval
     object_classification_inference_interval = gr.Slider(
         minimum=1,
-        maximum=5,
-        value=1,
+        maximum=6,
+        value=3,
         step=1,
         label="Object Classification Inference Interval",
         interactive=True,
@@ -890,6 +891,13 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
         elem_id="object_classification_reclassify_interval",
     )
 
+    pipeline_watermark_enabled = gr.Checkbox(
+        label="Overlay inference results on inference channels",
+        value=True,
+        elem_id="pipeline_watermark_enabled",
+    )
+
+
     # Run button
     run_button = gr.Button("Run")
 
@@ -934,6 +942,7 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
     components.add(object_classification_inference_interval)
     components.add(object_classification_nireq)
     components.add(object_classification_reclassify_interval)
+    components.add(pipeline_watermark_enabled)
 
     # Interface layout
     with gr.Blocks(theme=theme, css=css_code, title=title) as demo:
@@ -1293,6 +1302,9 @@ def create_interface(title: str = "Visual Pipeline and Platform Evaluation Tool"
                             # Recording Channels
                             recording_channels.render()
 
+                            # Whether to overlay result with watermarks
+                            pipeline_watermark_enabled.render()
+
                         # Benchmark Parameters Accordion
                         with gr.Accordion("Platform Ceiling Analysis Parameters", open=False):
 

tools/visual-pipeline-and-platform-evaluation-tool/pipelines/smartnvr/diagram.drawio.svg

@@ -28,43 +28,40 @@ def __init__(self):
             "h264parse ! "
             "mp4mux ! "
             "filesink "
-            "  location={VIDEO_OUTPUT_PATH} "
+            "  location={VIDEO_OUTPUT_PATH} async=false "
         )
 
+
         self._recording_stream = (
             "filesrc "
             "  location={VIDEO_PATH} ! "
             "qtdemux ! "
             "h264parse ! "
             "tee name=t{id} ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
+            "queue2 ! "
             "mp4mux ! "
             "filesink "
             "  location=/tmp/stream{id}.mp4 "
             "t{id}. ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
+            "queue2 ! "
             "{decoder} ! "
-            "gvafpscounter starting-frame=1000 ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
-            "{postprocessing} ! "
-            "video/x-raw,width=640,height=360 ! "
-            "comp.sink_{id} "
+            "gvafpscounter starting-frame=500 ! "
         )
 
-        self._inference_stream = (
+        self._inference_stream_decode_detect_track = (
             "filesrc "
             "  location={VIDEO_PATH} ! "
             "qtdemux ! "
             "h264parse ! "
             "tee name=t{id} ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
+            "queue2 ! "
             "mp4mux ! "
             "filesink "
             "  location=/tmp/stream{id}.mp4 "
             "t{id}. ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
+            "queue2 ! "
             "{decoder} ! "
-            "gvafpscounter starting-frame=1000 ! "
+            "gvafpscounter starting-frame=500 ! "
             "gvadetect "
             "  {detection_model_config} "
             "  model-instance-id=detect0 "
@@ -73,16 +70,13 @@ def __init__(self):
             "  batch-size={object_detection_batch_size} "
             "  inference-interval={object_detection_inference_interval} "
             "  nireq={object_detection_nireq} ! "
-            "queue2 "
-            "  max-size-buffers=0 "
-            "  max-size-bytes=0 "
-            "  max-size-time=0 ! "
+            "queue2 ! "
             "gvatrack "
             "  tracking-type=short-term-imageless ! "
-            "queue2 "
-            "  max-size-buffers=0 "
-            "  max-size-bytes=0 "
-            "  max-size-time=0 ! "
+            "queue2 ! "
+        )
+
+        self._inference_stream_classify = (
             "gvaclassify "
             "  {classification_model_config} "
             "  model-instance-id=classify0 "
@@ -92,19 +86,24 @@ def __init__(self):
             "  inference-interval={object_classification_inference_interval} "
             "  nireq={object_classification_nireq} "
             "  reclassify-interval={object_classification_reclassify_interval} ! "
-            "queue2 "
-            "  max-size-buffers=0 "
-            "  max-size-bytes=0 "
-            "  max-size-time=0 ! "
-            "gvawatermark ! "
+            "queue2 ! "
+        )
+
+        self._inference_stream_metadata_processing = (
             "gvametaconvert "
             "  format=json "
             "  json-indent=4 "
             "  source={VIDEO_PATH} ! "
             "gvametapublish "
             "  method=file "
             "  file-path=/dev/null ! "
-            "queue2 max-size-bytes=0 max-size-time=0 ! "
+        )
+
+        self._sink_to_compositor = (
+            "queue2 "
+            "  max-size-buffers={max_size_buffers} "
+            "  max-size-bytes=0 "
+            "  max-size-time=0 ! "
             "{postprocessing} ! "
             "video/x-raw,width=640,height=360 ! "
             "comp.sink_{id} "
@@ -240,18 +239,15 @@ def evaluate(
                 ),
             )
 
-        # Create the compositor
-        compositor = self._compositor.format(
-            **constants,
-            sinks=sinks,
-            encoder=_encoder_element,
-            compositor=_compositor_element,
-        )
+
 
         # Create the streams
         streams = ""
 
+        # Handle inference channels
         for i in range(inference_channels):
+
+            # Handle object detection parameters and constants
             detection_model_config = (
                 f"model={constants["OBJECT_DETECTION_MODEL_PATH"]} "
                 f"model-proc={constants["OBJECT_DETECTION_MODEL_PROC"]} "
@@ -262,26 +258,54 @@ def evaluate(
                     f"model={constants["OBJECT_DETECTION_MODEL_PATH"]} "
                 )
 
-            classification_model_config = (
-                f"model={constants["OBJECT_CLASSIFICATION_MODEL_PATH"]} "
-                f"model-proc={constants["OBJECT_CLASSIFICATION_MODEL_PROC"]} "
+            streams += self._inference_stream_decode_detect_track.format(
+                **parameters,
+                **constants,
+                id=i,
+                decoder=_decoder_element,
+                detection_model_config=detection_model_config,
             )
 
-            if not constants["OBJECT_CLASSIFICATION_MODEL_PROC"]:
+            # Handle object classification parameters and constants
+            # Do this only if the object classification model is not disabled or the device is not disabled
+            if not (constants["OBJECT_CLASSIFICATION_MODEL_PATH"] == "Disabled" 
+                    or parameters["object_classification_device"] == "Disabled") :
                 classification_model_config = (
                     f"model={constants["OBJECT_CLASSIFICATION_MODEL_PATH"]} "
+                    f"model-proc={constants["OBJECT_CLASSIFICATION_MODEL_PROC"]} "
                 )
 
-            streams += self._inference_stream.format(
+                if not constants["OBJECT_CLASSIFICATION_MODEL_PROC"]:
+                    classification_model_config = (
+                        f"model={constants["OBJECT_CLASSIFICATION_MODEL_PATH"]} "
+                    )
+
+                streams += self._inference_stream_classify.format(
+                    **parameters,
+                    **constants,
+                    id=i,
+                    classification_model_config=classification_model_config,
+                )
+
+            # Overlay inference results on the inferenced video if enabled
+            if parameters["pipeline_watermark_enabled"]:
+                streams += "gvawatermark ! "
+            
+            streams += self._inference_stream_metadata_processing.format(
                 **parameters,
                 **constants,
                 id=i,
-                decoder=_decoder_element,
-                postprocessing=_postprocessing_element,
-                detection_model_config=detection_model_config,
-                classification_model_config=classification_model_config,
             )
 
+            # sink to compositor or fake sink depending on the compose flag
+            streams += self._sink_to_compositor.format(
+                **parameters,
+                **constants,
+                id=i,
+                postprocessing=_postprocessing_element,
+                max_size_buffers=0,
+            )
+        # Handle regular channels
         for i in range(inference_channels, channels):
             streams += self._recording_stream.format(
                 **parameters,
@@ -290,6 +314,21 @@ def evaluate(
                 decoder=_decoder_element,
                 postprocessing=_postprocessing_element,
             )
+            # sink to compositor or fake sink depending on the compose flag
+            streams += self._sink_to_compositor.format(
+                **parameters,
+                **constants,
+                id=i,
+                postprocessing=_postprocessing_element,
+                max_size_buffers=1,
+            )
+        # Prepend the compositor 
+        streams = self._compositor.format(
+            **constants,
+            sinks=sinks,
+            encoder=_encoder_element,
+            compositor=_compositor_element,
+        ) + streams
 
         # Evaluate the pipeline
-        return "gst-launch-1.0 -q " + compositor + " " + streams
+        return "gst-launch-1.0 -q " + streams