# 视频与MobilenetSSD

本示例展示了如何在RGB输入帧上运行MobileNetv2SSD，该帧是从指定文件中读取的，而不是从RGB摄像头读取，并展示了如何显示RGB帧以及帧上的MobileNetv2SSD元数据结果。这里DepthAI仅用作处理单元。

### 类似示例：

 * [RGB与MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet.md)
 * [RGB与MobilenetSSD @ 4K](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet_4k.md)
 * [Mono与MobilenetSSD](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md)
 * [Mono与MobilenetSSD及深度](https://docs.luxonis.com/software/depthai/examples/mono_depth_mobilenetssd.md)

## 演示

## 设置

请运行[安装脚本](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py)以下载所有必需的依赖项。请注意，此脚本必须在 git
上下文中运行，因此您需要先下载 [depthai-python](https://github.com/luxonis/depthai-python) 仓库，然后运行该脚本。

```bash
git clone https://github.com/luxonis/depthai-python.git
cd depthai-python/examples
python3 install_requirements.py
```

更多信息，请参考[安装指南](https://docs.luxonis.com/software/depthai/manual-install.md)。

## 源代码

#### Python

```python
#!/usr/bin/env python3

from pathlib import Path
import sys
import cv2
import depthai as dai
import numpy as np
from time import monotonic

# Get argument first
parentDir = Path(__file__).parent
nnPath = str((parentDir / Path('../models/mobilenet-ssd_openvino_2021.4_8shave.blob')).resolve().absolute())
videoPath = str((parentDir / Path('../models/construction_vest.mp4')).resolve().absolute())
if len(sys.argv) > 2:
    nnPath = sys.argv[1]
    videoPath = sys.argv[2]

if not Path(nnPath).exists() or not Path(videoPath).exists():
    import sys
    raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"')

# MobilenetSSD label texts
labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow",
            "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]

# Create pipeline
pipeline = dai.Pipeline()

# Define sources and outputs
nn = pipeline.create(dai.node.MobileNetDetectionNetwork)

xinFrame = pipeline.create(dai.node.XLinkIn)
nnOut = pipeline.create(dai.node.XLinkOut)

xinFrame.setStreamName("inFrame")
nnOut.setStreamName("nn")

# Properties
nn.setConfidenceThreshold(0.5)
nn.setBlobPath(nnPath)
nn.setNumInferenceThreads(2)
nn.input.setBlocking(False)

# Linking
xinFrame.out.link(nn.input)
nn.out.link(nnOut.input)

# Connect to device and start pipeline
with dai.Device(pipeline) as device:

    # Input queue will be used to send video frames to the device.
    qIn = device.getInputQueue(name="inFrame")
    # Output queue will be used to get nn data from the video frames.
    qDet = device.getOutputQueue(name="nn", maxSize=4, blocking=False)

    frame = None
    detections = []

    # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
    def frameNorm(frame, bbox):
        normVals = np.full(len(bbox), frame.shape[0])
        normVals[::2] = frame.shape[1]
        return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

    def to_planar(arr: np.ndarray, shape: tuple) -> np.ndarray:
        return cv2.resize(arr, shape).transpose(2, 0, 1).flatten()

    def displayFrame(name, frame):
        for detection in detections:
            bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
            cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
            cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
            cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (255, 0, 0), 2)
        # Show the frame
        cv2.imshow(name, frame)

    cap = cv2.VideoCapture(videoPath)
    while cap.isOpened():
        read_correctly, frame = cap.read()
        if not read_correctly:
            break

        img = dai.ImgFrame()
        img.setData(to_planar(frame, (300, 300)))
        img.setTimestamp(monotonic())
        img.setWidth(300)
        img.setHeight(300)
        qIn.send(img)

        inDet = qDet.tryGet()

        if inDet is not None:
            detections = inDet.detections

        if frame is not None:
            displayFrame("rgb", frame)

        if cv2.waitKey(1) == ord('q'):
            break
```

#### C++

```cpp
#include <chrono>
#include <iostream>

#include "utility.hpp"

// Includes common necessary includes for development using depthai library
#include "depthai/depthai.hpp"

// MobilenetSSD label texts
static const std::vector<std::string> labelMap = {"background", "aeroplane", "bicycle",     "bird",  "boat",        "bottle", "bus",
                                                  "car",        "cat",       "chair",       "cow",   "diningtable", "dog",    "horse",
                                                  "motorbike",  "person",    "pottedplant", "sheep", "sofa",        "train",  "tvmonitor"};

int main(int argc, char** argv) {
    using namespace std;
    using namespace std::chrono;
    // Default blob path provided by Hunter private data download
    // Applicable for easier example usage only
    std::string nnPath(BLOB_PATH);
    std::string videoPath(VIDEO_PATH);

    // If path to blob specified, use that
    if(argc > 2) {
        nnPath = std::string(argv[1]);
        videoPath = std::string(argv[2]);
    }

    // Print which blob we are using
    printf("Using blob at path: %s\n", nnPath.c_str());
    printf("Using video at path: %s\n", videoPath.c_str());

    // Create pipeline
    dai::Pipeline pipeline;

    // Define source and outputs
    auto nn = pipeline.create<dai::node::MobileNetDetectionNetwork>();

    auto xinFrame = pipeline.create<dai::node::XLinkIn>();
    auto nnOut = pipeline.create<dai::node::XLinkOut>();

    xinFrame->setStreamName("inFrame");
    nnOut->setStreamName("nn");

    // Properties
    nn->setConfidenceThreshold(0.5);
    nn->setBlobPath(nnPath);
    nn->setNumInferenceThreads(2);
    nn->input.setBlocking(false);

    // Linking
    xinFrame->out.link(nn->input);
    nn->out.link(nnOut->input);

    // Connect to device and start pipeline
    dai::Device device(pipeline);

    // Input queue will be used to send video frames to the device.
    auto qIn = device.getInputQueue("inFrame");
    // Output queue will be used to get nn data from the video frames.
    auto qDet = device.getOutputQueue("nn", 4, false);

    // Add bounding boxes and text to the frame and show it to the user
    auto displayFrame = [](std::string name, cv::Mat frame, std::vector<dai::ImgDetection>& detections) {
        auto color = cv::Scalar(255, 0, 0);
        // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height
        for(auto& detection : detections) {
            int x1 = detection.xmin * frame.cols;
            int y1 = detection.ymin * frame.rows;
            int x2 = detection.xmax * frame.cols;
            int y2 = detection.ymax * frame.rows;

            uint32_t labelIndex = detection.label;
            std::string labelStr = to_string(labelIndex);
            if(labelIndex < labelMap.size()) {
                labelStr = labelMap[labelIndex];
            }
            cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
            std::stringstream confStr;
            confStr << std::fixed << std::setprecision(2) << detection.confidence * 100;
            cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color);
            cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX);
        }
        // Show the frame
        cv::imshow(name, frame);
    };

    cv::Mat frame;
    cv::VideoCapture cap(videoPath);

    cv::namedWindow("inFrame", cv::WINDOW_NORMAL);
    cv::resizeWindow("inFrame", 1280, 720);
    std::cout << "Resize video window with mouse drag!" << std::endl;

    while(cap.isOpened()) {
        // Read frame from video
        cap >> frame;
        if(frame.empty()) break;

        auto img = std::make_shared<dai::ImgFrame>();
        frame = resizeKeepAspectRatio(frame, cv::Size(300, 300), cv::Scalar(0));
        toPlanar(frame, img->getData());
        img->setTimestamp(steady_clock::now());
        img->setWidth(300);
        img->setHeight(300);
        qIn->send(img);

        auto inDet = qDet->get<dai::ImgDetections>();
        auto detections = inDet->detections;

        displayFrame("inFrame", frame, detections);

        int key = cv::waitKey(1);
        if(key == 'q' || key == 'Q') return 0;
    }
    return 0;
}
```

## 管道

### examples/video_mobilenet.pipeline.json

```json
{
  "pipeline": {
    "connections": [
      {
        "node1Id": 1,
        "node1Output": "out",
        "node1OutputGroup": "",
        "node2Id": 0,
        "node2Input": "in",
        "node2InputGroup": ""
      },
      {
        "node1Id": 0,
        "node1Output": "out",
        "node1OutputGroup": "",
        "node2Id": 2,
        "node2Input": "in",
        "node2InputGroup": ""
      }
    ],
    "globalProperties": {
      "calibData": null,
      "cameraTuningBlobSize": null,
      "cameraTuningBlobUri": "",
      "leonCssFrequencyHz": 700000000.0,
      "leonMssFrequencyHz": 700000000.0,
      "pipelineName": null,
      "pipelineVersion": null,
      "sippBufferSize": 18432,
      "sippDmaBufferSize": 16384,
      "xlinkChunkSize": -1
    },
    "nodes": [
      [
        0,
        {
          "id": 0,
          "ioInfo": [
            [
              [
                "",
                "in"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 1,
                "name": "in",
                "queueSize": 5,
                "type": 3,
                "waitForMessage": true
              }
            ],
            [
              [
                "",
                "out"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 2,
                "name": "out",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ],
            [
              [
                "",
                "passthrough"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 3,
                "name": "passthrough",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ]
          ],
          "name": "DetectionNetwork",
          "properties": {
            "blobSize": 14493888,
            "blobUri": "asset:__blob",
            "numFrames": 8,
            "numNCEPerThread": 0,
            "numThreads": 2,
            "parser": {
              "anchorMasks": {},
              "anchors": [],
              "classes": 0,
              "confidenceThreshold": 0.5,
              "coordinates": 0,
              "iouThreshold": 0.0,
              "nnFamily": 1
            }
          }
        }
      ],
      [
        1,
        {
          "id": 1,
          "ioInfo": [
            [
              [
                "",
                "out"
              ],
              {
                "blocking": false,
                "group": "",
                "id": 4,
                "name": "out",
                "queueSize": 8,
                "type": 0,
                "waitForMessage": false
              }
            ]
          ],
          "name": "XLinkIn",
          "properties": {
            "maxDataSize": 5242880,
            "numFrames": 8,
            "streamName": "inFrame"
          }
        }
      ],
      [
        2,
        {
          "id": 2,
          "ioInfo": [
            [
              [
                "",
                "in"
              ],
              {
                "blocking": true,
                "group": "",
                "id": 5,
                "name": "in",
                "queueSize": 8,
                "type": 3,
                "waitForMessage": true
              }
            ]
          ],
          "name": "XLinkOut",
          "properties": {
            "maxFpsLimit": -1.0,
            "metadataOnly": false,
            "streamName": "nn"
          }
        }
      ]
    ]
  }
}
```

### 需要帮助？

请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。