# RGB 编码和 MobileNetSSD 本示例展示了如何配置 depthai 视频编码器，使用 h.265 格式编码 RGB 摄像头输入，分辨率为全高清（Full-HD），帧率为 30FPS，并将编码后的视频通过 XLINK 传输到主机，保存为视频文件到磁盘。同时，在用于编码的同一 RGB 摄像头的帧上运行 MobileNetv2SSD 网络。 按 Ctrl+C 将停止录制，然后使用 ffmpeg 将其转换为 mp4 格式以便播放。请注意，要成功转换为 mp4，需要安装并运行 ffmpeg。 请注意，本示例将编码后的视频保存到主机存储中。如果长时间运行，可能会填满主机存储空间。 它是 [RGB 编码](https://docs.luxonis.com/software/depthai/examples/rgb_encoding.md) 和 [RGB 与 MobileNetSSD](https://docs.luxonis.com/software/depthai/examples/rgb_mobilenet.md) 的组合。 ### 类似示例： * [RGB 编码](https://docs.luxonis.com/software/depthai/examples/rgb_encoding.md) * [RGB 与 Mono 编码](https://docs.luxonis.com/software/depthai/examples/rgb_mono_encoding.md) * [编码最大限制](https://docs.luxonis.com/software/depthai/examples/encoding_max_limit.md) * [RGB 编码与 Mono 及 MobileNetSSD](https://docs.luxonis.com/software/depthai/examples/rgb_encoding_mono_mobilenet.md) * [RGB 编码与 Mono 结合 MobileNetSSD 及深度](https://docs.luxonis.com/software/depthai/examples/rgb_encoding_mono_mobilenet_depth.md) ## 演示 ## 设置 请运行[安装脚本](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py)以下载所有必需的依赖项。请注意，此脚本必须在 git 上下文中运行，因此您需要先下载 [depthai-python](https://github.com/luxonis/depthai-python) 仓库，然后运行该脚本。 ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` 更多信息，请参考[安装指南](https://docs.luxonis.com/software/depthai/manual-install.md)。 ## 源代码 #### Python ```python #!/usr/bin/env python3 from pathlib import Path import sys import cv2 import depthai as dai import numpy as np # Get argument first nnPath = str((Path(__file__).parent / Path('../models/mobilenet-ssd_openvino_2021.4_6shave.blob')).resolve().absolute()) if len(sys.argv) > 1: nnPath = sys.argv[1] if not Path(nnPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') # MobilenetSSD label texts labelMap = ["background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"] # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) videoEncoder = pipeline.create(dai.node.VideoEncoder) nn = pipeline.create(dai.node.MobileNetDetectionNetwork) xoutRgb = pipeline.create(dai.node.XLinkOut) videoOut = pipeline.create(dai.node.XLinkOut) nnOut = pipeline.create(dai.node.XLinkOut) xoutRgb.setStreamName("rgb") videoOut.setStreamName("h265") nnOut.setStreamName("nn") # Properties camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A) camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) camRgb.setPreviewSize(300, 300) camRgb.setInterleaved(False) videoEncoder.setDefaultProfilePreset(30, dai.VideoEncoderProperties.Profile.H265_MAIN) nn.setConfidenceThreshold(0.5) nn.setBlobPath(nnPath) nn.setNumInferenceThreads(2) nn.input.setBlocking(False) # Linking camRgb.video.link(videoEncoder.input) camRgb.preview.link(xoutRgb.input) camRgb.preview.link(nn.input) videoEncoder.bitstream.link(videoOut.input) nn.out.link(nnOut.input) # Connect to device and start pipeline with dai.Device(pipeline) as device, open('video.h265', 'wb') as videoFile: # Queues queue_size = 8 qRgb = device.getOutputQueue("rgb", queue_size) qDet = device.getOutputQueue("nn", queue_size) qRgbEnc = device.getOutputQueue('h265', maxSize=30, blocking=True) frame = None detections = [] def frameNorm(frame, bbox): normVals = np.full(len(bbox), frame.shape[0]) normVals[::2] = frame.shape[1] return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int) def displayFrame(name, frame): color = (255, 0, 0) for detection in detections: bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax)) cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color) cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color) cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2) # Show the frame cv2.imshow(name, frame) while True: inRgb = qRgb.tryGet() inDet = qDet.tryGet() while qRgbEnc.has(): qRgbEnc.get().getData().tofile(videoFile) if inRgb is not None: frame = inRgb.getCvFrame() if inDet is not None: detections = inDet.detections if frame is not None: displayFrame("rgb", frame) if cv2.waitKey(1) == ord('q'): break print("To view the encoded data, convert the stream file (.h265) into a video file (.mp4), using a command below:") print("ffmpeg -framerate 30 -i video.h265 -c copy video.mp4") ``` #### C++ ```cpp #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" // MobilenetSSD label texts static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; int main(int argc, char** argv) { using namespace std; // Default blob path provided by Hunter private data download // Applicable for easier example usage only std::string nnPath(BLOB_PATH); // If path to blob specified, use that if(argc > 1) { nnPath = std::string(argv[1]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); // Create pipeline dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto videoEncoder = pipeline.create(); auto nn = pipeline.create(); auto xoutRgb = pipeline.create(); auto videoOut = pipeline.create(); auto nnOut = pipeline.create(); xoutRgb->setStreamName("rgb"); videoOut->setStreamName("h265"); nnOut->setStreamName("nn"); // Properties camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P); camRgb->setPreviewSize(300, 300); camRgb->setInterleaved(false); videoEncoder->setDefaultProfilePreset(30, dai::VideoEncoderProperties::Profile::H265_MAIN); nn->setConfidenceThreshold(0.5); nn->setBlobPath(nnPath); nn->setNumInferenceThreads(2); nn->input.setBlocking(false); // Linking camRgb->video.link(videoEncoder->input); camRgb->preview.link(xoutRgb->input); camRgb->preview.link(nn->input); videoEncoder->bitstream.link(videoOut->input); nn->out.link(nnOut->input); // Connect to device and start pipeline dai::Device device(pipeline); // Queues int queueSize = 8; auto qRgb = device.getOutputQueue("rgb", queueSize); auto qDet = device.getOutputQueue("nn", queueSize); auto qRgbEnc = device.getOutputQueue("h265", 30, true); cv::Mat frame; std::vector detections; // Add bounding boxes and text to the frame and show it to the user auto displayFrame = [](std::string name, cv::Mat frame, std::vector& detections) { auto color = cv::Scalar(255, 0, 0); // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height for(auto& detection : detections) { int x1 = detection.xmin * frame.cols; int y1 = detection.ymin * frame.rows; int x2 = detection.xmax * frame.cols; int y2 = detection.ymax * frame.rows; uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if(labelIndex < labelMap.size()) { labelStr = labelMap[labelIndex]; } cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); std::stringstream confStr; confStr << std::fixed << std::setprecision(2) << detection.confidence * 100; cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, color); cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX); } // Show the frame cv::imshow(name, frame); }; auto videoFile = std::ofstream("video.h264", std::ios::binary); while(true) { auto inRgb = qRgb->tryGet(); auto inDet = qDet->tryGet(); auto out = qRgbEnc->get(); videoFile.write((char*)out->getData().data(), out->getData().size()); if(inRgb) { frame = inRgb->getCvFrame(); } if(inDet) { detections = inDet->detections; } if(!frame.empty()) { displayFrame("rgb", frame, detections); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { break; } } cout << "To view the encoded data, convert the stream file (.h265) into a video file (.mp4), using a command below:" << endl; cout << "ffmpeg -framerate 30 -i video.h264 -c copy video.mp4" << endl; return 0; } ``` ## 流水线 ### examples/rgb_encoding_mobilenet.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 0, "node1Output": "video", "node1OutputGroup": "", "node2Id": 1, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "bitstream", "node1OutputGroup": "", "node2Id": 4, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 5, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 30.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": false, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 9, "name": "in", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "bitstream" ], { "blocking": false, "group": "", "id": 10, "name": "bitstream", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 11, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "VideoEncoder", "properties": { "bitrate": 0, "frameRate": 30.0, "keyframeFrequency": 30, "lossless": false, "maxBitrate": 0, "numBFrames": 0, "numFramesPool": 0, "outputFrameSize": 0, "profile": 3, "quality": 80, "rateCtrlMode": 0 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "in" ], { "blocking": false, "group": "", "id": 12, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 13, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 14, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "DetectionNetwork", "properties": { "blobSize": 14499200, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2, "parser": { "anchorMasks": {}, "anchors": [], "classes": 0, "confidenceThreshold": 0.5, "coordinates": 0, "iouThreshold": 0.0, "nnFamily": 1 } } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 15, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "rgb" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 16, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "h265" } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 17, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nn" } } ] ] } } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。