# RGB 与 Tiny YOLO 本示例演示如何在 RGB 输入帧上运行 YOLO，以及如何在预览上同时显示 RGB 预览和 YOLO 模型的元数据结果。解码在 [RVC](https://docs.luxonis.com/hardware/platform/rvc/rvc2.md#rvc2) 上完成，而非主机电脑。 正确解码需要可配置的、网络相关的参数： * setNumClasses() - YOLO 类别数 * setCoordinateSize() - 坐标大小 * setAnchors() - YOLO 锚点 * setAnchorMasks() - anchorMasks26, anchorMasks13（full YOLOv4 还需 anchorMasks52） * setIouThreshold() - 交并比阈值 * setConfidenceThreshold() - 置信度阈值，高于该值的目标被检测 默认使用 Tiny YOLOv4。你可以添加 yolo3 作为 CMD 参数来使用 Tiny YOLOv3。 ## 演示 ## 设置 请运行[安装脚本](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py)以下载所有必需的依赖项。请注意，此脚本必须在 git 上下文中运行，因此您需要先下载 [depthai-python](https://github.com/luxonis/depthai-python) 仓库，然后运行该脚本。 ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` 更多信息，请参考[安装指南](https://docs.luxonis.com/software/depthai/manual-install.md)。 ## 源代码 #### Python ```python #!/usr/bin/env python3 """ The code is the same as for Tiny Yolo V3 and V4, the only difference is the blob file - Tiny YOLOv3: https://github.com/david8862/keras-YOLOv3-model-set - Tiny YOLOv4: https://github.com/TNTWEN/OpenVINO-YOLOV4 """ from pathlib import Path import sys import cv2 import depthai as dai import numpy as np import time # Get argument first nnPath = str((Path(__file__).parent / Path('../models/yolo-v4-tiny-tf_openvino_2021.4_6shave.blob')).resolve().absolute()) if 1 < len(sys.argv): arg = sys.argv[1] if arg == "yolo3": nnPath = str((Path(__file__).parent / Path('../models/yolo-v3-tiny-tf_openvino_2021.4_6shave.blob')).resolve().absolute()) elif arg == "yolo4": nnPath = str((Path(__file__).parent / Path('../models/yolo-v4-tiny-tf_openvino_2021.4_6shave.blob')).resolve().absolute()) else: nnPath = arg else: print("Using Tiny YoloV4 model. If you wish to use Tiny YOLOv3, call 'tiny_yolo.py yolo3'") if not Path(nnPath).exists(): import sys raise FileNotFoundError(f'Required file/s not found, please run "{sys.executable} install_requirements.py"') # tiny yolo v4 label texts labelMap = [ "person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" ] syncNN = True # Create pipeline pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork) xoutRgb = pipeline.create(dai.node.XLinkOut) nnOut = pipeline.create(dai.node.XLinkOut) xoutRgb.setStreamName("rgb") nnOut.setStreamName("nn") # Properties camRgb.setPreviewSize(416, 416) camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P) camRgb.setInterleaved(False) camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR) camRgb.setFps(40) # Network specific settings detectionNetwork.setConfidenceThreshold(0.5) detectionNetwork.setNumClasses(80) detectionNetwork.setCoordinateSize(4) detectionNetwork.setAnchors([10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319]) detectionNetwork.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]}) detectionNetwork.setIouThreshold(0.5) detectionNetwork.setBlobPath(nnPath) detectionNetwork.setNumInferenceThreads(2) detectionNetwork.input.setBlocking(False) # Linking camRgb.preview.link(detectionNetwork.input) if syncNN: detectionNetwork.passthrough.link(xoutRgb.input) else: camRgb.preview.link(xoutRgb.input) detectionNetwork.out.link(nnOut.input) # Connect to device and start pipeline with dai.Device(pipeline) as device: # Output queues will be used to get the rgb frames and nn data from the outputs defined above qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False) qDet = device.getOutputQueue(name="nn", maxSize=4, blocking=False) frame = None detections = [] startTime = time.monotonic() counter = 0 color2 = (255, 255, 255) # nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height def frameNorm(frame, bbox): normVals = np.full(len(bbox), frame.shape[0]) normVals[::2] = frame.shape[1] return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int) def displayFrame(name, frame): color = (255, 0, 0) for detection in detections: bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax)) cv2.putText(frame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.putText(frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255) cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2) # Show the frame cv2.imshow(name, frame) while True: if syncNN: inRgb = qRgb.get() inDet = qDet.get() else: inRgb = qRgb.tryGet() inDet = qDet.tryGet() if inRgb is not None: frame = inRgb.getCvFrame() cv2.putText(frame, "NN fps: {:.2f}".format(counter / (time.monotonic() - startTime)), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.4, color2) if inDet is not None: detections = inDet.detections counter += 1 if frame is not None: displayFrame("rgb", frame) if cv2.waitKey(1) == ord('q'): break ``` #### C++ ```cpp #include #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" /* The code is the same as for Tiny-yolo-V3, the only difference is the blob file. The blob was compiled following this tutorial: https://github.com/TNTWEN/OpenVINO-YOLOV4 */ static const std::vector labelMap = { "person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"}; static std::atomic syncNN{true}; int main(int argc, char** argv) { using namespace std; using namespace std::chrono; std::string nnPath(BLOB_PATH); // If path to blob specified, use that if(argc > 1) { nnPath = std::string(argv[1]); } // Print which blob we are using printf("Using blob at path: %s\n", nnPath.c_str()); // Create pipeline dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto detectionNetwork = pipeline.create(); auto xoutRgb = pipeline.create(); auto nnOut = pipeline.create(); xoutRgb->setStreamName("rgb"); nnOut->setStreamName("detections"); // Properties camRgb->setPreviewSize(416, 416); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P); camRgb->setInterleaved(false); camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR); camRgb->setFps(40); // Network specific settings detectionNetwork->setConfidenceThreshold(0.5f); detectionNetwork->setNumClasses(80); detectionNetwork->setCoordinateSize(4); detectionNetwork->setAnchors({10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319}); detectionNetwork->setAnchorMasks({{"side26", {1, 2, 3}}, {"side13", {3, 4, 5}}}); detectionNetwork->setIouThreshold(0.5f); detectionNetwork->setBlobPath(nnPath); detectionNetwork->setNumInferenceThreads(2); detectionNetwork->input.setBlocking(false); // Linking camRgb->preview.link(detectionNetwork->input); if(syncNN) { detectionNetwork->passthrough.link(xoutRgb->input); } else { camRgb->preview.link(xoutRgb->input); } detectionNetwork->out.link(nnOut->input); // Connect to device and start pipeline dai::Device device(pipeline); // Output queues will be used to get the rgb frames and nn data from the outputs defined above auto qRgb = device.getOutputQueue("rgb", 4, false); auto qDet = device.getOutputQueue("detections", 4, false); cv::Mat frame; std::vector detections; auto startTime = steady_clock::now(); int counter = 0; float fps = 0; auto color2 = cv::Scalar(255, 255, 255); // Add bounding boxes and text to the frame and show it to the user auto displayFrame = [](std::string name, cv::Mat frame, std::vector& detections) { auto color = cv::Scalar(255, 0, 0); // nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height for(auto& detection : detections) { int x1 = detection.xmin * frame.cols; int y1 = detection.ymin * frame.rows; int x2 = detection.xmax * frame.cols; int y2 = detection.ymax * frame.rows; uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if(labelIndex < labelMap.size()) { labelStr = labelMap[labelIndex]; } cv::putText(frame, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); std::stringstream confStr; confStr << std::fixed << std::setprecision(2) << detection.confidence * 100; cv::putText(frame, confStr.str(), cv::Point(x1 + 10, y1 + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); cv::rectangle(frame, cv::Rect(cv::Point(x1, y1), cv::Point(x2, y2)), color, cv::FONT_HERSHEY_SIMPLEX); } // Show the frame cv::imshow(name, frame); }; while(true) { std::shared_ptr inRgb; std::shared_ptr inDet; if(syncNN) { inRgb = qRgb->get(); inDet = qDet->get(); } else { inRgb = qRgb->tryGet(); inDet = qDet->tryGet(); } counter++; auto currentTime = steady_clock::now(); auto elapsed = duration_cast>(currentTime - startTime); if(elapsed > seconds(1)) { fps = counter / elapsed.count(); counter = 0; startTime = currentTime; } if(inRgb) { frame = inRgb->getCvFrame(); std::stringstream fpsStr; fpsStr << "NN fps: " << std::fixed << std::setprecision(2) << fps; cv::putText(frame, fpsStr.str(), cv::Point(2, inRgb->getHeight() - 4), cv::FONT_HERSHEY_TRIPLEX, 0.4, color2); } if(inDet) { detections = inDet->detections; } if(!frame.empty()) { displayFrame("rgb", frame, detections); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { return 0; } } return 0; } ``` ## 管道 ### examples/tiny_yolo.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 0, "node1Output": "preview", "node1OutputGroup": "", "node2Id": 1, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 2, "node2Input": "in", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": -1, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 40.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": false, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 416, "previewKeepAspectRatio": true, "previewWidth": 416, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "in" ], { "blocking": false, "group": "", "id": 9, "name": "in", "queueSize": 5, "type": 3, "waitForMessage": true } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthrough" ], { "blocking": false, "group": "", "id": 11, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "DetectionNetwork", "properties": { "blobSize": 12168064, "blobUri": "asset:__blob", "numFrames": 8, "numNCEPerThread": 0, "numThreads": 2, "parser": { "anchorMasks": { "side13": [ 3, 4, 5 ], "side26": [ 1, 2, 3 ] }, "anchors": [ 10.0, 14.0, 23.0, 27.0, 37.0, 58.0, 81.0, 82.0, 135.0, 169.0, 344.0, 319.0 ], "classes": 80, "confidenceThreshold": 0.5, "coordinates": 4, "iouThreshold": 0.5, "nnFamily": 0 } } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 12, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "rgb" } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 13, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "nn" } } ] ] } } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。