# 自定义可视化 本示例创建一个 Python 的 DepthAI 管道，用于执行 YOLOv6-Nano 目标检测，生成带有边界框和文本的自定义图像标注，并将结果连同 640x480 NV12 摄像头帧一起流式传输到远程连接进行可视化。 这个示例需要DepthAI v3 API，参见[安装说明](https://docs.luxonis.com/software-v3/depthai.md)。 ## 管道 ### examples/custom_visualizations.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 5, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 3, "node2Input": "imageIn", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "1", "node1OutputGroup": "dynamicOutputs", "node2Id": 7, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[7, {"alias": "", "id": 7, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 12, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_0_1"}}], [5, {"alias": "", "id": 5, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 11, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_3__out"}}], [3, {"alias": "detectionParser", "id": 3, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "imageIn"], {"blocking": false, "group": "", "id": 9, "name": "imageIn", "queueSize": 1, "type": 3, "waitForMessage": true}], [["", "in"], {"blocking": true, "group": "", "id": 8, "name": "in", "queueSize": 1, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "DetectionParser", "parentId": 1, "properties": {"networkInputs": {"images": {"dataType": 1, "dims": [416, 416, 3, 1], "name": "images", "numDimensions": 4, "offset": 0, "order": 17185, "qpScale": 1.0, "qpZp": 0.0, "quantization": false, "strides": []}}, "numFramesPool": 8, "parser": {"anchorMasks": {}, "anchors": [], "anchorsV2": [], "classNames": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"], "classes": 80, "confidenceThreshold": 0.5, "coordinates": 4, "iouThreshold": 0.5, "nnFamily": 0, "subtype": "yolov6"}}}], [2, {"alias": "neuralNetwork", "id": 2, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 7, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 6, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 5, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": 1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 8689834, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 4, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["dynamicOutputs", "1"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "1", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 100663297, "aeRegion": {"height": 0, "priority": 4155921504, "width": 0, "x": 4163, "y": 1}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 25042, "y": 0}, "antiBandingMode": 182, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 168, "brightness": -105, "captureIntent": 247, "chromaDenoise": 0, "cmdMask": 0, "contrast": -33, "controlMode": 210, "effectMode": 97, "enableHdr": false, "expCompensation": 39, "expManual": {"exposureTimeUs": 25042, "frameDurationUs": 25042, "sensitivityIso": 4155921612}, "frameSyncMode": 0, "lensPosAutoInfinity": 224, "lensPosAutoMacro": 84, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": -34, "sceneMode": 84, "sharpness": 0, "strobeConfig": {"activeLevel": 8, "enable": 0, "gpioNumber": 91}, "strobeTimings": {"durationUs": 16, "exposureBeginOffsetUs": 25042, "exposureEndOffsetUs": 17}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [416, 416]}}, "type": 8}, {"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [640, 480]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## 源代码 #### Python ```python #!/usr/bin/env python3 import depthai as dai class ImgDetectionsExtended(dai.ImgDetections): def __init__(self, detections: dai.ImgDetections): dai.ImgDetections.__init__(self) self.detections = detections.detections # The function can return dai.ImgAnnotations or dai.ImgFrame def getVisualizationMessage(self): detections = self.detections imgAnnt = dai.ImgAnnotations() # Setting the timestamp is important, as the visualizer uses it to synchronize the data imgAnnt.setTimestamp(self.getTimestamp()) annotation = dai.ImgAnnotation() for detection in detections: pointsAnnotation = dai.PointsAnnotation() pointsAnnotation.type = dai.PointsAnnotationType.LINE_STRIP pointsAnnotation.points = dai.VectorPoint2f([ dai.Point2f(detection.xmin, detection.ymin), dai.Point2f(detection.xmax, detection.ymin), dai.Point2f(detection.xmax, detection.ymax), dai.Point2f(detection.xmin, detection.ymax), ]) outlineColor = dai.Color(1.0, 0.5, 0.5, 1.0) pointsAnnotation.outlineColor = outlineColor fillColor = dai.Color(0.5, 1.0, 0.5, 0.5) pointsAnnotation.fillColor = fillColor pointsAnnotation.thickness = 2.0 text = dai.TextAnnotation() text.position = dai.Point2f(detection.xmin, detection.ymin) text.text = f"Test annotation" text.fontSize = 50.5 textColor = dai.Color(0.5, 0.5, 1.0, 1.0) text.textColor = textColor backgroundColor = dai.Color(1.0, 1.0, 0.5, 1.0) text.backgroundColor = backgroundColor annotation.points.append(pointsAnnotation) annotation.texts.append(text) imgAnnt.annotations.append(annotation) return imgAnnt class ImgAnnotationsGenerator(dai.node.ThreadedHostNode): def __init__(self): super().__init__() self.inputDet = self.createInput() self.output = self.createOutput() def run(self): while self.isRunning(): nnData = self.inputDet.get() extended = ImgDetectionsExtended(nnData) # Setting the timestamp is important, as the visualizer uses it to synchronize the data extended.setTimestamp(nnData.getTimestamp()) self.output.send(extended) remoteConnector = dai.RemoteConnection() # Create pipeline with dai.Pipeline() as pipeline: cameraNode = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A) detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build( cameraNode, dai.NNModelDescription("yolov6-nano") ) imageAnnotationsGenerator = pipeline.create(ImgAnnotationsGenerator) outputToVisualize = cameraNode.requestOutput((640,480), type=dai.ImgFrame.Type.NV12) detectionNetwork.out.link(imageAnnotationsGenerator.inputDet) # Add the remote connector topics remoteConnector.addTopic("encoded", outputToVisualize, "images") remoteConnector.addTopic("detections", detectionNetwork.out, "images") remoteConnector.addTopic("annotations", imageAnnotationsGenerator.output, "images") pipeline.start() # Register the pipeline with the remote connector remoteConnector.registerPipeline(pipeline) while pipeline.isRunning(): if remoteConnector.waitKey(1) == ord("q"): pipeline.stop() break ``` #### C++ ```cpp #include #include #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImgAnnotations.hpp" #include "depthai/remote_connection/RemoteConnection.hpp" // Global flag for graceful shutdown std::atomic quitEvent(false); // Signal handler void signalHandler(int signum) { quitEvent = true; } // Extended detections class class ImgDetectionsExtended : public dai::ImgDetections { public: ImgDetectionsExtended(const std::shared_ptr& detections) { this->detections = detections->detections; this->setTimestamp(detections->getTimestamp()); } std::shared_ptr getVisualizationMessage() { auto imgAnnt = std::make_shared(); imgAnnt->setTimestamp(this->getTimestamp()); auto annotation = std::make_shared(); for(const auto& detection : this->detections) { // Create points annotation for bounding box auto pointsAnnotation = std::make_shared(); pointsAnnotation->type = dai::PointsAnnotationType::LINE_STRIP; pointsAnnotation->points = {dai::Point2f(detection.xmin, detection.ymin), dai::Point2f(detection.xmax, detection.ymin), dai::Point2f(detection.xmax, detection.ymax), dai::Point2f(detection.xmin, detection.ymax)}; // Set colors and thickness pointsAnnotation->outlineColor = dai::Color(1.0f, 0.5f, 0.5f, 1.0f); pointsAnnotation->fillColor = dai::Color(0.5f, 1.0f, 0.5f, 0.5f); pointsAnnotation->thickness = 2.0f; // Create text annotation auto text = std::make_shared(); text->position = dai::Point2f(detection.xmin, detection.ymin); text->text = "Test annotation"; text->fontSize = 50.5f; text->textColor = dai::Color(0.5f, 0.5f, 1.0f, 1.0f); text->backgroundColor = dai::Color(1.0f, 1.0f, 0.5f, 1.0f); annotation->points.push_back(*pointsAnnotation); annotation->texts.push_back(*text); } imgAnnt->annotations.push_back(*annotation); return imgAnnt; } }; // Custom host node for image annotations class ImgAnnotationsGenerator : public dai::NodeCRTP { public: Input& inputDet = inputs["detections"]; Output& output = out; std::shared_ptr build(Output& detections) { detections.link(inputDet); return std::static_pointer_cast(this->shared_from_this()); } std::shared_ptr processGroup(std::shared_ptr in) override { auto nnData = in->get("detections"); auto extended = std::make_shared(nnData); return extended->getVisualizationMessage(); } }; int main() { // Set up signal handlers signal(SIGTERM, signalHandler); signal(SIGINT, signalHandler); try { // Create remote connection dai::RemoteConnection remoteConnector; // Create pipeline dai::Pipeline pipeline; // Create nodes auto cameraNode = pipeline.create(); cameraNode->build(dai::CameraBoardSocket::CAM_A); auto detectionNetwork = pipeline.create(); dai::NNModelDescription modelDesc; modelDesc.model = "yolov6-nano"; detectionNetwork->build(cameraNode, modelDesc); auto imageAnnotationsGenerator = pipeline.create(); auto outputToVisualize = cameraNode->requestOutput(std::make_pair(640, 480), dai::ImgFrame::Type::NV12); // Linking detectionNetwork->out.link(imageAnnotationsGenerator->inputDet); // Add remote connector topics remoteConnector.addTopic("encoded", *outputToVisualize, "images"); remoteConnector.addTopic("detections", detectionNetwork->out, "images"); remoteConnector.addTopic("annotations", imageAnnotationsGenerator->output, "images"); // Start pipeline pipeline.start(); // Register pipeline with remote connector remoteConnector.registerPipeline(pipeline); // Main loop while(pipeline.isRunning() && !quitEvent) { if(remoteConnector.waitKey(1) == 'q') { pipeline.stop(); break; } } // Cleanup pipeline.stop(); pipeline.wait(); } catch(const std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; } return 0; } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。