# Visualizer Encoded 该示例利用DepthAI创建了一个管道，执行YOLOv6-Nano物体检测，生成带有边界框和标签的自定义图像注释，将高分辨率（1920x1440）摄像头帧编码为MJPEG，并将结果流式传输到远程连接以供可视化。 这个示例需要DepthAI v3 API，参见[安装说明](https://docs.luxonis.com/software-v3/depthai.md)。 ## 管道 ### examples/visualizer_encoded.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 8, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 3, "node2Input": "imageIn", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 6, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "1", "node1OutputGroup": "dynamicOutputs", "node2Id": 5, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[8, {"alias": "", "id": 8, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 15, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_5_out"}}], [6, {"alias": "", "id": 6, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 14, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_3__out"}}], [5, {"alias": "", "id": 5, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 13, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "bitstream"], {"blocking": false, "group": "", "id": 12, "name": "bitstream", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 11, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "VideoEncoder", "parentId": -1, "properties": {"bitrate": 0, "frameRate": 30.0, "keyframeFrequency": 30, "lossless": false, "maxBitrate": 0, "numBFrames": 0, "numFramesPool": 0, "outputFrameSize": 0, "profile": 4, "quality": 95, "rateCtrlMode": 0}}], [3, {"alias": "detectionParser", "id": 3, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 10, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "imageIn"], {"blocking": false, "group": "", "id": 9, "name": "imageIn", "queueSize": 1, "type": 3, "waitForMessage": true}], [["", "in"], {"blocking": true, "group": "", "id": 8, "name": "in", "queueSize": 1, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "DetectionParser", "parentId": 1, "properties": {"networkInputs": {"images": {"dataType": 1, "dims": [416, 416, 3, 1], "name": "images", "numDimensions": 4, "offset": 0, "order": 17185, "qpScale": 1.0, "qpZp": 0.0, "quantization": false, "strides": []}}, "numFramesPool": 8, "parser": {"anchorMasks": {}, "anchors": [], "anchorsV2": [], "classNames": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"], "classes": 80, "confidenceThreshold": 0.5, "coordinates": 4, "iouThreshold": 0.5, "nnFamily": 0, "subtype": "yolov6"}}}], [2, {"alias": "neuralNetwork", "id": 2, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 7, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 6, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 5, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": 1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 8689834, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 4, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["dynamicOutputs", "1"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "1", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 100663297, "aeRegion": {"height": 0, "priority": 2434620032, "width": 0, "x": 4163, "y": 1}, "afRegion": {"height": 0, "priority": 0, "width": 0, "x": 21857, "y": 0}, "antiBandingMode": 29, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 200, "brightness": -105, "captureIntent": 145, "chromaDenoise": 0, "cmdMask": 0, "contrast": -33, "controlMode": 97, "effectMode": 85, "enableHdr": false, "expCompensation": 39, "expManual": {"exposureTimeUs": 21857, "frameDurationUs": 21857, "sensitivityIso": 2434620140}, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 91, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": -34, "sceneMode": 90, "sharpness": 0, "strobeConfig": {"activeLevel": 40, "enable": 0, "gpioNumber": 97}, "strobeTimings": {"durationUs": 16, "exposureBeginOffsetUs": 21857, "exposureEndOffsetUs": 17}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [416, 416]}}, "type": 8}, {"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [1920, 1440]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## 源代码 #### Python ```python #!/usr/bin/env python3 import depthai as dai from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("--webSocketPort", type=int, default=8765) parser.add_argument("--httpPort", type=int, default=8082) args = parser.parse_args() remoteConnector = dai.RemoteConnection(webSocketPort=args.webSocketPort, httpPort=args.httpPort) ENCODER_PROFILE = dai.VideoEncoderProperties.Profile.MJPEG class ImgAnnotationsGenerator(dai.node.ThreadedHostNode): def __init__(self): super().__init__() self.inputDet = self.createInput() self.output = self.createOutput() def setLabelMap(self, labelMap): self.labelMap = labelMap def run(self): while self.mainLoop(): nnData = self.inputDet.get() detections = nnData.detections imgAnnt = dai.ImgAnnotations() imgAnnt.setTimestamp(nnData.getTimestamp()) imgAnnt.setTransformation(nnData.getTransformation()) annotation = dai.ImgAnnotation() for detection in detections: pointsAnnotation = dai.PointsAnnotation() pointsAnnotation.type = dai.PointsAnnotationType.LINE_STRIP pointsAnnotation.points = dai.VectorPoint2f([ dai.Point2f(detection.xmin, detection.ymin), dai.Point2f(detection.xmax, detection.ymin), dai.Point2f(detection.xmax, detection.ymax), dai.Point2f(detection.xmin, detection.ymax), ]) outlineColor = dai.Color(1.0, 0.5, 0.5, 1.0) pointsAnnotation.outlineColor = outlineColor fillColor = dai.Color(0.5, 1.0, 0.5, 0.5) pointsAnnotation.fillColor = fillColor pointsAnnotation.thickness = 2.0 text = dai.TextAnnotation() text.position = dai.Point2f(detection.xmin, detection.ymin) text.text = f"{self.labelMap[detection.label]} {int(detection.confidence * 100)}%" text.fontSize = 50.5 textColor = dai.Color(0.5, 0.5, 1.0, 1.0) text.textColor = textColor backgroundColor = dai.Color(1.0, 1.0, 0.5, 1.0) text.backgroundColor = backgroundColor annotation.points.append(pointsAnnotation) annotation.texts.append(text) imgAnnt.annotations.append(annotation) self.output.send(imgAnnt) # Create pipeline with dai.Pipeline() as pipeline: cameraNode = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A) detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build( cameraNode, dai.NNModelDescription("yolov6-nano") ) imageAnnotationsGenerator = pipeline.create(ImgAnnotationsGenerator) outputToEncode = cameraNode.requestOutput((1920, 1440), type=dai.ImgFrame.Type.NV12) encoder = pipeline.create(dai.node.VideoEncoder) encoder.setDefaultProfilePreset(30, ENCODER_PROFILE) outputToEncode.link(encoder.input) detectionNetwork.out.link(imageAnnotationsGenerator.inputDet) labelMap = detectionNetwork.getClasses() imageAnnotationsGenerator.setLabelMap(labelMap) # Add the remote connector topics remoteConnector.addTopic("encoded", encoder.out, "images") remoteConnector.addTopic("detections", detectionNetwork.out, "images") remoteConnector.addTopic("annotations", imageAnnotationsGenerator.output, "images") pipeline.start() # Register the pipeline with the remote connector remoteConnector.registerPipeline(pipeline) while pipeline.isRunning(): if remoteConnector.waitKey(1) == ord("q"): pipeline.stop() break ``` #### C++ ```cpp #include #include #include #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImgAnnotations.hpp" #include "depthai/remote_connection/RemoteConnection.hpp" // Global flag for graceful shutdown std::atomic quitEvent(false); // Signal handler void signalHandler(int signum) { quitEvent = true; } // Custom host node for image annotations class ImgAnnotationsGenerator : public dai::NodeCRTP { public: Input& inputDet = inputs["detections"]; Output& output = out; std::vector labelMap; std::shared_ptr build(Output& detections) { detections.link(inputDet); return std::static_pointer_cast(this->shared_from_this()); } std::shared_ptr processGroup(std::shared_ptr in) override { auto nnData = in->get("detections"); auto detections = nnData->detections; auto imgAnnt = std::make_shared(); imgAnnt->setTimestamp(nnData->getTimestamp()); imgAnnt->transformation = nnData->transformation; dai::ImgAnnotation annotation; for(const auto& detection : detections) { // Create points annotation for bounding box auto pointsAnnotation = std::make_shared(); pointsAnnotation->type = dai::PointsAnnotationType::LINE_STRIP; pointsAnnotation->points = {dai::Point2f(detection.xmin, detection.ymin), dai::Point2f(detection.xmax, detection.ymin), dai::Point2f(detection.xmax, detection.ymax), dai::Point2f(detection.xmin, detection.ymax)}; // Set colors and thickness pointsAnnotation->outlineColor = dai::Color(1.0f, 0.5f, 0.5f, 1.0f); pointsAnnotation->fillColor = dai::Color(0.5f, 1.0f, 0.5f, 0.5f); pointsAnnotation->thickness = 2.0f; // Create text annotation auto text = std::make_shared(); text->position = dai::Point2f(detection.xmin, detection.ymin); // Get label text std::string labelText; try { labelText = labelMap[detection.label]; } catch(...) { labelText = std::to_string(detection.label); } text->text = labelText + " " + std::to_string(static_cast(detection.confidence * 100)) + "%"; text->fontSize = 50.5f; text->textColor = dai::Color(0.5f, 0.5f, 1.0f, 1.0f); text->backgroundColor = dai::Color(1.0f, 1.0f, 0.5f, 1.0f); annotation.points.push_back(*pointsAnnotation); annotation.texts.push_back(*text); } imgAnnt->annotations.push_back(annotation); return imgAnnt; } }; int main(int argc, char** argv) { // Set up signal handlers signal(SIGTERM, signalHandler); signal(SIGINT, signalHandler); try { // Create remote connection dai::RemoteConnection remoteConnector; // Create pipeline dai::Pipeline pipeline; // Create nodes auto cameraNode = pipeline.create(); cameraNode->build(dai::CameraBoardSocket::CAM_A); auto detectionNetwork = pipeline.create(); dai::NNModelDescription modelDesc; modelDesc.model = "yolov6-nano"; detectionNetwork->build(cameraNode, modelDesc); auto imageAnnotationsGenerator = pipeline.create(); auto outputToEncode = cameraNode->requestOutput(std::make_pair(1920, 1440), dai::ImgFrame::Type::NV12); auto encoder = pipeline.create(); encoder->setDefaultProfilePreset(30, dai::VideoEncoderProperties::Profile::MJPEG); // Linking outputToEncode->link(encoder->input); detectionNetwork->out.link(imageAnnotationsGenerator->inputDet); // Set label map imageAnnotationsGenerator->labelMap = detectionNetwork->getClasses().value(); // Add remote connector topics remoteConnector.addTopic("encoded", encoder->out, "images"); remoteConnector.addTopic("detections", detectionNetwork->out, "images"); remoteConnector.addTopic("annotations", imageAnnotationsGenerator->output, "images"); // Start pipeline pipeline.start(); // Register pipeline with remote connector remoteConnector.registerPipeline(pipeline); // Main loop while(pipeline.isRunning() && !quitEvent) { if(remoteConnector.waitKey(1) == 'q') { pipeline.stop(); break; } } // Cleanup pipeline.stop(); pipeline.wait(); } catch(const std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; } return 0; } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。