# 检测网络重映射 本示例演示了在 RGB 流上运行 YOLOv6 物体检测，同时处理立体深度数据，并在 RGB 图像和彩色深度帧上显示边界框。 这个示例需要DepthAI v3 API，参见[安装说明](https://docs.luxonis.com/software-v3/depthai.md)。 ## 流水线 ### examples/detection_network_remap.pipeline.json ```json {"pipeline": {"connections": [{"node1Id": 6, "node1Output": "disparity", "node1OutputGroup": "", "node2Id": 11, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 5, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 6, "node2Input": "right", "node2InputGroup": ""}, {"node1Id": 4, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 6, "node2Input": "left", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 7, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "passthrough", "node1OutputGroup": "", "node2Id": 3, "node2Input": "imageIn", "node2InputGroup": ""}, {"node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 3, "node1Output": "out", "node1OutputGroup": "", "node2Id": 9, "node2Input": "in", "node2InputGroup": ""}, {"node1Id": 0, "node1Output": "0", "node1OutputGroup": "dynamicOutputs", "node2Id": 2, "node2Input": "in", "node2InputGroup": ""}], "globalProperties": {"calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1}, "nodes": [[11, {"alias": "", "id": 11, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 37, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_6_disparity"}}], [9, {"alias": "", "id": 9, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 36, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_3_out"}}], [7, {"alias": "", "id": 7, "ioInfo": [[["", "in"], {"blocking": true, "group": "", "id": 35, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "XLinkOut", "parentId": -1, "properties": {"maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "__x_2_passthrough"}}], [6, {"alias": "", "id": 6, "ioInfo": [[["", "confidenceMap"], {"blocking": false, "group": "", "id": 34, "name": "confidenceMap", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "debugDispCostDump"], {"blocking": false, "group": "", "id": 33, "name": "debugDispCostDump", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "debugExtDispLrCheckIt2"], {"blocking": false, "group": "", "id": 32, "name": "debugExtDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "debugDispLrCheckIt2"], {"blocking": false, "group": "", "id": 30, "name": "debugDispLrCheckIt2", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "debugExtDispLrCheckIt1"], {"blocking": false, "group": "", "id": 31, "name": "debugExtDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "debugDispLrCheckIt1"], {"blocking": false, "group": "", "id": 29, "name": "debugDispLrCheckIt1", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "outConfig"], {"blocking": false, "group": "", "id": 28, "name": "outConfig", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "rectifiedRight"], {"blocking": false, "group": "", "id": 27, "name": "rectifiedRight", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "rectifiedLeft"], {"blocking": false, "group": "", "id": 26, "name": "rectifiedLeft", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "depth"], {"blocking": false, "group": "", "id": 22, "name": "depth", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "right"], {"blocking": true, "group": "", "id": 21, "name": "right", "queueSize": 3, "type": 3, "waitForMessage": false}], [["", "left"], {"blocking": true, "group": "", "id": 20, "name": "left", "queueSize": 3, "type": 3, "waitForMessage": false}], [["", "syncedRight"], {"blocking": false, "group": "", "id": 25, "name": "syncedRight", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "syncedLeft"], {"blocking": false, "group": "", "id": 24, "name": "syncedLeft", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputAlignTo"], {"blocking": false, "group": "", "id": 19, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true}], [["", "disparity"], {"blocking": false, "group": "", "id": 23, "name": "disparity", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "inputConfig"], {"blocking": true, "group": "", "id": 18, "name": "inputConfig", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "StereoDepth", "parentId": -1, "properties": {"alphaScaling": null, "baseline": null, "depthAlignCamera": -1, "depthAlignmentUseSpecTranslation": null, "disparityToDepthUseSpecTranslation": null, "enableFrameSync": true, "enableRectification": true, "enableRuntimeStereoModeSwitch": false, "focalLength": null, "focalLengthFromCalibration": true, "height": null, "initialConfig": {"algorithmControl": {"centerAlignmentShiftFactor": null, "customDepthUnitMultiplier": 1000.0, "depthAlign": 1, "depthUnit": 2, "disparityShift": 0, "enableExtended": true, "enableLeftRightCheck": true, "enableSubpixel": true, "enableSwLeftRightCheck": false, "leftRightCheckThreshold": 10, "numInvalidateEdgePixels": 0, "subpixelFractionalBits": 5}, "censusTransform": {"enableMeanMode": true, "kernelMask": 0, "kernelSize": -1, "noiseThresholdOffset": 1, "noiseThresholdScale": 1, "threshold": 0}, "confidenceMetrics": {"flatnessConfidenceThreshold": 2, "flatnessConfidenceWeight": 8, "flatnessOverride": false, "motionVectorConfidenceThreshold": 1, "motionVectorConfidenceWeight": 4, "occlusionConfidenceWeight": 20}, "costAggregation": {"divisionFactor": 1, "horizontalPenaltyCostP1": 250, "horizontalPenaltyCostP2": 500, "p1Config": {"defaultValue": 11, "edgeThreshold": 15, "edgeValue": 10, "enableAdaptive": true, "smoothThreshold": 5, "smoothValue": 22}, "p2Config": {"defaultValue": 33, "edgeValue": 22, "enableAdaptive": true, "smoothValue": 63}, "verticalPenaltyCostP1": 250, "verticalPenaltyCostP2": 500}, "costMatching": {"confidenceThreshold": 55, "disparityWidth": 1, "enableCompanding": false, "enableSwConfidenceThresholding": false, "invalidDisparityValue": 0, "linearEquationParameters": {"alpha": 0, "beta": 2, "threshold": 127}}, "filtersBackend": 2, "postProcessing": {"adaptiveMedianFilter": {"confidenceThreshold": 200, "enable": true}, "bilateralSigmaValue": 0, "brightnessFilter": {"maxBrightness": 256, "minBrightness": 0}, "decimationFilter": {"decimationFactor": 1, "decimationMode": 0}, "filteringOrder": [3, 1, 2, 4, 5], "holeFilling": {"enable": true, "fillConfidenceThreshold": 200, "highConfidenceThreshold": 210, "invalidateDisparities": true, "minValidDisparity": 1}, "median": 0, "spatialFilter": {"alpha": 0.5, "delta": 0, "enable": false, "holeFillingRadius": 2, "numIterations": 1}, "speckleFilter": {"differenceThreshold": 2, "enable": false, "speckleRange": 50}, "temporalFilter": {"alpha": 0.4000000059604645, "delta": 0, "enable": false, "persistencyMode": 3}, "thresholdFilter": {"maxRange": 65535, "minRange": 0}}}, "mesh": {"meshLeftUri": "", "meshRightUri": "", "meshSize": null, "stepHeight": 16, "stepWidth": 16}, "numFramesPool": 3, "numPostProcessingMemorySlices": -1, "numPostProcessingShaves": -1, "outHeight": null, "outKeepAspectRatio": true, "outWidth": null, "rectificationUseSpecTranslation": null, "rectifyEdgeFillColor": 0, "useHomographyRectification": null, "width": null}}], [5, {"alias": "", "id": 5, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 17, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 16, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 15, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 14, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 2, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 48, "aeRegion": {"height": 49249, "priority": 3611420857, "width": 26231, "x": 50422, "y": 17299}, "afRegion": {"height": 44658, "priority": 3128807845, "width": 19054, "x": 30871, "y": 864}, "antiBandingMode": 134, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 16, "brightness": 0, "captureIntent": 108, "chromaDenoise": 87, "cmdMask": 0, "contrast": 10, "controlMode": 27, "effectMode": 138, "enableHdr": false, "expCompensation": 0, "expManual": {"exposureTimeUs": 3098448084, "frameDurationUs": 2264028624, "sensitivityIso": 3256143576}, "frameSyncMode": 185, "lensPosAutoInfinity": 169, "lensPosAutoMacro": 93, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 202, "miscControls": [], "saturation": -35, "sceneMode": 158, "sharpness": 101, "strobeConfig": {"activeLevel": 51, "enable": 89, "gpioNumber": -8}, "strobeTimings": {"durationUs": 0, "exposureBeginOffsetUs": -1175339376, "exposureEndOffsetUs": 272}, "wbColorTemp": 0}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [1280, 720]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}], [4, {"alias": "", "id": 4, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 13, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 12, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 11, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 10, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 1, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 740438574, "aeRegion": {"height": 8202, "priority": 2099257376, "width": 23840, "x": 8224, "y": 8224}, "afRegion": {"height": 27938, "priority": 1818584175, "width": 8224, "x": 2604, "y": 8224}, "antiBandingMode": 110, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 99, "brightness": 32, "captureIntent": 118, "chromaDenoise": 100, "cmdMask": 0, "contrast": 32, "controlMode": 101, "effectMode": 114, "enableHdr": false, "expCompensation": 32, "expManual": {"exposureTimeUs": 1663986026, "frameDurationUs": 538976266, "sensitivityIso": 577138287}, "frameSyncMode": 116, "lensPosAutoInfinity": 100, "lensPosAutoMacro": 110, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 111, "lowPowerNumFramesDiscard": 112, "lumaDenoise": 111, "miscControls": [], "saturation": 34, "sceneMode": 111, "sharpness": 109, "strobeConfig": {"activeLevel": 114, "enable": 101, "gpioNumber": 95}, "strobeTimings": {"durationUs": 825110562, "exposureBeginOffsetUs": 1869181810, "exposureEndOffsetUs": 540680814}, "wbColorTemp": 24428}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [1280, 720]}}, "type": 22}], "resolutionHeight": -1, "resolutionWidth": -1}}], [3, {"alias": "detectionParser", "id": 3, "ioInfo": [[["", "out"], {"blocking": false, "group": "", "id": 9, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "imageIn"], {"blocking": false, "group": "", "id": 8, "name": "imageIn", "queueSize": 1, "type": 3, "waitForMessage": true}], [["", "in"], {"blocking": true, "group": "", "id": 7, "name": "in", "queueSize": 1, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "DetectionParser", "parentId": 1, "properties": {"networkInputs": {"images": {"dataType": 1, "dims": [416, 416, 3, 1], "name": "images", "numDimensions": 4, "offset": 0, "order": 17185, "qpScale": 1.0, "qpZp": 0.0, "quantization": false, "strides": []}}, "numFramesPool": 8, "parser": {"anchorMasks": {}, "anchors": [], "anchorsV2": [], "classNames": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"], "classes": 80, "confidenceThreshold": 0.5, "coordinates": 4, "iouThreshold": 0.5, "nnFamily": 0, "subtype": "yolov6"}}}], [2, {"alias": "neuralNetwork", "id": 2, "ioInfo": [[["", "passthrough"], {"blocking": false, "group": "", "id": 6, "name": "passthrough", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "out"], {"blocking": false, "group": "", "id": 5, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "in"], {"blocking": true, "group": "", "id": 4, "name": "in", "queueSize": 3, "type": 3, "waitForMessage": true}]], "logLevel": 3, "name": "NeuralNetwork", "parentId": 1, "properties": {"backend": "", "backendProperties": {}, "blobSize": 8689834, "blobUri": "asset:__blob", "modelSource": 0, "modelUri": "", "numFrames": 8, "numNCEPerThread": 0, "numShavesPerThread": 0, "numThreads": 0}}], [0, {"alias": "", "id": 0, "ioInfo": [[["dynamicOutputs", "0"], {"blocking": false, "group": "dynamicOutputs", "id": 3, "name": "0", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "raw"], {"blocking": false, "group": "", "id": 2, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false}], [["", "mockIsp"], {"blocking": true, "group": "", "id": 1, "name": "mockIsp", "queueSize": 8, "type": 3, "waitForMessage": false}], [["", "inputControl"], {"blocking": true, "group": "", "id": 0, "name": "inputControl", "queueSize": 3, "type": 3, "waitForMessage": false}]], "logLevel": 3, "name": "Camera", "parentId": -1, "properties": {"boardSocket": 0, "cameraName": "", "fps": -1.0, "imageOrientation": -1, "initialControl": {"aeLockMode": false, "aeMaxExposureTimeUs": 65544, "aeRegion": {"height": 0, "priority": 0, "width": 0, "x": 0, "y": 0}, "afRegion": {"height": 43, "priority": 131077, "width": 12, "x": 0, "y": 0}, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 136, "cmdMask": 0, "contrast": -116, "controlMode": 12, "effectMode": 0, "enableHdr": false, "expCompensation": 0, "expManual": {"exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0}, "frameSyncMode": 6, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 1, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "miscControls": [], "saturation": 0, "sceneMode": 0, "sharpness": 2, "strobeConfig": {"activeLevel": 110, "enable": 0, "gpioNumber": 111}, "strobeTimings": {"durationUs": 2949132, "exposureBeginOffsetUs": 6649189, "exposureEndOffsetUs": 1704760}, "wbColorTemp": 1}, "isp3aFps": 0, "mockIspHeight": -1, "mockIspWidth": -1, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "outputRequests": [{"enableUndistortion": null, "fps": {"value": null}, "resizeMode": 0, "size": {"value": {"index": 0, "value": [416, 416]}}, "type": 8}], "resolutionHeight": -1, "resolutionWidth": -1}}]]}} ``` ## 源代码 #### Python ```python #!/usr/bin/env python3 import cv2 import depthai as dai import numpy as np def colorizeDepth(frameDepth): invalidMask = frameDepth == 0 # Log the depth, minDepth and maxDepth try: minDepth = np.percentile(frameDepth[frameDepth != 0], 3) maxDepth = np.percentile(frameDepth[frameDepth != 0], 95) logDepth = np.log(frameDepth, where=frameDepth != 0) logMinDepth = np.log(minDepth) logMaxDepth = np.log(maxDepth) np.nan_to_num(logDepth, copy=False, nan=logMinDepth) # Clip the values to be in the 0-255 range logDepth = np.clip(logDepth, logMinDepth, logMaxDepth) # Interpolate only valid logDepth values, setting the rest based on the mask depthFrameColor = np.interp(logDepth, (logMinDepth, logMaxDepth), (0, 255)) depthFrameColor = np.nan_to_num(depthFrameColor) depthFrameColor = depthFrameColor.astype(np.uint8) depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_JET) # Set invalid depth pixels to black depthFrameColor[invalidMask] = 0 except IndexError: # Frame is likely empty depthFrameColor = np.zeros((frameDepth.shape[0], frameDepth.shape[1], 3), dtype=np.uint8) except Exception as e: raise e return depthFrameColor # Create pipeline with dai.Pipeline() as pipeline: cameraNode = pipeline.create(dai.node.Camera).build() detectionNetwork = pipeline.create(dai.node.DetectionNetwork).build(cameraNode, dai.NNModelDescription("yolov6-nano")) labelMap = detectionNetwork.getClasses() monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B) monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C) stereo = pipeline.create(dai.node.StereoDepth) # Linking monoLeftOut = monoLeft.requestOutput((1280, 720)) monoRightOut = monoRight.requestOutput((1280, 720)) monoLeftOut.link(stereo.left) monoRightOut.link(stereo.right) stereo.setRectification(True) stereo.setExtendedDisparity(True) stereo.setLeftRightCheck(True) stereo.setSubpixel(True) qRgb = detectionNetwork.passthrough.createOutputQueue() qDet = detectionNetwork.out.createOutputQueue() qDepth = stereo.disparity.createOutputQueue() pipeline.start() def displayFrame(name: str, frame: dai.ImgFrame, imgDetections: dai.ImgDetections): color = (0, 255, 0) assert imgDetections.getTransformation() is not None cvFrame = frame.getFrame() if frame.getType() == dai.ImgFrame.Type.RAW16 else frame.getCvFrame() if(frame.getType() == dai.ImgFrame.Type.RAW16): cvFrame = colorizeDepth(cvFrame) for detection in imgDetections.detections: # Get the shape of the frame from which the detections originated for denormalization normShape = imgDetections.getTransformation().getSize() # Create rotated rectangle to remap # Here we use an intermediate dai.Rect to create a dai.RotatedRect to simplify construction and denormalization rotRect = dai.RotatedRect(dai.Rect(dai.Point2f(detection.xmin, detection.ymin), dai.Point2f(detection.xmax, detection.ymax)).denormalize(normShape[0], normShape[1]), 0) # Remap the detection rectangle to target frame remapped = imgDetections.getTransformation().remapRectTo(frame.getTransformation(), rotRect) # Remapped rectangle could be rotated, so we get the bounding box bbox = [int(l) for l in remapped.getOuterRect()] cv2.putText( cvFrame, labelMap[detection.label], (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255, ) cv2.putText( cvFrame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255, ) cv2.rectangle(cvFrame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2) # Show the frame cv2.imshow(name, cvFrame) while pipeline.isRunning(): inRgb: dai.ImgFrame = qRgb.get() inDet: dai.ImgDetections = qDet.get() inDepth: dai.ImgFrame = qDepth.get() hasRgb = inRgb is not None hasDepth = inDepth is not None hasDet = inDet is not None if hasRgb: displayFrame("rgb", inRgb, inDet) if hasDepth: displayFrame("depth", inDepth, inDet) if cv2.waitKey(1) == ord("q"): pipeline.stop() break ``` #### C++ ```cpp #include // Required for std::sort and std::unique #include // Required for std::log, std::isnan, std::isinf #include #include #include #include #include #include "depthai/depthai.hpp" #include "xtensor/containers/xadapt.hpp" #include "xtensor/core/xmath.hpp" std::atomic quitEvent(false); void signalHandler(int) { quitEvent = true; } cv::Mat colorizeDepth(cv::Mat frameDepth) { cv::Mat invalidMask = frameDepth == 0; cv::Mat depthFrameColor; try { cv::Mat frameDepthFloat; frameDepth.convertTo(frameDepthFloat, CV_32F); xt::xtensor depth = xt::adapt((float*)frameDepthFloat.data, {static_cast(frameDepthFloat.rows), static_cast(frameDepthFloat.cols)}); // Get valid depth values (non-zero) std::vector validDepth; validDepth.reserve(depth.size()); std::copy_if(depth.begin(), depth.end(), std::back_inserter(validDepth), [](float x) { return x != 0; }); if(validDepth.size() == 0) { return cv::Mat::zeros(frameDepth.rows, frameDepth.cols, CV_8UC3); } // Calculate percentiles std::sort(validDepth.begin(), validDepth.end()); float minDepth = validDepth[static_cast(validDepth.size() * 0.03)]; float maxDepth = validDepth[static_cast(validDepth.size() * 0.95)]; // Take log of depth values auto logDepth = xt::eval(xt::log(depth)); float logMinDepth = std::log(minDepth); float logMaxDepth = std::log(maxDepth); // Replace invalid values with logMinDepth using a naive implementation auto logDepthData = logDepth.data(); auto depthData = depth.data(); const size_t size = depth.size(); for(size_t i = 0; i < size; i++) { if(std::isnan(logDepthData[i]) || std::isinf(logDepthData[i]) || depthData[i] == 0.0f) { logDepthData[i] = logMinDepth; } } // Clip values logDepth = xt::clip(logDepth, logMinDepth, logMaxDepth); // Normalize to 0-255 range auto normalizedDepth = (logDepth - logMinDepth) / (logMaxDepth - logMinDepth) * 255.0f; // Convert to CV_8UC1 cv::Mat depthMat(frameDepth.rows, frameDepth.cols, CV_8UC1); std::transform(normalizedDepth.begin(), normalizedDepth.end(), depthMat.data, [](float x) { return static_cast(x); }); // Apply colormap cv::applyColorMap(depthMat, depthFrameColor, cv::COLORMAP_JET); // Set invalid pixels to black depthFrameColor.setTo(cv::Scalar(0, 0, 0), invalidMask); } catch(const std::exception& e) { std::cerr << "Error in colorizeDepth: " << e.what() << std::endl; return cv::Mat::zeros(frameDepth.rows, frameDepth.cols, CV_8UC3); } return depthFrameColor; } // Helper function to display frames with detections void displayFrame(const std::string& name, std::shared_ptr frame, std::shared_ptr imgDetections, const std::vector& labelMap) { cv::Scalar color(0, 255, 0); cv::Mat cvFrame; if(frame->getType() == dai::ImgFrame::Type::RAW16) { cvFrame = colorizeDepth(frame->getFrame()); } else { cvFrame = frame->getCvFrame(); } if(!imgDetections || !imgDetections->transformation.has_value()) { // std::cout << "No detections or transformation data for " << name << std::endl; cv::imshow(name, cvFrame); return; } const auto& sourceTransform = *(imgDetections->transformation); const auto& targetTransform = frame->transformation; for(const auto& detection : imgDetections->detections) { auto normShape = sourceTransform.getSize(); dai::Rect rect(dai::Point2f(detection.xmin, detection.ymin), dai::Point2f(detection.xmax, detection.ymax)); rect = rect.denormalize(static_cast(normShape.first), static_cast(normShape.second)); dai::RotatedRect rotRect(rect, 0); auto remapped = sourceTransform.remapRectTo(targetTransform, rotRect); auto bbox = remapped.getOuterRect(); cv::putText(cvFrame, labelMap[detection.label], cv::Point(static_cast(bbox[0]) + 10, static_cast(bbox[1]) + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, cv::Scalar(255, 255, 255)); cv::putText(cvFrame, std::to_string(static_cast(detection.confidence * 100)) + "%", cv::Point(static_cast(bbox[0]) + 10, static_cast(bbox[1]) + 40), cv::FONT_HERSHEY_TRIPLEX, 0.5, cv::Scalar(255, 255, 255)); cv::rectangle(cvFrame, cv::Point(static_cast(bbox[0]), static_cast(bbox[1])), cv::Point(static_cast(bbox[2]), static_cast(bbox[3])), color, 2); } cv::imshow(name, cvFrame); } int main() { signal(SIGTERM, signalHandler); signal(SIGINT, signalHandler); dai::Pipeline pipeline; auto cameraNode = pipeline.create(); cameraNode->build(); auto detectionNetwork = pipeline.create(); dai::NNModelDescription modelDescription; modelDescription.model = "yolov6-nano"; detectionNetwork->build(cameraNode, modelDescription); auto labelMap = detectionNetwork->getClasses().value_or(std::vector{}); auto monoLeft = pipeline.create(); monoLeft->build(dai::CameraBoardSocket::CAM_B); auto monoRight = pipeline.create(); monoRight->build(dai::CameraBoardSocket::CAM_C); auto stereo = pipeline.create(); // Linking auto monoLeftOut = monoLeft->requestOutput(std::make_pair(1280, 720)); auto monoRightOut = monoRight->requestOutput(std::make_pair(1280, 720)); monoLeftOut->link(stereo->left); monoRightOut->link(stereo->right); stereo->setRectification(true); stereo->setExtendedDisparity(true); stereo->setLeftRightCheck(true); stereo->setSubpixel(true); auto qRgb = detectionNetwork->passthrough.createOutputQueue(); auto qDet = detectionNetwork->out.createOutputQueue(); auto qDepth = stereo->disparity.createOutputQueue(); pipeline.start(); while(pipeline.isRunning() && !quitEvent) { auto inRgb = qRgb->tryGet(); auto inDet = qDet->tryGet(); auto inDepth = qDepth->tryGet(); bool hasRgb = inRgb != nullptr; bool hasDepth = inDepth != nullptr; bool hasDet = inDet != nullptr; if(hasRgb && hasDet) { displayFrame("rgb", inRgb, inDet, labelMap); } if(hasDepth && hasDet) { displayFrame("depth", inDepth, inDet, labelMap); } if(cv::waitKey(1) == 'q') { pipeline.stop(); break; } } return 0; } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。