# RGB-热成像对齐 此示例演示了如何将热成像相机的热信息与RGB相机对齐。该设置适用于需要叠加或比较热成像与彩色数据的应用场景。将创建一个OpenCV窗口来显示RGB与对齐后热成像数据的混合图像。提供滑块以调整混合比例。 ## 演示 这个示例需要DepthAI v3 API，参见[安装说明](https://docs.luxonis.com/software-v3/depthai.md)。 ## 源代码 #### Python ```python #!/usr/bin/env python3 """ Due to an issue with our calibration, you might receive the following error when running this script on early release OAK Thermal devices: ```bash [ImageAlign(4)] [error] Failed to get calibration data: Extrinsic connection between the requested cameraId's doesn't exist. Please recalibrate or modify your calibration data ``` If this happens, please download the calibration data + script from https://drive.google.com/drive/folders/1Q_MZMqWMKDC1eOqVHGPeDO-NJgFmnY5U, place them into the same folder, connect the camera to the computer and run the script. This will update the calibration and add required extrinsics between the camera sensors. """ import cv2 import depthai as dai import numpy as np import time from datetime import timedelta FPS = 25.0 RGB_SOCKET = dai.CameraBoardSocket.CAM_A COLOR_RESOLUTION = dai.ColorCameraProperties.SensorResolution.THE_1080_P class FPSCounter: def __init__(self): self.frameTimes = [] def tick(self): now = time.time() self.frameTimes.append(now) self.frameTimes = self.frameTimes[-100:] def getFps(self): if len(self.frameTimes) <= 1: return 0 # Calculate the FPS return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0]) device = dai.Device() thermalWidth, thermalHeight = -1, -1 thermalFound = False for features in device.getConnectedCameraFeatures(): if dai.CameraSensorType.THERMAL in features.supportedTypes: thermalFound = True thermalSocket = features.socket thermalWidth, thermalHeight = features.width, features.height break if not thermalFound: raise RuntimeError("No thermal camera found!") ISP_SCALE = 3 calibrationHandler = device.readCalibration() rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET) distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET) if distortionModel != dai.CameraModel.Perspective: raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.") pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) thermalCam = pipeline.create(dai.node.Camera) thermalCam.setBoardSocket(thermalSocket) thermalCam.setFps(FPS) sync = pipeline.create(dai.node.Sync) out = pipeline.create(dai.node.XLinkOut) align = pipeline.create(dai.node.ImageAlign) cfgIn = pipeline.create(dai.node.XLinkIn) camRgb.setBoardSocket(RGB_SOCKET) camRgb.setResolution(COLOR_RESOLUTION) camRgb.setFps(FPS) camRgb.setIspScale(1,ISP_SCALE) out.setStreamName("out") sync.setSyncThreshold(timedelta(seconds=0.5 / FPS)) cfgIn.setStreamName("config") cfg = align.initialConfig.get() staticDepthPlane = cfg.staticDepthPlane # Linking align.outputAligned.link(sync.inputs["aligned"]) camRgb.isp.link(sync.inputs["rgb"]) camRgb.isp.link(align.inputAlignTo) thermalCam.raw.link(align.input) sync.out.link(out.input) cfgIn.out.link(align.inputConfig) rgbWeight = 0.4 thermalWeight = 0.6 def updateBlendWeights(percentRgb): """ Update the rgb and depth weights used to blend depth/rgb image @param[in] percent_rgb The rgb weight expressed as a percentage (0..100) """ global thermalWeight global rgbWeight rgbWeight = float(percentRgb) / 100.0 thermalWeight = 1.0 - rgbWeight def updateDepthPlane(depth): global staticDepthPlane staticDepthPlane = depth # Connect to device and start pipeline with device: device.startPipeline(pipeline) queue = device.getOutputQueue("out", 8, False) cfgQ = device.getInputQueue("config") # Configure windows; trackbar adjusts blending ratio of rgb/depth windowName = "rgb-thermal" # Set the window to be resizable and the initial size cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) cv2.resizeWindow(windowName, 1280, 720) cv2.createTrackbar( "RGB Weight %", windowName, int(rgbWeight * 100), 100, updateBlendWeights, ) cv2.createTrackbar( "Static Depth Plane [mm]", windowName, 0, 2000, updateDepthPlane, ) fpsCounter = FPSCounter() while True: messageGroup = queue.get() assert isinstance(messageGroup, dai.MessageGroup) frameRgb = messageGroup["rgb"] assert isinstance(frameRgb, dai.ImgFrame) thermalAligned = messageGroup["aligned"] assert isinstance(thermalAligned, dai.ImgFrame) frameRgbCv = frameRgb.getCvFrame() fpsCounter.tick() rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(frameRgbCv.shape[1]), int(frameRgbCv.shape[0])) cvFrameUndistorted = cv2.undistort( frameRgbCv, np.array(rgbIntrinsics), np.array(rgbDistortion), ) # Colorize the aligned depth thermalFrame = thermalAligned.getCvFrame().astype(np.float32) # Create a mask for nan values mask = np.isnan(thermalFrame) # Replace nan values with a mean for visualization thermalFrame[mask] = np.nanmean(thermalFrame) thermalFrame = cv2.normalize(thermalFrame, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U) colormappedFrame = cv2.applyColorMap(thermalFrame, cv2.COLORMAP_MAGMA) # Apply the mask back with black pixels (0) colormappedFrame[mask] = 0 blended = cv2.addWeighted(cvFrameUndistorted, rgbWeight, colormappedFrame, thermalWeight, 0) cv2.putText( blended, f"FPS: {fpsCounter.getFps():.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, ) cv2.imshow(windowName, blended) key = cv2.waitKey(1) if key == ord("q"): break cfg.staticDepthPlane = staticDepthPlane cfgQ.send(cfg) ``` #### C++ ```cpp #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImageAlignConfig.hpp" constexpr auto FPS = 25.0; constexpr auto RGB_SOCKET = dai::CameraBoardSocket::CAM_A; constexpr auto COLOR_RESOLUTION = dai::ColorCameraProperties::SensorResolution::THE_1080_P; class FPSCounter { public: void tick() { auto now = std::chrono::steady_clock::now(); frameTimes.push(now); if(frameTimes.size() > 100) { frameTimes.pop(); } } double getFps() { if(frameTimes.size() <= 1) return 0; auto duration = std::chrono::duration_cast(frameTimes.back() - frameTimes.front()).count(); return (frameTimes.size() - 1) * 1000.0 / duration; } private: std::queue frameTimes; }; double rgbWeight = 0.4; double thermalWeight = 0.6; int staticDepthPlane = 0; void updateBlendWeights(int percentRgb, void*) { rgbWeight = static_cast(percentRgb) / 100.0; thermalWeight = 1.0 - rgbWeight; } void updateDepthPlane(int depth, void*) { staticDepthPlane = depth; } cv::Mat createNaNMask(const cv::Mat& frame) { cv::Mat nanMask = cv::Mat::zeros(frame.size(), CV_8UC1); for(int r = 0; r < frame.rows; ++r) { for(int c = 0; c < frame.cols; ++c) { if(std::isnan(frame.at(r, c))) { nanMask.at(r, c) = 255; } } } return nanMask; } int main() { dai::Device device; int thermalWidth = -1, thermalHeight = -1; bool thermalFound = false; dai::CameraBoardSocket thermalSocket; for(const auto& features : device.getConnectedCameraFeatures()) { if(std::find(features.supportedTypes.begin(), features.supportedTypes.end(), dai::CameraSensorType::THERMAL) != features.supportedTypes.end()) { thermalFound = true; thermalSocket = features.socket; thermalWidth = features.width; thermalHeight = features.height; break; } } if(!thermalFound) { throw std::runtime_error("No thermal camera found!"); } dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto thermalCam = pipeline.create(); auto sync = pipeline.create(); auto out = pipeline.create(); auto align = pipeline.create(); auto cfgIn = pipeline.create(); thermalCam->setBoardSocket(thermalSocket); thermalCam->setFps(FPS); camRgb->setBoardSocket(RGB_SOCKET); camRgb->setResolution(COLOR_RESOLUTION); camRgb->setFps(FPS); camRgb->setIspScale(1, 3); out->setStreamName("out"); sync->setSyncThreshold(std::chrono::milliseconds(static_cast((1 / FPS) * 1000.0 * 0.5))); cfgIn->setStreamName("config"); align->outputAligned.link(sync->inputs["aligned"]); camRgb->isp.link(sync->inputs["rgb"]); camRgb->isp.link(align->inputAlignTo); thermalCam->raw.link(align->input); sync->out.link(out->input); cfgIn->out.link(align->inputConfig); device.startPipeline(pipeline); auto queue = device.getOutputQueue("out", 8, false); auto cfgQ = device.getInputQueue("config"); FPSCounter fpsCounter; std::string windowName = "rgb-thermal"; cv::namedWindow(windowName, cv::WINDOW_NORMAL); cv::resizeWindow(windowName, 1280, 720); cv::createTrackbar("RGB Weight %", windowName, nullptr, 100, updateBlendWeights); cv::createTrackbar("Static Depth Plane [mm]", windowName, nullptr, 2000, updateDepthPlane); while(true) { auto messageGroup = queue->get(); fpsCounter.tick(); auto frameRgb = messageGroup->get("rgb"); auto thermalAligned = messageGroup->get("aligned"); if(frameRgb && thermalAligned) { auto frameRgbCv = frameRgb->getCvFrame(); auto thermalFrame = thermalAligned->getFrame(true); // Colorize the aligned depth cv::Mat mask; cv::Mat colormappedFrame; cv::Mat thermalFrameFloat; thermalFrame.convertTo(thermalFrameFloat, CV_32F); // Get a mask for the nan values mask = createNaNMask(thermalFrameFloat); auto meanValue = cv::mean(thermalFrameFloat, ~mask); thermalFrameFloat.setTo(meanValue, mask); cv::normalize(thermalFrameFloat, thermalFrameFloat, 0, 255, cv::NORM_MINMAX, CV_8U); cv::applyColorMap(thermalFrameFloat, colormappedFrame, cv::COLORMAP_MAGMA); colormappedFrame.setTo(cv::Scalar(0, 0, 0), mask); cv::Mat blended; cv::addWeighted(frameRgbCv, rgbWeight, colormappedFrame, thermalWeight, 0, blended); cv::putText(blended, "FPS: " + std::to_string(fpsCounter.getFps()), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2); cv::imshow(windowName, blended); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { break; } auto cfg = align->initialConfig.get(); cfg.staticDepthPlane = staticDepthPlane; auto alignConfig = std::make_shared(); alignConfig->set(cfg); cfgQ->send(alignConfig); } return 0; } ``` ## 流水线 ### examples/thermal_align.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 4, "node1Output": "outputAligned", "node1OutputGroup": "", "node2Id": 2, "node2Input": "aligned", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 2, "node2Input": "rgb", "node2InputGroup": "inputs" }, { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputConfig", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputAlignTo", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "raw", "node1OutputGroup": "", "node2Id": 4, "node2Input": "input", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 3, "horizNumerator": 1, "vertDenominator": 3, "vertNumerator": 1 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 9, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 14, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 15, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 10, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 11, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 12, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 13, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 16, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Camera", "properties": { "boardSocket": -1, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "inputs", "aligned" ], { "blocking": true, "group": "inputs", "id": 17, "name": "aligned", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "inputs", "rgb" ], { "blocking": true, "group": "inputs", "id": 18, "name": "rgb", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 19, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "properties": { "syncAttempts": -1, "syncThresholdNs": 20000000 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 20, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "out" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 21, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 22, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "input" ], { "blocking": false, "group": "", "id": 23, "name": "input", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "outputAligned" ], { "blocking": false, "group": "", "id": 24, "name": "outputAligned", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthroughInput" ], { "blocking": false, "group": "", "id": 25, "name": "passthroughInput", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageAlign", "properties": { "alignHeight": 0, "alignWidth": 0, "initialConfig": { "staticDepthPlane": 0 }, "interpolation": -1, "numFramesPool": 4, "numShaves": 2, "outKeepAspectRatio": true, "warpHwIds": [] } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 26, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "config" } } ] ] } } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。