# RGB-热成像对齐 本示例演示如何将热成像相机的热信息对齐到RGB相机。这种设置对于需要叠加或比较热成像和彩色数据的应用非常有用。创建一个OpenCV窗口来显示RGB和对齐后的热成像数据的混合图像。提供轨迹条以调整混合比例。 > **无效的外参** > 由于校准问题，您可能在早期版本的OAK热成像设备上遇到以下错误： > ```bash > [ImageAlign(4)] [error] Failed to get calibration data: Extrinsic connection between the requested cameraId's doesn't exist. Please recalibrate or modify your calibration data > ``` > 如果发生此情况，请下载 > [校准数据+脚本](https://drive.google.com/drive/folders/1Q_MZMqWMKDC1eOqVHGPeDO-NJgFmnY5U) > ，将其放入同一文件夹，连接相机到计算机并运行脚本。这将更新校准并添加相机传感器之间所需的外参。 ## 演示 ## 安装 请运行[安装脚本](https://github.com/luxonis/depthai-python/blob/main/examples/install_requirements.py)以下载所有必需的依赖项。请注意，此脚本必须在 git 上下文中运行，因此您需要先下载 [depthai-python](https://github.com/luxonis/depthai-python) 仓库，然后运行该脚本。 ```bash git clone https://github.com/luxonis/depthai-python.git cd depthai-python/examples python3 install_requirements.py ``` 更多信息，请参考[安装指南](https://docs.luxonis.com/software/depthai/manual-install.md)。 ## 源代码 #### Python ```python #!/usr/bin/env python3 """ Due to an issue with our calibration, you might receive the following error when running this script on early release OAK Thermal devices: ```bash [ImageAlign(4)] [error] Failed to get calibration data: Extrinsic connection between the requested cameraId's doesn't exist. Please recalibrate or modify your calibration data ``` If this happens, please download the calibration data + script from https://drive.google.com/drive/folders/1Q_MZMqWMKDC1eOqVHGPeDO-NJgFmnY5U, place them into the same folder, connect the camera to the computer and run the script. This will update the calibration and add required extrinsics between the camera sensors. """ import cv2 import depthai as dai import numpy as np import time from datetime import timedelta FPS = 25.0 RGB_SOCKET = dai.CameraBoardSocket.CAM_A COLOR_RESOLUTION = dai.ColorCameraProperties.SensorResolution.THE_1080_P class FPSCounter: def __init__(self): self.frameTimes = [] def tick(self): now = time.time() self.frameTimes.append(now) self.frameTimes = self.frameTimes[-100:] def getFps(self): if len(self.frameTimes) <= 1: return 0 # Calculate the FPS return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0]) device = dai.Device() thermalWidth, thermalHeight = -1, -1 thermalFound = False for features in device.getConnectedCameraFeatures(): if dai.CameraSensorType.THERMAL in features.supportedTypes: thermalFound = True thermalSocket = features.socket thermalWidth, thermalHeight = features.width, features.height break if not thermalFound: raise RuntimeError("No thermal camera found!") ISP_SCALE = 3 calibrationHandler = device.readCalibration() rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET) distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET) if distortionModel != dai.CameraModel.Perspective: raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.") pipeline = dai.Pipeline() # Define sources and outputs camRgb = pipeline.create(dai.node.ColorCamera) thermalCam = pipeline.create(dai.node.Camera) thermalCam.setBoardSocket(thermalSocket) thermalCam.setFps(FPS) sync = pipeline.create(dai.node.Sync) out = pipeline.create(dai.node.XLinkOut) align = pipeline.create(dai.node.ImageAlign) cfgIn = pipeline.create(dai.node.XLinkIn) camRgb.setBoardSocket(RGB_SOCKET) camRgb.setResolution(COLOR_RESOLUTION) camRgb.setFps(FPS) camRgb.setIspScale(1,ISP_SCALE) out.setStreamName("out") sync.setSyncThreshold(timedelta(seconds=0.5 / FPS)) cfgIn.setStreamName("config") cfg = align.initialConfig.get() staticDepthPlane = cfg.staticDepthPlane # Linking align.outputAligned.link(sync.inputs["aligned"]) camRgb.isp.link(sync.inputs["rgb"]) camRgb.isp.link(align.inputAlignTo) thermalCam.raw.link(align.input) sync.out.link(out.input) cfgIn.out.link(align.inputConfig) rgbWeight = 0.4 thermalWeight = 0.6 def updateBlendWeights(percentRgb): """ Update the rgb and depth weights used to blend depth/rgb image @param[in] percent_rgb The rgb weight expressed as a percentage (0..100) """ global thermalWeight global rgbWeight rgbWeight = float(percentRgb) / 100.0 thermalWeight = 1.0 - rgbWeight def updateDepthPlane(depth): global staticDepthPlane staticDepthPlane = depth # Connect to device and start pipeline with device: device.startPipeline(pipeline) queue = device.getOutputQueue("out", 8, False) cfgQ = device.getInputQueue("config") # Configure windows; trackbar adjusts blending ratio of rgb/depth windowName = "rgb-thermal" # Set the window to be resizable and the initial size cv2.namedWindow(windowName, cv2.WINDOW_NORMAL) cv2.resizeWindow(windowName, 1280, 720) cv2.createTrackbar( "RGB Weight %", windowName, int(rgbWeight * 100), 100, updateBlendWeights, ) cv2.createTrackbar( "Static Depth Plane [mm]", windowName, 0, 2000, updateDepthPlane, ) fpsCounter = FPSCounter() while True: messageGroup = queue.get() assert isinstance(messageGroup, dai.MessageGroup) frameRgb = messageGroup["rgb"] assert isinstance(frameRgb, dai.ImgFrame) thermalAligned = messageGroup["aligned"] assert isinstance(thermalAligned, dai.ImgFrame) frameRgbCv = frameRgb.getCvFrame() fpsCounter.tick() rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(frameRgbCv.shape[1]), int(frameRgbCv.shape[0])) cvFrameUndistorted = cv2.undistort( frameRgbCv, np.array(rgbIntrinsics), np.array(rgbDistortion), ) # Colorize the aligned depth thermalFrame = thermalAligned.getCvFrame().astype(np.float32) # Create a mask for nan values mask = np.isnan(thermalFrame) # Replace nan values with a mean for visualization thermalFrame[mask] = np.nanmean(thermalFrame) thermalFrame = cv2.normalize(thermalFrame, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U) colormappedFrame = cv2.applyColorMap(thermalFrame, cv2.COLORMAP_MAGMA) # Apply the mask back with black pixels (0) colormappedFrame[mask] = 0 blended = cv2.addWeighted(cvFrameUndistorted, rgbWeight, colormappedFrame, thermalWeight, 0) cv2.putText( blended, f"FPS: {fpsCounter.getFps():.2f}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, ) cv2.imshow(windowName, blended) key = cv2.waitKey(1) if key == ord("q"): break cfg.staticDepthPlane = staticDepthPlane cfgQ.send(cfg) ``` #### C++ ```cpp #include #include #include #include "depthai/depthai.hpp" #include "depthai/pipeline/datatype/ImageAlignConfig.hpp" constexpr auto FPS = 25.0; constexpr auto RGB_SOCKET = dai::CameraBoardSocket::CAM_A; constexpr auto COLOR_RESOLUTION = dai::ColorCameraProperties::SensorResolution::THE_1080_P; class FPSCounter { public: void tick() { auto now = std::chrono::steady_clock::now(); frameTimes.push(now); if(frameTimes.size() > 100) { frameTimes.pop(); } } double getFps() { if(frameTimes.size() <= 1) return 0; auto duration = std::chrono::duration_cast(frameTimes.back() - frameTimes.front()).count(); return (frameTimes.size() - 1) * 1000.0 / duration; } private: std::queue frameTimes; }; double rgbWeight = 0.4; double thermalWeight = 0.6; int staticDepthPlane = 0; void updateBlendWeights(int percentRgb, void*) { rgbWeight = static_cast(percentRgb) / 100.0; thermalWeight = 1.0 - rgbWeight; } void updateDepthPlane(int depth, void*) { staticDepthPlane = depth; } cv::Mat createNaNMask(const cv::Mat& frame) { cv::Mat nanMask = cv::Mat::zeros(frame.size(), CV_8UC1); for(int r = 0; r < frame.rows; ++r) { for(int c = 0; c < frame.cols; ++c) { if(std::isnan(frame.at(r, c))) { nanMask.at(r, c) = 255; } } } return nanMask; } int main() { dai::Device device; int thermalWidth = -1, thermalHeight = -1; bool thermalFound = false; dai::CameraBoardSocket thermalSocket; for(const auto& features : device.getConnectedCameraFeatures()) { if(std::find(features.supportedTypes.begin(), features.supportedTypes.end(), dai::CameraSensorType::THERMAL) != features.supportedTypes.end()) { thermalFound = true; thermalSocket = features.socket; thermalWidth = features.width; thermalHeight = features.height; break; } } if(!thermalFound) { throw std::runtime_error("No thermal camera found!"); } dai::Pipeline pipeline; // Define sources and outputs auto camRgb = pipeline.create(); auto thermalCam = pipeline.create(); auto sync = pipeline.create(); auto out = pipeline.create(); auto align = pipeline.create(); auto cfgIn = pipeline.create(); thermalCam->setBoardSocket(thermalSocket); thermalCam->setFps(FPS); camRgb->setBoardSocket(RGB_SOCKET); camRgb->setResolution(COLOR_RESOLUTION); camRgb->setFps(FPS); camRgb->setIspScale(1, 3); out->setStreamName("out"); sync->setSyncThreshold(std::chrono::milliseconds(static_cast((1 / FPS) * 1000.0 * 0.5))); cfgIn->setStreamName("config"); align->outputAligned.link(sync->inputs["aligned"]); camRgb->isp.link(sync->inputs["rgb"]); camRgb->isp.link(align->inputAlignTo); thermalCam->raw.link(align->input); sync->out.link(out->input); cfgIn->out.link(align->inputConfig); device.startPipeline(pipeline); auto queue = device.getOutputQueue("out", 8, false); auto cfgQ = device.getInputQueue("config"); FPSCounter fpsCounter; std::string windowName = "rgb-thermal"; cv::namedWindow(windowName, cv::WINDOW_NORMAL); cv::resizeWindow(windowName, 1280, 720); cv::createTrackbar("RGB Weight %", windowName, nullptr, 100, updateBlendWeights); cv::createTrackbar("Static Depth Plane [mm]", windowName, nullptr, 2000, updateDepthPlane); while(true) { auto messageGroup = queue->get(); fpsCounter.tick(); auto frameRgb = messageGroup->get("rgb"); auto thermalAligned = messageGroup->get("aligned"); if(frameRgb && thermalAligned) { auto frameRgbCv = frameRgb->getCvFrame(); auto thermalFrame = thermalAligned->getFrame(true); // Colorize the aligned depth cv::Mat mask; cv::Mat colormappedFrame; cv::Mat thermalFrameFloat; thermalFrame.convertTo(thermalFrameFloat, CV_32F); // Get a mask for the nan values mask = createNaNMask(thermalFrameFloat); auto meanValue = cv::mean(thermalFrameFloat, ~mask); thermalFrameFloat.setTo(meanValue, mask); cv::normalize(thermalFrameFloat, thermalFrameFloat, 0, 255, cv::NORM_MINMAX, CV_8U); cv::applyColorMap(thermalFrameFloat, colormappedFrame, cv::COLORMAP_MAGMA); colormappedFrame.setTo(cv::Scalar(0, 0, 0), mask); cv::Mat blended; cv::addWeighted(frameRgbCv, rgbWeight, colormappedFrame, thermalWeight, 0, blended); cv::putText(blended, "FPS: " + std::to_string(fpsCounter.getFps()), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2); cv::imshow(windowName, blended); } int key = cv::waitKey(1); if(key == 'q' || key == 'Q') { break; } auto cfg = align->initialConfig.get(); cfg.staticDepthPlane = staticDepthPlane; auto alignConfig = std::make_shared(); alignConfig->set(cfg); cfgQ->send(alignConfig); } return 0; } ``` ## 流水线 ### examples/thermal_align.pipeline.json ```json { "pipeline": { "connections": [ { "node1Id": 4, "node1Output": "outputAligned", "node1OutputGroup": "", "node2Id": 2, "node2Input": "aligned", "node2InputGroup": "inputs" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 2, "node2Input": "rgb", "node2InputGroup": "inputs" }, { "node1Id": 5, "node1Output": "out", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputConfig", "node2InputGroup": "" }, { "node1Id": 0, "node1Output": "isp", "node1OutputGroup": "", "node2Id": 4, "node2Input": "inputAlignTo", "node2InputGroup": "" }, { "node1Id": 1, "node1Output": "raw", "node1OutputGroup": "", "node2Id": 4, "node2Input": "input", "node2InputGroup": "" }, { "node1Id": 2, "node1Output": "out", "node1OutputGroup": "", "node2Id": 3, "node2Input": "in", "node2InputGroup": "" } ], "globalProperties": { "calibData": null, "cameraTuningBlobSize": null, "cameraTuningBlobUri": "", "leonCssFrequencyHz": 700000000.0, "leonMssFrequencyHz": 700000000.0, "pipelineName": null, "pipelineVersion": null, "sippBufferSize": 18432, "sippDmaBufferSize": 16384, "xlinkChunkSize": -1 }, "nodes": [ [ 0, { "id": 0, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 1, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 6, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 7, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 2, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 3, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 4, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 5, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 8, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ColorCamera", "properties": { "boardSocket": 0, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 3, "horizNumerator": 1, "vertDenominator": 3, "vertNumerator": 1 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolution": 0, "sensorCropX": -1.0, "sensorCropY": -1.0, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1 } } ], [ 1, { "id": 1, "ioInfo": [ [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 9, "name": "inputConfig", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "raw" ], { "blocking": false, "group": "", "id": 14, "name": "raw", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "still" ], { "blocking": false, "group": "", "id": 15, "name": "still", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "inputControl" ], { "blocking": true, "group": "", "id": 10, "name": "inputControl", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "video" ], { "blocking": false, "group": "", "id": 11, "name": "video", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "isp" ], { "blocking": false, "group": "", "id": 12, "name": "isp", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "preview" ], { "blocking": false, "group": "", "id": 13, "name": "preview", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "frameEvent" ], { "blocking": false, "group": "", "id": 16, "name": "frameEvent", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Camera", "properties": { "boardSocket": -1, "calibAlpha": null, "cameraName": "", "colorOrder": 0, "fp16": false, "fps": 25.0, "imageOrientation": -1, "initialControl": { "aeLockMode": false, "aeMaxExposureTimeUs": 0, "aeRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "afRegion": { "height": 0, "priority": 0, "width": 0, "x": 0, "y": 0 }, "antiBandingMode": 0, "autoFocusMode": 3, "awbLockMode": false, "awbMode": 0, "brightness": 0, "captureIntent": 0, "chromaDenoise": 0, "cmdMask": 0, "contrast": 0, "controlMode": 0, "effectMode": 0, "expCompensation": 0, "expManual": { "exposureTimeUs": 0, "frameDurationUs": 0, "sensitivityIso": 0 }, "frameSyncMode": 0, "lensPosAutoInfinity": 0, "lensPosAutoMacro": 0, "lensPosition": 0, "lensPositionRaw": 0.0, "lowPowerNumFramesBurst": 0, "lowPowerNumFramesDiscard": 0, "lumaDenoise": 0, "saturation": 0, "sceneMode": 0, "sharpness": 0, "strobeConfig": { "activeLevel": 0, "enable": 0, "gpioNumber": 0 }, "strobeTimings": { "durationUs": 0, "exposureBeginOffsetUs": 0, "exposureEndOffsetUs": 0 }, "wbColorTemp": 0 }, "interleaved": true, "isp3aFps": 0, "ispScale": { "horizDenominator": 0, "horizNumerator": 0, "vertDenominator": 0, "vertNumerator": 0 }, "numFramesPoolIsp": 3, "numFramesPoolPreview": 4, "numFramesPoolRaw": 3, "numFramesPoolStill": 4, "numFramesPoolVideo": 4, "previewHeight": 300, "previewKeepAspectRatio": true, "previewWidth": 300, "rawPacked": null, "resolutionHeight": -1, "resolutionWidth": -1, "sensorCropX": -1.0, "sensorCropY": -1.0, "sensorType": -1, "stillHeight": -1, "stillWidth": -1, "videoHeight": -1, "videoWidth": -1, "warpMeshHeight": 0, "warpMeshSource": -1, "warpMeshStepHeight": 32, "warpMeshStepWidth": 32, "warpMeshUri": "", "warpMeshWidth": 0 } } ], [ 2, { "id": 2, "ioInfo": [ [ [ "inputs", "aligned" ], { "blocking": true, "group": "inputs", "id": 17, "name": "aligned", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "inputs", "rgb" ], { "blocking": true, "group": "inputs", "id": 18, "name": "rgb", "queueSize": 8, "type": 3, "waitForMessage": false } ], [ [ "", "out" ], { "blocking": false, "group": "", "id": 19, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "Sync", "properties": { "syncAttempts": -1, "syncThresholdNs": 20000000 } } ], [ 3, { "id": 3, "ioInfo": [ [ [ "", "in" ], { "blocking": true, "group": "", "id": 20, "name": "in", "queueSize": 8, "type": 3, "waitForMessage": true } ] ], "name": "XLinkOut", "properties": { "maxFpsLimit": -1.0, "metadataOnly": false, "streamName": "out" } } ], [ 4, { "id": 4, "ioInfo": [ [ [ "", "inputAlignTo" ], { "blocking": false, "group": "", "id": 21, "name": "inputAlignTo", "queueSize": 1, "type": 3, "waitForMessage": true } ], [ [ "", "inputConfig" ], { "blocking": false, "group": "", "id": 22, "name": "inputConfig", "queueSize": 4, "type": 3, "waitForMessage": false } ], [ [ "", "input" ], { "blocking": false, "group": "", "id": 23, "name": "input", "queueSize": 4, "type": 3, "waitForMessage": true } ], [ [ "", "outputAligned" ], { "blocking": false, "group": "", "id": 24, "name": "outputAligned", "queueSize": 8, "type": 0, "waitForMessage": false } ], [ [ "", "passthroughInput" ], { "blocking": false, "group": "", "id": 25, "name": "passthroughInput", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "ImageAlign", "properties": { "alignHeight": 0, "alignWidth": 0, "initialConfig": { "staticDepthPlane": 0 }, "interpolation": -1, "numFramesPool": 4, "numShaves": 2, "outKeepAspectRatio": true, "warpHwIds": [] } } ], [ 5, { "id": 5, "ioInfo": [ [ [ "", "out" ], { "blocking": false, "group": "", "id": 26, "name": "out", "queueSize": 8, "type": 0, "waitForMessage": false } ] ], "name": "XLinkIn", "properties": { "maxDataSize": 5242880, "numFrames": 8, "streamName": "config" } } ] ] } } ``` ### 需要帮助？ 请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。