# 相机高帧率 (HFR)

DepthAI 3.4.0 中的高帧率 (HFR) 模式在搭载 RVC4 平台和 IMX586 传感器的 OAK4 设备上实现了超快感知。这些流水线可以捕捉和处理最高 480 FPS 的视频流，并对每一帧进行神经网络推理。

> **Preview feature**
> HFR 模式是 DepthAI
> `3.4.0`
> 中的早期预览功能。当前模式固定为
> `1920x1080 @ 240 FPS`
> 和
> `1280x720 @ 480 FPS`
> 。

## HFR 为何重要

 * 捕捉快速运动，减少模糊。
 * 以全吞吐能力对每一帧进行神经推理。
 * 降低闭环系统的端到端感知延迟。

## 支持的 HFR 模式

| 分辨率 | 帧率 |
| --- | --- |
| 1920 x 1080 | 240 FPS |
| 1280 x 720 | 480 FPS |

尚不支持任意 FPS 和自定义 HFR 分辨率。如果您的模型需要不同的输入形状，请使用
[ImageManip](https://docs.luxonis.com/software-v3/depthai/depthai-components/nodes/image_manip.md) 进行设备端适配。

## 示例应用

 * 快速零件运动的工业自动化。
 * 需要更快控制回路的机器人工作负载。
 * 体育分析和高速运动追踪。
 * 高速视觉检测系统。

## 包含的示例流水线

### 目标检测

以高达 480 FPS 的 HFR 输入运行 YOLOv6。

[目标检测](https://github.com/luxonis/depthai-core/blob/v3.4.0/examples/python/HFR/hfr_nn.py)

### 小型实时预览

以 240/480 FPS 显示轻量级预览流。

[小型实时预览](https://github.com/luxonis/depthai-core/blob/v3.4.0/examples/python/HFR/hfr_small_preview.py)

### 视频编码

编码并保存高帧率视频流。

[视频编码](https://github.com/luxonis/depthai-core/blob/v3.4.0/examples/python/HFR/hfr_save_encoded.py)

这个示例需要DepthAI v3 API，参见[安装说明](https://docs.luxonis.com/software-v3/depthai.md)。

## 源码

### 高帧率目标检测 (YOLOv6)

#### Python

```python
#!/usr/bin/env python3
import depthai as dai
import sys

FPS = 480

with dai.Pipeline() as pipeline:
    device = pipeline.getDefaultDevice()
    platform = device.getPlatform()
    if platform != dai.Platform.RVC4:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    # Exit cleanly if the selected HFR mode is not advertised by CAM_A.
    supportsRequestedFps = False
    for cameraFeature in device.getConnectedCameraFeatures():
        if cameraFeature.socket != dai.CameraBoardSocket.CAM_A:
            continue
        for config in cameraFeature.configs:
            if config.width == 1280 and config.height == 720 and config.maxFps >= FPS:
                supportsRequestedFps = True
                break
        break
    if not supportsRequestedFps:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    # Download the model
    nnArchivePath = dai.getModelFromZoo(dai.NNModelDescription("yolov6-nano", platform="RVC4"))
    nnArchive = dai.NNArchive(nnArchivePath)
    inputSize = nnArchive.getInputSize()
    cameraNode = pipeline.create(dai.node.Camera).build()

    # Configure the ImageManip as in HFR mode requesting arbitrary outputs is not yet supported
    cameraOutput = cameraNode.requestOutput((1280, 720), fps=FPS)
    imageManip = pipeline.create(dai.node.ImageManip)
    imageManip.initialConfig.setOutputSize(inputSize[0], inputSize[1])
    imageManip.setMaxOutputFrameSize(int(inputSize[0] * inputSize[1] * 3))
    imageManip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888i)
    imageManip.inputImage.setMaxSize(12)
    cameraOutput.link(imageManip.inputImage)

    # Configure the DetectionNetwork
    detectionNetwork = pipeline.create(dai.node.DetectionNetwork)
    detectionNetwork.setNNArchive(nnArchive)
    imageManip.out.link(detectionNetwork.input)

    benchmarkIn = pipeline.create(dai.node.BenchmarkIn)
    benchmarkIn.setRunOnHost(True)
    benchmarkIn.sendReportEveryNMessages(FPS)
    detectionNetwork.out.link(benchmarkIn.input)

    qDet = detectionNetwork.out.createOutputQueue()
    pipeline.start()

    while pipeline.isRunning():
        inDet: dai.ImgDetections = qDet.get()
        # print(f"Got {len(inDet.detections)} nn detections ")
```

### 小型实时预览

#### Python

```python
#!/usr/bin/env python3
import depthai as dai
import sys
import time
import cv2

SIZE = (1280, 720)
FPS = 480

# SIZE = (1920, 1080)
# FPS = 240

with dai.Pipeline() as pipeline:
    device = pipeline.getDefaultDevice()
    platform = device.getPlatform()
    if platform != dai.Platform.RVC4:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    # Exit cleanly if the selected HFR mode is not advertised by CAM_A.
    supportsRequestedFps = False
    for cameraFeature in device.getConnectedCameraFeatures():
        if cameraFeature.socket != dai.CameraBoardSocket.CAM_A:
            continue
        for config in cameraFeature.configs:
            if config.width == SIZE[0] and config.height == SIZE[1] and config.maxFps >= FPS:
                supportsRequestedFps = True
                break
        break
    if not supportsRequestedFps:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    cam = pipeline.create(dai.node.Camera).build()
    benchmarkIn = pipeline.create(dai.node.BenchmarkIn)
    benchmarkIn.setRunOnHost(True)
    benchmarkIn.sendReportEveryNMessages(FPS)

    imageManip = pipeline.create(dai.node.ImageManip)
    imageManip.initialConfig.setOutputSize(250, 250)
    imageManip.setMaxOutputFrameSize(int(250* 250 * 1.6))

    # One of the two modes can be selected
    # NOTE: Generic resolutions are not yet supported through camera node when using HFR mode
    output = cam.requestOutput(SIZE, fps=FPS)

    output.link(imageManip.inputImage)
    imageManip.out.link(benchmarkIn.input)

    outputQueue = imageManip.out.createOutputQueue()

    pipeline.start()
    while pipeline.isRunning():
        imgFrame = outputQueue.get()
        assert isinstance(imgFrame, dai.ImgFrame)
        cv2.imshow("frame", imgFrame.getCvFrame())
        cv2.waitKey(1)
```

### 保存编码流

#### Python

```python
import depthai as dai

# Capture Ctrl+C and set a flag to stop the loop
import time
import cv2
import threading
import signal
import sys

PROFILE = dai.VideoEncoderProperties.Profile.H264_MAIN

quitEvent = threading.Event()
signal.signal(signal.SIGTERM, lambda *_args: quitEvent.set())
signal.signal(signal.SIGINT, lambda *_args: quitEvent.set())

SIZE = (1280, 720)
FPS = 480

# SIZE = (1920, 1080)
# FPS = 240

class VideoSaver(dai.node.HostNode):
    def __init__(self, *args, **kwargs):
        dai.node.HostNode.__init__(self, *args, **kwargs)
        self.file_handle = open('video_hfr.encoded', 'wb')

    def build(self, *args):
        self.link_args(*args)
        return self

    def process(self, frame):
        frame.getData().tofile(self.file_handle)

with dai.Pipeline() as pipeline:
    device = pipeline.getDefaultDevice()
    platform = device.getPlatform()
    if platform != dai.Platform.RVC4:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    # Exit cleanly if the selected HFR mode is not advertised by CAM_A.
    supportsRequestedFps = False
    for cameraFeature in device.getConnectedCameraFeatures():
        if cameraFeature.socket != dai.CameraBoardSocket.CAM_A:
            continue
        for config in cameraFeature.configs:
            if config.width == SIZE[0] and config.height == SIZE[1] and config.maxFps >= FPS:
                supportsRequestedFps = True
                break
        break
    if not supportsRequestedFps:
        print("This example is only supported on IMX586 and Luxonis OS 1.20.5 or higher", file=sys.stderr)
        sys.exit(0)

    camRgb = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)
    output = camRgb.requestOutput(SIZE, fps=FPS)

    # ImageManip is added to workaround a limitation with VideoEncoder with native resolutions
    # This limitation will be lifted in the future
    imageManip = pipeline.create(dai.node.ImageManip)
    imageManip.initialConfig.setOutputSize(SIZE[0], SIZE[1] + 10) # To avoid a passthrough
    imageManip.setMaxOutputFrameSize(int(SIZE[0] * (SIZE[1] + 10) * 1.6))
    imageManip.inputImage.setMaxSize(12)
    output.link(imageManip.inputImage)
    output = imageManip.out

    benchmarkIn = pipeline.create(dai.node.BenchmarkIn)
    benchmarkIn.setRunOnHost(True)

    encoded = pipeline.create(dai.node.VideoEncoder).build(output,
            frameRate = FPS,
            profile = PROFILE)
    encoded.out.link(benchmarkIn.input)
    saver = pipeline.create(VideoSaver).build(encoded.out)

    pipeline.start()
    print("Started to save video to video.encoded")
    print("Press Ctrl+C to stop")
    timeStart = time.monotonic()
    while pipeline.isRunning() and not quitEvent.is_set():
        time.sleep(1)
    pipeline.stop()
    pipeline.wait()
    saver.file_handle.close()

print("To view the encoded data, convert the stream file (.encoded) into a video file (.mp4) using a command below:")
print(f"ffmpeg -framerate {FPS} -i video_hfr.encoded -c copy video_hfr.mp4")

print("If the FPS is not set correctly, you can ask ffmpeg to generate it with the command below")

print(f"""
ffmpeg -fflags +genpts -r {FPS} -i video_hfr.encoded \\
  -vsync cfr -fps_mode cfr \\
  -video_track_timescale {FPS}00 \\
  -c:v copy \\
  video_hfr.mp4
""")
```

### 需要帮助？

请前往 [OAKChina 官网](https://www.oakchina.cn/) 获取技术支持或解答您的任何疑问。