Zero-Copy Camera Pipeline — V4L2·DMA-BUF·GPU Import·NPU 직결

1
Producer 측 (예 V4L2 camera driver)
2
  ↓ VIDIOC_EXPBUF
3
  fd (file descriptor) 발급
4
  ↓
5
Consumer 측 (예 EGL / CUDA / VAAPI)
6
  ↓ eglCreateImageKHR / cudaImportExternalMemory
7
  same physical page를 자기 driver의 handle로 mapping

1
int cam = open("/dev/video0", O_RDWR);
2

3
struct v4l2_format fmt = {
4
    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
5
    .fmt.pix_mp = {
6
        .width = 1920, .height = 1080,
7
        .pixelformat = V4L2_PIX_FMT_NV12,
8
        .num_planes = 2,
9
    },
10
};
11
ioctl(cam, VIDIOC_S_FMT, &fmt);
12

13
struct v4l2_requestbuffers req = {
14
    .count  = 4,
15
    .type   = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
16
    .memory = V4L2_MEMORY_DMABUF,
17
};
18
ioctl(cam, VIDIOC_REQBUFS, &req);
19

20
int dma_fds[4];
21
for (int i = 0; i < 4; i++) {
22
    struct v4l2_exportbuffer exp = {
23
        .type  = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
24
        .index = i,
25
    };
26
    ioctl(cam, VIDIOC_EXPBUF, &exp);
27
    dma_fds[i] = exp.fd;
28
}

1
EGLint attrs[] = {
2
    EGL_WIDTH,                     1920,
3
    EGL_HEIGHT,                    1080,
4
    EGL_LINUX_DRM_FOURCC_EXT,      DRM_FORMAT_NV12,
5
    EGL_DMA_BUF_PLANE0_FD_EXT,     dma_fd,
6
    EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
7
    EGL_DMA_BUF_PLANE0_PITCH_EXT,  1920,
8
    EGL_DMA_BUF_PLANE1_FD_EXT,     dma_fd,
9
    EGL_DMA_BUF_PLANE1_OFFSET_EXT, 1920 * 1080,
10
    EGL_DMA_BUF_PLANE1_PITCH_EXT,  1920,
11
    EGL_NONE,
12
};
13
EGLImageKHR image = eglCreateImageKHR(
14
    egl_display, EGL_NO_CONTEXT,
15
    EGL_LINUX_DMA_BUF_EXT, NULL, attrs);
16

17
GLuint tex;
18
glGenTextures(1, &tex);
19
glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex);
20
glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, image);

1
cudaExternalMemoryHandleDesc desc = {
2
    .type = cudaExternalMemoryHandleTypeOpaqueFd,
3
    .handle.fd = dma_fd,
4
    .size = 1920 * 1080 * 3 / 2,
5
};
6
cudaExternalMemory_t ext_mem;
7
cudaImportExternalMemory(&ext_mem, &desc);
8

9
cudaExternalMemoryBufferDesc buf_desc = {
10
    .offset = 0,
11
    .size   = 1920 * 1080 * 3 / 2,
12
};
13
void *device_ptr;
14
cudaExternalMemoryGetMappedBuffer(&device_ptr, ext_mem, &buf_desc);
15

16
/* device_ptr를 TensorRT setTensorAddress에 그대로 줄 수 있음 */
17
ctx->setTensorAddress("input", device_ptr);
18
ctx->enqueueV3(stream);

1
for (int i = 0; i < 4; i++) {
2
    struct v4l2_buffer buf = {
3
        .type   = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
4
        .memory = V4L2_MEMORY_DMABUF,
5
        .index  = i,
6
        .m.fd   = dma_fds[i],
7
    };
8
    ioctl(cam, VIDIOC_QBUF, &buf);
9
}
10

11
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
12
ioctl(cam, VIDIOC_STREAMON, &type);
13

14
while (running) {
15
    struct v4l2_buffer buf = {
16
        .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
17
        .memory = V4L2_MEMORY_DMABUF,
18
    };
19
    ioctl(cam, VIDIOC_DQBUF, &buf);
20
    int idx = buf.index;
21

22
    inference_on_dma_fd(dma_fds[idx]);
23
    display_on_dma_fd(dma_fds[idx]);
24

25
    ioctl(cam, VIDIOC_QBUF, &buf);
26
}

1
gst-launch-1.0 \
2
  nvarguscamerasrc sensor-id=0 ! \
3
  'video/x-raw(memory:NVMM),width=1920,height=1080,format=NV12,framerate=60/1' ! \
4
  nvvidconv ! \
5
  nvinfer config-file-path=yolo.txt ! \
6
  nvtracker ll-config-file=tracker.yml ! \
7
  nvdsosd ! \
8
  nvegltransform ! nveglglessink

1
#include <libcamera/libcamera.h>
2

3
camera->configure(config.get());
4

5
for (auto &fb : framebuffers) {
6
    auto req = camera->createRequest();
7
    req->addBuffer(stream, fb.get());
8
    camera->queueRequest(req.get());
9
}
10

11
/* requestCompleted signal */
12
camera->requestCompleted.connect([](Request *r) {
13
    auto &bufs = r->buffers();
14
    for (auto &[s, fb] : bufs) {
15
        int fd = fb->planes()[0].fd.get();
16
        process_dma_fd(fd);
17
    }
18
    r->reuse(Request::ReuseBuffers);
19
    camera->queueRequest(r);
20
});

1
struct drm_prime_handle prime = { .fd = dma_fd };
2
ioctl(drm_fd, DRM_IOCTL_PRIME_FD_TO_HANDLE, &prime);
3

4
uint32_t handles[4] = { prime.handle };
5
uint32_t pitches[4] = { 1920 };
6
uint32_t offsets[4] = { 0 };
7
uint32_t fb_id;
8
drmModeAddFB2(drm_fd, 1920, 1080, DRM_FORMAT_NV12,
9
              handles, pitches, offsets, &fb_id, 0);
10
drmModeSetCrtc(drm_fd, crtc_id, fb_id, 0, 0, &conn_id, 1, &mode);

1
#version 300 es
2
#extension GL_OES_EGL_image_external_essl3 : require
3
precision highp float;
4

5
uniform samplerExternalOES tex;   /* YUV NV12 직접 sample */
6
in vec2 v_tex;
7
out vec4 color;
8

9
void main() {
10
    color = texture(tex, v_tex);   /* driver가 자동 YUV→RGB */
11
}

1
Pipeline                                 fps   CPU 사용률   Memory BW
2
v4l2src ! videoconvert ! appsink          25    180%         3.8 GB/s
3
v4l2src ! nvvidconv ! appsink             45     90%         1.7 GB/s
4
nvarguscamerasrc ! nvvidconv ! nvinfer    60     20%         0.6 GB/s
5
                  (NVMM zero-copy)

1
req.memory = V4L2_MEMORY_MMAP;
2
/* user는 mmap된 buffer를 보고 zero-copy라 생각 */
3
/* 하지만 GPU·NPU에 넘기려면 copy 발생 */

1
ioctl(VIDIOC_EXPBUF);   /* fd 4개 */
2
/* close(fd) 빠뜨림 → buffer leak */

1
Camera 4 KB page · GPU MMU 64 KB page
2
→ alignment fail → import error

1
EGL import NV12, GL shader는 RGB texture로 sample
2
→ 화면 검정 또는 색 뒤틀림

1
USB cam → URB → system memory copy → 어떤 trick도 zero-copy 안 됨

1
yuv420_to_rgb_scalar(src, dst);   /* CPU 50% */