feat: double buffered video input

This commit is contained in:
2025-11-24 18:50:50 +01:00
parent aa42e9d2aa
commit c66a5c166e
16 changed files with 162 additions and 82 deletions
+2 -1
View File
@@ -126,6 +126,7 @@ make -f Makefile.dev release-arch
- [x] Monitor improvements
- [ ] Ignore some values in auto random
- [x] build without video in
- [ ] Auto discover video devices
- [ ] Update README monitor/keymap
- [ ] Auto reconnect midi input
- [ ] Auto reconnect video device
@@ -136,4 +137,4 @@ make -f Makefile.dev release-arch
- [ ] Play from record text file
- [ ] Fixes
- [ ] Try to write NanoKontrol config
- [ ] Investigate video device fps loss (bad unregister ?)
- [x] Investigate video device fps loss (bad unregister ?)
-2
View File
@@ -479,8 +479,6 @@ Unfortunately, V4L2 is very slow compared to driver-specific decoding.
You can check your device real FPS on [V4L2 UCP](https://github.com/HedgeHawk/v4l2ucp) or [GTK UVC Viewer](https://github.com/jaswdr/guvcview).
Sadly, OpenEGL doesn't support double-buffered DMA buffers so FPS in FORGE may be halved.
### My video feed got strange lines
You need to decode the [V4L2 YUYV format](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/pixfmt-yuyv.html).
+13 -9
View File
@@ -51,6 +51,8 @@ UNIFORM_STATE_PREFIX=iState
UNIFORM_ACTIVE_PREFIX=iActive
# Input X format raw integer value
UNIFORM_IN_FORMAT_PREFIX=iInputFormat
# Input X should use swap texture or not (0/1)
UNIFORM_IN_SWAP_PREFIX=iInputSwap
# --- uniform vec2 ---
@@ -93,7 +95,7 @@ SUB_2_PREFIX=fx_
# Total number of internal textures
TEX_COUNT=10
TEX_COUNT=12
# === VIDEO DEVICES
# Video devices will be read from CLI arguments
@@ -104,10 +106,12 @@ IN_COUNT=2
# To which texture will be bound video device X
IN_1_OUT=1
IN_2_OUT=2
# To which texture will be bound swap buffers (for double-buffering)
IN_1_SWAP_OUT=3
IN_2_SWAP_OUT=4
# === FRAGMENT SHADERS
# Fragment shaders will be read from the CLI directory as "fragX.glsl"
# Special shader "frag0.glsl" will be prepend to each one
# Prefix of fragment shaders to detect
FRAG_FILE_PREFIX=frag
@@ -116,13 +120,13 @@ FRAG_FILE_PREFIX=frag
FRAG_COUNT=10
# To which texture will the shader fragX.glsl will render to
FRAG_1_OUT=3
FRAG_2_OUT=4
FRAG_3_OUT=5
FRAG_4_OUT=6
FRAG_5_OUT=7
FRAG_6_OUT=8
FRAG_7_OUT=9
FRAG_1_OUT=5
FRAG_2_OUT=6
FRAG_3_OUT=7
FRAG_4_OUT=8
FRAG_5_OUT=9
FRAG_6_OUT=10
FRAG_7_OUT=11
FRAG_8_OUT=0
# Which fragment shader renders to output window
FRAG_OUTPUT=9
+8 -4
View File
@@ -2,8 +2,8 @@
// VIDEO 1
// -----------
// IN: 1 (RAW IN A)
// OUT: 3 (IN A)
// IN: 1+3 (RAW IN A)
// OUT: 5 (IN A)
in vec2 vUV;
out vec4 fragColor;
@@ -12,14 +12,18 @@ out vec4 fragColor;
uniform sampler2D iTex0;
uniform sampler2D iTex1;
uniform sampler2D iTex3;
uniform int iInputFormat1;
uniform int iInputSwap1;
uniform vec2 iInputResolution1;
void main() {
if (iInputFormat1 == YUYV_FOURCC) {
if (iInputSwap1 > 0) {
fragColor = yuyvTex(iTex3, vUV, int(iInputResolution1.x));
} else {
fragColor = yuyvTex(iTex1, vUV, int(iInputResolution1.x));
} else if (iInputResolution1.x > 0) {
fragColor = texture(iTex1, vUV);
}
} else {
fragColor = texture(iTex0, vUV);
}
+9 -11
View File
@@ -9,15 +9,13 @@ out vec4 fragColor;
#include inc_debug.glsl
uniform sampler2D iTex0;
uniform sampler2D iTex1;
uniform sampler2D iTex2;
uniform sampler2D iTex3;
uniform sampler2D iTex4;
uniform sampler2D iTex5;
uniform sampler2D iTex6;
uniform sampler2D iTex7;
uniform sampler2D iTex8;
uniform sampler2D iTex9;
uniform sampler2D iTex10;
uniform sampler2D iTex11;
uniform int iFPS;
uniform int iInputFPS1;
uniform int iInputFPS2;
@@ -52,14 +50,14 @@ void main() {
vec4 c = vec4(0);
c += s(uv2,1,2) * texture(iTex5, uv2);
c += s(uv2,2,2) * texture(iTex7, uv2);
c += s(uv2,1,2) * texture(iTex7, uv2);
c += s(uv2,2,2) * texture(iTex9, uv2);
c += s(uv2,1,0) * texture(iTex6, uv2);
c += s(uv2,2,0) * texture(iTex8, uv2);
c += s(uv2,1,0) * texture(iTex8, uv2);
c += s(uv2,2,0) * texture(iTex10, uv2);
c += s(uv2,0,1) * debug(mod(uv2, 1));
c += s(uv2,1,1) * texture(iTex9, uv2);
c += s(uv2,1,1) * texture(iTex11, uv2);
c += s(uv2,2,1) * texture(iTex0, uv2);
float sel = 0;
@@ -79,7 +77,7 @@ void main() {
t += write_5(uv3, vec2(-37,28), texts[0]);
if (iInputResolution1.x > 0) {
c += s(uv2,0,2) * texture(iTex3, uv2);
c += s(uv2,0,2) * texture(iTex5, uv2);
f += rect(uv3, vec2(-35, 26.75), vec2(2.8, 0.7));
t += write_int(uv3, vec2(-37.6,26.1), iInputFPS1, 2);
t += write_5(uv3, vec2(-35.1,26.1), texts[8]);
@@ -92,7 +90,7 @@ void main() {
t += write_5(uv3, vec2(-37,-12), texts[1]);
if (iInputResolution2.x > 0) {
c += s(uv2,0,0) * texture(iTex4, uv2);
c += s(uv2,0,0) * texture(iTex6, uv2);
f += rect(uv3, vec2(-35, -13.25), vec2(2.8, 0.7));
t += write_int(uv3, vec2(-37.6,-13.9), iInputFPS2, 2);
t += write_5(uv3, vec2(-35.1,-13.9), texts[8]);
+8 -4
View File
@@ -2,8 +2,8 @@
// VIDEO 2
// -----------
// IN: 2 (RAW IN B)
// OUT: 4 (IN B)
// IN: 2+4 (RAW IN B)
// OUT: 6 (IN B)
in vec2 vUV;
out vec4 fragColor;
@@ -12,14 +12,18 @@ out vec4 fragColor;
uniform sampler2D iTex0;
uniform sampler2D iTex2;
uniform sampler2D iTex4;
uniform int iInputFormat2;
uniform int iInputSwap2;
uniform vec2 iInputResolution2;
void main() {
if (iInputFormat2 == YUYV_FOURCC) {
if (iInputSwap2 > 0) {
fragColor = yuyvTex(iTex4, vUV, int(iInputResolution2.x));
} else {
fragColor = yuyvTex(iTex2, vUV, int(iInputResolution2.x));
} else if (iInputResolution2.x > 0) {
fragColor = texture(iTex2, vUV);
}
} else {
fragColor = texture(iTex0, vUV);
}
+1 -1
View File
@@ -3,7 +3,7 @@
// SRC A
// -----------
// OUT: 5 (FX A)
// OUT: 7 (FX A)
in vec2 vUV;
out vec4 fragColor;
+1 -1
View File
@@ -2,7 +2,7 @@
// SRC B
// -----------
// OUT: 6 (FX B)
// OUT: 8 (FX B)
in vec2 vUV;
out vec4 fragColor;
+5 -5
View File
@@ -2,20 +2,20 @@
// FX A
// -------------
// IN: 5 (SRC A)
// IN: 7 (FX A)
// OUT: 7 (A+B)
// IN: 7 (SRC A)
// IN: 9 (FX A)
// OUT: 9 (A+B)
in vec2 vUV;
out vec4 fragColor;
#include inc_fx.glsl
uniform sampler2D iTex5;
uniform sampler2D iTex7;
uniform sampler2D iTex9;
uniform int iSeed5;
uniform vec3 iMidi2_1[7];
void main() {
fragColor = fx_stage(vUV, iTex5, iTex7, iSeed5, iMidi2_1[0], iMidi2_1[1].xy, iMidi2_1[2], iMidi2_1[3].xy, iMidi2_1[4], iMidi2_1[5].xy, iMidi2_1[6]);
fragColor = fx_stage(vUV, iTex7, iTex9, iSeed5, iMidi2_1[0], iMidi2_1[1].xy, iMidi2_1[2], iMidi2_1[3].xy, iMidi2_1[4], iMidi2_1[5].xy, iMidi2_1[6]);
}
+5 -5
View File
@@ -2,20 +2,20 @@
// FX B
// -------------
// IN: 6 (SRC B)
// IN: 8 (FX B)
// OUT: 8 (A+B)
// IN: 8 (SRC B)
// IN: 10 (FX B)
// OUT: 10 (A+B)
in vec2 vUV;
out vec4 fragColor;
#include inc_fx.glsl
uniform sampler2D iTex6;
uniform sampler2D iTex8;
uniform sampler2D iTex10;
uniform int iSeed6;
uniform vec3 iMidi2_2[7];
void main() {
fragColor = fx_stage(vUV, iTex6, iTex8, iSeed6, iMidi2_2[0], iMidi2_2[1].xy, iMidi2_2[2], iMidi2_2[3].xy, iMidi2_2[4], iMidi2_2[5].xy, iMidi2_2[6]);
fragColor = fx_stage(vUV, iTex8, iTex10, iSeed6, iMidi2_2[0], iMidi2_2[1].xy, iMidi2_2[2], iMidi2_2[3].xy, iMidi2_2[4], iMidi2_2[5].xy, iMidi2_2[6]);
}
+7 -7
View File
@@ -2,9 +2,9 @@
// A+B
// ------------
// IN: 7 (FX A)
// IN: 8 (FX B)
// OUT: 9 (MFX)
// IN: 9 (FX A)
// IN: 10 (FX B)
// OUT: 11 (MFX)
in vec2 vUV;
out vec4 fragColor;
@@ -13,8 +13,8 @@ out vec4 fragColor;
#include inc_functions.glsl
uniform int iDemo;
uniform sampler2D iTex7;
uniform sampler2D iTex8;
uniform sampler2D iTex9;
uniform sampler2D iTex10;
uniform int iSeed7;
uniform vec3 iMidi3_1[2];
@@ -22,8 +22,8 @@ void main() {
float mix_value = magic(iMidi3_1[1].xy, vec3(1, 0, 0), iSeed7);
bool mix_type = magic_trigger(vec3(iMidi3_1[0].x, 0, 0), iSeed7 + 10);
vec4 color_a = texture(iTex7, vUV);
vec4 color_b = texture(iTex8, vUV);
vec4 color_a = texture(iTex9, vUV);
vec4 color_b = texture(iTex10, vUV);
float k = mean(color_a);
+3 -3
View File
@@ -2,7 +2,7 @@
// MFX
// ------------
// IN: 9 (A+B)
// IN: 11 (A+B)
// IN: 0 (OUT)
// OUT: 0 (OUT)
@@ -11,7 +11,7 @@ out vec4 fragColor;
#include inc_fx.glsl
uniform sampler2D iTex9;
uniform sampler2D iTex11;
uniform sampler2D iTex0;
uniform int iSeed8;
uniform vec3 iMidi2_3[7];
@@ -20,7 +20,7 @@ uniform int iDemo;
uniform int iAutoRand;
void main() {
vec4 color = fx_stage(vUV, iTex9, iTex0, iSeed8, iMidi2_3[0], iMidi2_3[1].xy, iMidi2_3[2], iMidi2_3[3].xy, iMidi2_3[4], iMidi2_3[5].xy, iMidi2_3[6]);
vec4 color = fx_stage(vUV, iTex11, iTex0, iSeed8, iMidi2_3[0], iMidi2_3[1].xy, iMidi2_3[2], iMidi2_3[3].xy, iMidi2_3[4], iMidi2_3[5].xy, iMidi2_3[6]);
if (iDemo < 1 && iAutoRand < 1) {
color = mix(color, vec4(0), iMidi3_1[0].y);
+1
View File
@@ -68,6 +68,7 @@ static void init_context(const Parameters *params) {
memset(context->input_resolutions, 0, sizeof(context->input_resolutions));
memset(context->input_formats, 0, sizeof(context->input_formats));
memset(context->input_fps, 0, sizeof(context->input_fps));
memset(context->input_swap, 0, sizeof(context->input_swap));
}
static void free_context() { shared_close_context(context); }
+31 -7
View File
@@ -88,8 +88,10 @@ static void rebind_textures(const ShaderProgram *program) {
#ifdef VIDEO_IN
static void link_input_to_texture(ShaderProgram *program, VideoCapture *input,
unsigned int texture_index) {
input->dma_image = EGL_NO_IMAGE_KHR;
unsigned int texture_index, bool swap) {
EGLImageKHR dma_image;
dma_image = EGL_NO_IMAGE_KHR;
// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_image_dma_buf_import.txt
const EGLint attrib_list[] = {EGL_WIDTH,
@@ -99,18 +101,23 @@ static void link_input_to_texture(ShaderProgram *program, VideoCapture *input,
EGL_LINUX_DRM_FOURCC_EXT,
input->pixelformat,
EGL_DMA_BUF_PLANE0_FD_EXT,
input->exp_fd,
swap ? input->exp_fd_swap : input->exp_fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT,
0,
EGL_DMA_BUF_PLANE0_PITCH_EXT,
input->bytesperline,
EGL_NONE};
input->dma_image =
eglCreateImageKHR(program->egl_display, EGL_NO_CONTEXT,
dma_image = eglCreateImageKHR(program->egl_display, EGL_NO_CONTEXT,
EGL_LINUX_DMA_BUF_EXT, NULL, attrib_list);
if (input->dma_image == EGL_NO_IMAGE_KHR) {
if (swap) {
input->dma_image_swap = dma_image;
} else {
input->dma_image = dma_image;
}
if (dma_image == EGL_NO_IMAGE_KHR) {
log_error("(%s) eglCreateImageKHR failed %04x", input->name, eglGetError());
return;
}
@@ -137,7 +144,12 @@ static void init_input(ShaderProgram *program, const ConfigFile *config,
if (i < inputs->length && !inputs->values[i].error) {
snprintf(name, STR_LEN, "IN_%d_OUT", i + 1);
tex_i = config_file_get_int(config, name, 0);
link_input_to_texture(program, &inputs->values[i], tex_i);
link_input_to_texture(program, &inputs->values[i], tex_i, false);
if (inputs->values[i].with_swap) {
snprintf(name, STR_LEN, "IN_%d_SWAP_OUT", i + 1);
tex_i = config_file_get_int(config, name, 0);
link_input_to_texture(program, &inputs->values[i], tex_i, true);
}
} else {
log_warn("Cannot link input %d", i + 1);
}
@@ -321,6 +333,13 @@ static void init_single_program(ShaderProgram *program, unsigned int i,
glGetUniformLocation(program->programs[i], name);
}
prefix = config_file_get_str(config, "UNIFORM_IN_SWAP_PREFIX", "iInputSwap");
for (unsigned int j = 0; j < program->in_count; j++) {
snprintf(name, STR_LEN, "%s%d", prefix, j + 1);
program->iinswap_locations[i * program->in_count + j] =
glGetUniformLocation(program->programs[i], name);
}
prefix = config_file_get_str(config, "UNIFORM_SEED_PREFIX", "iSeed");
for (unsigned int j = 0; j < program->frag_count; j++) {
snprintf(name, STR_LEN, "%s%d", prefix, j + 1);
@@ -491,6 +510,8 @@ static void use_program(const ShaderProgram *program, int i, bool output,
context->input_formats[j]);
write_uniform_1i(program->iinfps_locations[i * program->in_count + j],
context->input_fps[j]);
write_uniform_1i(program->iinswap_locations[i * program->in_count + j],
context->input_swap[j] ? 1 : 0);
}
// set seeds uniforms
@@ -673,5 +694,8 @@ void shaders_free_input(const ShaderProgram *program,
if (!input->error && input->dma_image != EGL_NO_IMAGE_KHR) {
eglDestroyImageKHR(program->egl_display, input->dma_image);
}
if (!input->error && input->dma_image_swap != EGL_NO_IMAGE_KHR) {
eglDestroyImageKHR(program->egl_display, input->dma_image_swap);
}
#endif /* VIDEO_IN */
}
+6
View File
@@ -104,6 +104,7 @@ typedef struct ShaderProgram {
GLuint iinres_locations[ARRAY_SIZE];
GLuint iinfmt_locations[ARRAY_SIZE];
GLuint iinfps_locations[ARRAY_SIZE];
GLuint iinswap_locations[ARRAY_SIZE];
GLuint idemo_locations[ARRAY_SIZE];
GLuint iautorand_locations[ARRAY_SIZE];
GLuint iautorandcycle_locations[ARRAY_SIZE];
@@ -137,15 +138,19 @@ typedef struct ShaderProgram {
typedef struct VideoCapture {
char name[STR_LEN];
bool error;
bool with_swap;
int fd;
int exp_fd;
int exp_fd_swap;
unsigned int width;
unsigned int height;
unsigned int pixelformat;
unsigned int bytesperline;
#ifdef VIDEO_IN
struct v4l2_buffer buf;
struct v4l2_buffer buf_swap;
EGLImageKHR dma_image;
EGLImageKHR dma_image_swap;
#endif /* VIDEO_IN */
} VideoCapture;
@@ -190,6 +195,7 @@ typedef struct SharedContext {
unsigned int fps;
unsigned int input_formats[MAX_VIDEO];
unsigned int input_fps[MAX_VIDEO];
bool input_swap[MAX_VIDEO];
bool stop;
} SharedContext;
+59 -19
View File
@@ -75,7 +75,6 @@ static void open_device(VideoCapture *video_capture, const char *name) {
strlcpy(video_capture->name, name, STR_LEN);
video_capture->error = false;
video_capture->fd = -1;
video_capture->exp_fd = -1;
video_capture->fd = open(name, O_RDWR | O_NONBLOCK);
if (video_capture->fd == -1) {
@@ -201,7 +200,7 @@ static bool request_buffers(VideoCapture *video_capture) {
reqbuf.type = buf_type;
reqbuf.memory = V4L2_MEMORY_MMAP;
reqbuf.count = 1;
reqbuf.count = 2;
if (ioctl(video_capture->fd, VIDIOC_REQBUFS, &reqbuf) == -1) {
ioctl_error(video_capture, "VIDIOC_REQBUFS",
@@ -211,18 +210,21 @@ static bool request_buffers(VideoCapture *video_capture) {
log_info("(%s) V4L2 Buffer Count: %d", video_capture->name, reqbuf.count);
video_capture->with_swap = reqbuf.count > 1;
return true;
}
static bool export_buffer(VideoCapture *video_capture) {
static bool export_buffer(VideoCapture *video_capture, int *fd,
unsigned int index) {
struct v4l2_exportbuffer expbuf;
video_capture->exp_fd = -1;
*fd = -1;
memset(&expbuf, 0, sizeof(expbuf));
expbuf.type = buf_type;
expbuf.index = 0;
expbuf.index = index;
expbuf.flags = O_RDONLY;
if (ioctl(video_capture->fd, VIDIOC_EXPBUF, &expbuf) == -1) {
@@ -232,11 +234,24 @@ static bool export_buffer(VideoCapture *video_capture) {
return false;
}
video_capture->exp_fd = expbuf.fd;
*fd = expbuf.fd;
return true;
}
static bool export_buffers(VideoCapture *video_capture) {
bool result;
result = export_buffer(video_capture, &video_capture->exp_fd, 0);
if (result && video_capture->with_swap) {
result =
result && export_buffer(video_capture, &video_capture->exp_fd_swap, 1);
}
return result;
}
static bool open_stream(VideoCapture *video_capture) {
if (ioctl(video_capture->fd, VIDIOC_STREAMON, &buf_type) == -1) {
ioctl_error(
@@ -248,30 +263,47 @@ static bool open_stream(VideoCapture *video_capture) {
return true;
}
static void create_image_buffer(VideoCapture *video_capture) {
memset(&video_capture->buf, 0, sizeof(video_capture->buf));
static void create_image_buffer(const VideoCapture *video_capture,
struct v4l2_buffer *buf, unsigned int index) {
memset(buf, 0, sizeof(*buf));
video_capture->buf.type = buf_type;
video_capture->buf.memory = V4L2_MEMORY_MMAP;
video_capture->buf.index = 0;
buf->type = buf_type;
buf->memory = V4L2_MEMORY_MMAP;
buf->index = index;
ioctl(video_capture->fd, VIDIOC_PREPARE_BUF);
ioctl(video_capture->fd, VIDIOC_QBUF, buf);
}
ioctl(video_capture->fd, VIDIOC_QBUF, &video_capture->buf);
static void create_image_buffers(VideoCapture *video_capture) {
create_image_buffer(video_capture, &video_capture->buf, 0);
if (video_capture->with_swap) {
create_image_buffer(video_capture, &video_capture->buf_swap, 1);
}
}
static void close_stream(const VideoCapture *video_capture) {
ioctl(video_capture->fd, VIDIOC_STREAMOFF, &buf_type);
}
static bool read_video(VideoCapture *video_capture) {
static unsigned int read_video(const VideoCapture *video_capture) {
int result;
result = 0;
if (ioctl(video_capture->fd, VIDIOC_DQBUF, &video_capture->buf) != -1) {
ioctl(video_capture->fd, VIDIOC_QBUF, &video_capture->buf);
return true;
result = 1;
} else if (video_capture->with_swap &&
ioctl(video_capture->fd, VIDIOC_DQBUF, &video_capture->buf_swap) !=
-1) {
ioctl(video_capture->fd, VIDIOC_QBUF, &video_capture->buf_swap);
result = 2;
}
return false;
return result;
}
void video_init(VideoCapture *video_capture, const char *name,
@@ -298,7 +330,7 @@ void video_init(VideoCapture *video_capture, const char *name,
return;
}
if (!export_buffer(video_capture)) {
if (!export_buffers(video_capture)) {
return;
}
@@ -306,7 +338,7 @@ void video_init(VideoCapture *video_capture, const char *name,
return;
}
create_image_buffer(video_capture);
create_image_buffers(video_capture);
}
bool video_background_read(VideoCapture *video_capture, SharedContext *context,
@@ -314,6 +346,7 @@ bool video_background_read(VideoCapture *video_capture, SharedContext *context,
pid_t pid;
Timer timer;
double fps;
unsigned int video_result;
pid = fork();
if (pid < 0) {
@@ -328,7 +361,8 @@ bool video_background_read(VideoCapture *video_capture, SharedContext *context,
timer_init(&timer, 30);
while (!context->stop) {
if (read_video(video_capture) && timer_inc(&timer)) {
video_result = read_video(video_capture);
if (video_result > 0 && timer_inc(&timer)) {
fps = timer_reset(&timer);
context->input_fps[input_index] = (unsigned int)round(fps);
@@ -336,6 +370,9 @@ bool video_background_read(VideoCapture *video_capture, SharedContext *context,
log_trace("(%s) %.2ffps", video_capture->name, fps);
}
}
if (video_result > 0) {
context->input_swap[input_index] = video_result == 2;
}
}
if (context->stop) {
log_info("(%s) background acquisition stopped by main thread (pid: %d)",
@@ -355,6 +392,9 @@ void video_free(const VideoCapture *video_capture) {
if (video_capture->exp_fd != -1) {
close(video_capture->exp_fd);
}
if (video_capture->exp_fd_swap != -1) {
close(video_capture->exp_fd);
}
if (video_capture->fd != -1) {
close(video_capture->fd);
}