feat: select fourcc and handle hw decoded

README.md

@@ -174,9 +174,9 @@ These are configurable in the [`forge_project.cfg`](#forge_projectcfg).
 ### CLI arguments
 
 ```txt
-forge steel-VERSION
+forge steel-dev
 
-usage: forge [-h] [-v] [-p=PROJECT_PATH] [-c=CFG_FILE] [-hr] [-s=SCREEN] [-m=SCREEN] [-mo] [-w] [-t=TEMPO] [-d] [-ar / -nar] [-arc=CYCLES] [-vi=FILE] [-vs=SIZE] [-vb=COUNT] [-vr / -nvr] [-is=SIZE] [-ls / -nls] [-ss / -nss] [-mr / -nmr] [-tm] [-tf]
+usage: forge [-h] [-v] [-p=PROJECT_PATH] [-c=CFG_FILE] [-hr] [-s=SCREEN] [-m=SCREEN] [-mo] [-w] [-t=TEMPO] [-d] [-ar / -nar] [-arc=CYCLES] [-vi=FILE] [-vs=SIZE] [-vb=COUNT] [-vr / -nvr] [-vf=FOURCC] [-is=SIZE] [-ls / -nls] [-ss / -nss] [-mr / -nmr] [-tm] [-tf]
 
 Fusion Of Real-time Generative Effects.
 
@@ -200,6 +200,7 @@ options:
   -vs, --video-size           video capture desired height (default: internal texture height)
   -vr, --video-reconnect      auto-reconnect video (default)
   -nvr, --no-video-reconnect  do not auto-reconnect video
+  -vf, --video-fourcc         video codec fourcc (default: YUYV)
   -is, --internal-size        internal texture height (default: 720)
   -ls, --load-state           load saved state (default)
   -nls, --no-load-state       do not load saved state
@@ -499,7 +500,7 @@ You can check your device real FPS on [V4L2 UCP](https://github.com/HedgeHawk/v4
 
 ### My video feed got strange lines
 
-You need to decode the [V4L2 YUYV format](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/pixfmt-yuyv.html).
+You may need to decode the [V4L2 YUYV format](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/pixfmt-yuyv.html).
 
 The code is available in the default project [default/inc_yuyv.glsl](./default/inc_yuyv.glsl)
 

default/forge_project.cfg

@@ -269,7 +269,7 @@ GROUP_3_1_Y=58
 GROUP_3_1_Z=
 GROUP_3_2_X=0
 GROUP_3_2_Y=16
-GROUP_3_2_Z=
+GROUP_3_2_Z=11089
 
 # =====
 # OTHER

default/frag1.glsl

@@ -15,9 +15,12 @@ uniform sampler2D iTex1;
 uniform sampler2D iTex3;
 uniform int iInputFormat1;
 uniform vec2 iInputResolution1;
+uniform vec3 iGroup3_1[2];
 
 void main() {
-    if (iInputFormat1 == YUYV_FOURCC) {
+    if (iGroup3_1[1].z > 0) {
+        fragColor = texture(iTex1, vUV * vec2(1, -1));
+    } else if (iInputFormat1 == YUYV_FOURCC) {
         fragColor = yuyvTex(iTex1, vUV, int(iInputResolution1.x));
     } else {
         fragColor = texture(iTex0, vUV);

default/frag2.glsl

@@ -15,9 +15,12 @@ uniform sampler2D iTex2;
 uniform sampler2D iTex4;
 uniform int iInputFormat2;
 uniform vec2 iInputResolution2;
+uniform vec3 iGroup3_1[2];
 
 void main() {
-    if (iInputFormat2 == YUYV_FOURCC) {
+    if (iGroup3_1[1].z > 0) {
+        fragColor = texture(iTex2, vUV * vec2(1, -1));
+    } else if (iInputFormat2 == YUYV_FOURCC) {
         fragColor = yuyvTex(iTex2, vUV, int(iInputResolution2.x));
     } else {
         fragColor = texture(iTex0, vUV);

default/inc_yuyv.glsl

@@ -3,11 +3,27 @@
 
 const int YUYV_FOURCC = 1448695129;
 
-const mat3x3 yuyv_to_rgb = {{1,1,1},{0,-0.39465,2.03211},{1.13983,-0.5806,0}};
+// https://en.wikipedia.org/wiki/Y%E2%80%B2UV
+const mat3x3 yuyv_to_rgb_bt709 = {
+        {
+            1,
+            1,
+            1
+        },
+        {
+            0,
+            -0.21482,
+            2.12798
+        },
+        {
+            1.28033,
+            -0.38059,
+            0
+        }
+    };
 
 vec4 yuyvTex(sampler2D tex, vec2 vUV, int base_width) {
     float w = base_width - 1;
-
     int x = int(vUV.x * w);
 
     int xU = x - x % 2;
@@ -17,12 +33,12 @@ vec4 yuyvTex(sampler2D tex, vec2 vUV, int base_width) {
     vec4 tV = texture(tex, vec2(xV / w, 1 - vUV.y));
 
     vec3 yuv = vec3(
-        x % 2 == 0 ? tU.x : tV.x,
-        tU.y - 0.5,
-        tV.y - 0.5
-    );
+            x % 2 == 0 ? tU.x : tV.x,
+            tU.y - 0.5,
+            tV.y - 0.5
+        );
 
-    return vec4(yuyv_to_rgb * yuv, 1.0);
+    return vec4(yuyv_to_rgb_bt709 * yuv, 1.0);
 }
 
 #endif

src/args.c

@@ -34,6 +34,7 @@ static void print_help(int status_code) {
       "[-vs=SIZE] "
       "[-vb=COUNT] "
       "[-vr / -nvr] "
+      "[-vf=FOURCC] "
 #endif /* VIDEO_IN */
       "[-is=SIZE] "
       "[-ls / -nls] "
@@ -70,6 +71,7 @@ static void print_help(int status_code) {
       "internal texture height)\n"
       "  -vr, --video-reconnect      auto-reconnect video (default)\n"
       "  -nvr, --no-video-reconnect  do not auto-reconnect video\n"
+      "  -vf, --video-fourcc         video codec fourcc (default: YUYV)\n"
 #endif /* VIDEO_IN */
       "  -is, --internal-size        internal texture height (default: 720)\n"
       "  -ls, --load-state           load saved state (default)\n"
@@ -142,6 +144,7 @@ void args_parse(Parameters *params, int argc, char **argv) {
   params->video_buffers = 2;
   params->video_size = 0;
   params->video_reconnect = true;
+  strlcpy(params->video_fourcc, "YUYV", 5);
 #endif /* VIDEO_IN */
   params->internal_size = 720;
   params->load_state = true;
@@ -243,6 +246,11 @@ void args_parse(Parameters *params, int argc, char **argv) {
         params->video_reconnect = true;
       } else if (is_arg(arg, "-nvr") || is_arg(arg, "--no-video-reconnect")) {
         params->video_reconnect = false;
+      } else if (is_arg(arg, "-vf") || is_arg(arg, "--video-fourcc")) {
+        if (strlen(value) == 0) {
+          invalid_value(arg, value);
+        }
+        strlcpy(params->video_fourcc, value, 5);
       } else {
         invalid_arg(arg);
       }

src/forge.c

@@ -85,7 +85,8 @@ static void init_inputs() {
 
   for (unsigned int i = 0; i < init_params.video_in.length; i++) {
     video_init(&video_captures.values[i], init_params.video_in.values[i],
-               init_params.video_size, init_params.video_buffers, true);
+               init_params.video_size, init_params.video_buffers,
+               init_params.video_fourcc, true);
 
     if (!video_captures.values[i].error) {
       context.input_resolutions[i][0] = video_captures.values[i].width;
@@ -125,7 +126,8 @@ background_reconnect_video_captures(__attribute__((unused)) void *args) {
       if (video_captures.values[i].disconnected) {
         video_free(&video_captures.values[i]);
         video_init(&video_captures.values[i], init_params.video_in.values[i],
-                   init_params.video_size, init_params.video_buffers, false);
+                   init_params.video_size, init_params.video_buffers,
+                   init_params.video_fourcc, false);
 
         if (!video_captures.values[i].error) {
           context.input_resolutions[i][0] = video_captures.values[i].width;

src/shaders.c

@@ -245,12 +245,6 @@ static bool link_input_to_texture(ShaderProgram *program, VideoCapture *input,
     return false;
   }
 
-  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, input->width, input->height, 0, GL_RGB,
-               GL_UNSIGNED_BYTE, 0);
-  if (check_glerror(program, "link_input_to_texture/glTexImage2D")) {
-    return false;
-  }
-
   // https://registry.khronos.org/OpenGL/extensions/EXT/EXT_EGL_image_storage.txt
   glEGLImageTargetTexStorageEXT(GL_TEXTURE_2D, dma_image, NULL);
   if (check_eglerror(program,

src/types.h

@@ -50,6 +50,7 @@ typedef struct Parameters {
   unsigned int video_buffers;
   unsigned int video_size;
   bool video_reconnect;
+  char video_fourcc[5];
 #endif /* VIDEO_IN */
   unsigned int internal_size;
   bool load_state;

src/video.c

@@ -16,7 +16,6 @@
 #include "timer.h"
 #include "video.h"
 
-static const unsigned int pixel_format = V4L2_PIX_FMT_YUYV;
 static const enum v4l2_buf_type buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 
 static void ioctl_error(VideoCapture *video_capture, const char *operation,
@@ -70,6 +69,19 @@ static void ioctl_error(VideoCapture *video_capture, const char *operation,
   video_capture->error = true;
 }
 
+static void fourcc_to_string(unsigned int fourcc, char *str) {
+  str[0] = (char)(fourcc & 0xFF);
+  str[1] = (char)((fourcc >> 8) & 0xFF);
+  str[2] = (char)((fourcc >> 16) & 0xFF);
+  str[3] = (char)((fourcc >> 24) & 0xFF);
+  str[4] = '\0';
+}
+
+static int string_to_fourcc(const char *str) {
+  return (unsigned int)str[0] | ((unsigned int)str[1] << 8) |
+         ((unsigned int)str[2] << 16) | ((unsigned int)str[3] << 24);
+}
+
 static void open_device(VideoCapture *video_capture, const char *name,
                         bool log_error) {
   strlcpy(video_capture->name, name, STR_LEN);
@@ -112,7 +124,8 @@ static bool check_caps(VideoCapture *video_capture) {
 }
 
 static bool get_available_sizes(VideoCapture *video_capture,
-                                unsigned int preferred_height) {
+                                unsigned int preferred_height,
+                                unsigned int pixel_format) {
   struct v4l2_frmsizeenum fmt_enum;
   unsigned int index;
   bool found = false;
@@ -165,8 +178,9 @@ static bool get_available_sizes(VideoCapture *video_capture,
   return true;
 }
 
-static bool set_format(VideoCapture *video_capture) {
+static bool set_format(VideoCapture *video_capture, unsigned int pixel_format) {
   struct v4l2_format fmt;
+  char fourcc[STR_LEN];
 
   memset(&fmt, 0, sizeof(fmt));
 
@@ -187,9 +201,9 @@ static bool set_format(VideoCapture *video_capture) {
   video_capture->pixelformat = fmt.fmt.pix.pixelformat;
   video_capture->bytesperline = fmt.fmt.pix.bytesperline;
 
-  log_info("(%s) Format fourcc:  %c%c%c%c", video_capture->name,
-           fmt.fmt.pix.pixelformat, fmt.fmt.pix.pixelformat >> 8,
-           fmt.fmt.pix.pixelformat >> 16, fmt.fmt.pix.pixelformat >> 24);
+  fourcc_to_string(fmt.fmt.pix.pixelformat, fourcc);
+
+  log_info("(%s) Format fourcc:  %s", video_capture->name, fourcc);
   log_info("(%s) Resolution:  %dx%d", video_capture->name, fmt.fmt.pix.width,
            fmt.fmt.pix.height);
 
@@ -326,7 +340,7 @@ static unsigned int read_video(VideoCapture *video_capture) {
 
 void video_init(VideoCapture *video_capture, const char *name,
                 unsigned int preferred_height, unsigned int buffer_count,
-                bool log_error) {
+                const char *video_fourcc, bool log_error) {
   open_device(video_capture, name, log_error);
 
   if (video_capture->error) {
@@ -339,13 +353,14 @@ void video_init(VideoCapture *video_capture, const char *name,
     return;
   }
 
-  if (!get_available_sizes(video_capture, preferred_height)) {
+  if (!get_available_sizes(video_capture, preferred_height,
+                           string_to_fourcc(video_fourcc))) {
     video_capture->error = true;
     video_free(video_capture);
     return;
   }
 
-  if (!set_format(video_capture)) {
+  if (!set_format(video_capture, string_to_fourcc(video_fourcc))) {
     video_capture->error = true;
     video_free(video_capture);
     return;

src/video.h

@@ -7,7 +7,7 @@
 
 void video_init(VideoCapture *video_capture, const char *name,
                 unsigned int preferred_height, unsigned int buffer_count,
-                bool log_error);
+                const char *video_fourcc, bool log_error);
 
 void *video_background_read(void *args);