From 173487b8955c8ae7e23908d129889c723b4f0d10 Mon Sep 17 00:00:00 2001
From: "byte[]" <byteslice@airmail.cc>
Date: Wed, 27 May 2020 01:04:23 -0400
Subject: [PATCH] be pickier about image/video format distinction

---
 src/validation.c | 49 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/validation.c b/src/validation.c
index 6156358..9996519 100644
--- a/src/validation.c
+++ b/src/validation.c
@@ -8,10 +8,11 @@
 const AVRational t_1h = { 3600, 1 };
 const AVRational t_0h  = { 0, 1 };
 
-static int validate_video_pixel_format(enum AVPixelFormat format)
+static int validate_image_pixel_format(enum AVPixelFormat format)
 {
     switch (format) {
     // Still image formats
+    case AV_PIX_FMT_GBRAP:
     case AV_PIX_FMT_MONOBLACK:
     case AV_PIX_FMT_MONOWHITE:
     case AV_PIX_FMT_YA16LE:
@@ -40,7 +41,16 @@ static int validate_video_pixel_format(enum AVPixelFormat format)
     case AV_PIX_FMT_YUVA420P:
     case AV_PIX_FMT_YUVA422P:
     case AV_PIX_FMT_YUVA444P:
+        return true;
 
+    default:
+        return false;
+    }
+}
+
+static int validate_video_pixel_format(enum AVPixelFormat format)
+{
+    switch (format) {
     // Video frame formats
     case AV_PIX_FMT_YUV420P:
     case AV_PIX_FMT_YUV420P10LE:
@@ -88,21 +98,9 @@ int mediatools_validate_video(AVFormatContext *format)
         if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
             num_vstreams++;
             vstream_idx = i;
-
-            if (!validate_video_pixel_format(codecpar->format)) {
-                printf("Found unsupported pixel format %s\n", av_get_pix_fmt_name(codecpar->format));
-                return false;
-            }
-
         } else if (codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
             num_astreams++;
             astream_idx = i;
-
-            if (!validate_audio_sample_format(codecpar->format)) {
-                printf("Found unsupported audio sample format %s\n", av_get_sample_fmt_name(codecpar->format));
-                return false;
-            }
-
         } else if (codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
             // Allow subtitles
         } else {
@@ -138,6 +136,11 @@ int mediatools_validate_video(AVFormatContext *format)
             ;
         }
 
+        if (!validate_video_pixel_format(vpar->format)) {
+            printf("Found unsupported pixel format %s\n", av_get_pix_fmt_name(vpar->format));
+            return false;
+        }
+
         if (apar) {
             switch (apar->codec_id) {
             default:
@@ -147,6 +150,11 @@ int mediatools_validate_video(AVFormatContext *format)
             case AV_CODEC_ID_OPUS:
                 ;
             }
+
+            if (!validate_audio_sample_format(apar->format)) {
+                printf("Found unsupported audio sample format %s\n", av_get_sample_fmt_name(apar->format));
+                return false;
+            }
         }
     } else if (strcmp(iformat->name, "gif") == 0) {
         switch (vpar->codec_id) {
@@ -156,6 +164,11 @@ int mediatools_validate_video(AVFormatContext *format)
         case AV_CODEC_ID_GIF:
             ;
         }
+
+        if (!validate_image_pixel_format(vpar->format)) {
+            printf("Found unsupported pixel format %s\n", av_get_pix_fmt_name(vpar->format));
+            return false;
+        }
     } else if (strcmp(iformat->name, "image2") == 0 || strcmp(iformat->name, "jpeg_pipe") == 0) {
         switch (vpar->codec_id) {
         default:
@@ -164,6 +177,11 @@ int mediatools_validate_video(AVFormatContext *format)
         case AV_CODEC_ID_MJPEG:
             ;
         }
+
+        if (!validate_image_pixel_format(vpar->format)) {
+            printf("Found unsupported pixel format %s\n", av_get_pix_fmt_name(vpar->format));
+            return false;
+        }
     } else if (strcmp(iformat->name, "png_pipe") == 0 || strcmp(iformat->name, "apng") == 0) {
         switch (vpar->codec_id) {
         default:
@@ -173,6 +191,11 @@ int mediatools_validate_video(AVFormatContext *format)
         case AV_CODEC_ID_APNG:
             ;
         }
+
+        if (!validate_image_pixel_format(vpar->format)) {
+            printf("Found unsupported pixel format %s\n", av_get_pix_fmt_name(vpar->format));
+            return false;
+        }
     } else if (strcmp(iformat->name, "svg_pipe") == 0) {
         switch (vpar->codec_id) {
         default: