[FFmpeg-devel] [PATCH 1/2] avutil: Add MSB packed YUV444P10 format

[Philip Langdale]

nvenc supports a YUV444P10 format with the same bit layout as P010,
with the data bits at the MSB end.

Unfortunately, the current YUV444P10 format we have defined puts
the data bits at the LSB end.

This mismatch led to us introducing a fudge in nvenc where we
mapped their 444P10 format to the ffmpeg 444P16 format. This
ensured that the data ends up in the right place if you are
starting with 10bit content, but it leads to other problems.

Specifically:

* >10bit content will be preferrentially converted to 444P16
  to pass to nvenc to 'preserve' the additional bits. However, they
  aren't actually preserved, but are discarded, so we're worse off
  because no dithering took place in swscale.
* On top of that, the input data is almost certainly 4:2:0 and so
  the conversion to 4:4:4 is pointless, while also leading to an
  output file that's incompatible with many playback scenarios
  (no hardware decoder supports 4:4:4 content).

So, for the sake of accuracy, introduce an explicit pixfmt for this
situation. There's no conversion support because you'll basically
never have a use for it. If someone ever finds one, they can write
the swscale code.

In the mean time, common 12 bit content (YUV420P12 or P016) will be
correctly converted to P010 for nvenc.

Signed-off-by: Philip Langdale <philipl at overt.org>
---
 libavutil/pixdesc.c | 24 ++++++++++++++++++++++++
 libavutil/pixfmt.h  |  5 +++++
 libavutil/version.h |  2 +-
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 3b9c45d..fe9b59d 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -1565,6 +1565,30 @@ static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = {
         },
         .flags = AV_PIX_FMT_FLAG_BE | AV_PIX_FMT_FLAG_PLANAR,
     },
+    [AV_PIX_FMT_YUV444P10MSBLE] = {
+        .name = "yuv444p10msble",
+        .nb_components = 3,
+        .log2_chroma_w = 0,
+        .log2_chroma_h = 0,
+        .comp = {
+            { 0, 2, 0, 6, 10, 1, 9, 1 },        /* Y */
+            { 1, 2, 0, 6, 10, 1, 9, 1 },        /* U */
+            { 2, 2, 0, 6, 10, 1, 9, 1 },        /* V */
+        },
+        .flags = AV_PIX_FMT_FLAG_PLANAR,
+    },
+    [AV_PIX_FMT_YUV444P10MSBBE] = {
+        .name = "yuv444p10msbbe",
+        .nb_components = 3,
+        .log2_chroma_w = 0,
+        .log2_chroma_h = 0,
+        .comp = {
+            { 0, 2, 0, 6, 10, 1, 9, 1 },        /* Y */
+            { 1, 2, 0, 6, 10, 1, 9, 1 },        /* U */
+            { 2, 2, 0, 6, 10, 1, 9, 1 },        /* V */
+        },
+        .flags = AV_PIX_FMT_FLAG_BE | AV_PIX_FMT_FLAG_PLANAR,
+    },
     [AV_PIX_FMT_YUV444P9LE] = {
         .name = "yuv444p9le",
         .nb_components = 3,
diff --git a/libavutil/pixfmt.h b/libavutil/pixfmt.h
index dfb1b11..46bf9c0 100644
--- a/libavutil/pixfmt.h
+++ b/libavutil/pixfmt.h
@@ -314,6 +314,9 @@ enum AVPixelFormat {
     AV_PIX_FMT_P016LE, ///< like NV12, with 16bpp per component, little-endian
     AV_PIX_FMT_P016BE, ///< like NV12, with 16bpp per component, big-endian
 
+    AV_PIX_FMT_YUV444P10MSBLE, /// like YUV444P10 but with data in the high bits, zeros in the low bits, little-endian
+    AV_PIX_FMT_YUV444P10MSBBE, /// like YUV444P10 but with data in the high bits, zeros in the low bits, big-endian
+
     AV_PIX_FMT_NB         ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions
 };
 
@@ -394,6 +397,8 @@ enum AVPixelFormat {
 #define AV_PIX_FMT_P010       AV_PIX_FMT_NE(P010BE,  P010LE)
 #define AV_PIX_FMT_P016       AV_PIX_FMT_NE(P016BE,  P016LE)
 
+#define AV_PIX_FMT_YUV444P10MSB AV_PIX_FMT_NE(YUV444P10MSBBE, YUV444P10MSBLE)
+
 /**
   * Chromaticity coordinates of the source primaries.
   */
diff --git a/libavutil/version.h b/libavutil/version.h
index 8866064..a8b00bf 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  55
-#define LIBAVUTIL_VERSION_MINOR  46
+#define LIBAVUTIL_VERSION_MINOR  47
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
-- 
2.9.3