[FFmpeg-cvslog] avcodec/nvdec: fix 10bit output pixel formats

[Timo Rothenpieler]

ffmpeg | branch: master | Timo Rothenpieler <timo at rothenpieler.org> | Fri Jul  4 17:20:40 2025 +0200| [bf5f3f1f2e6ef56b060c454de9d27c6aabf30b78] | committer: Timo Rothenpieler

avcodec/nvdec: fix 10bit output pixel formats

Fixes #11655

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=bf5f3f1f2e6ef56b060c454de9d27c6aabf30b78
---

 libavcodec/nvdec.c         | 14 ++++++++++++--
 libavutil/hwcontext_cuda.c |  3 ++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c
index d444135fd7..3a71d9d97e 100644
--- a/libavcodec/nvdec.c
+++ b/libavcodec/nvdec.c
@@ -754,15 +754,25 @@ int ff_nvdec_frame_params(AVCodecContext *avctx,
         }
         break;
     case 10:
+        if (chroma_444) {
+            frames_ctx->sw_format = AV_PIX_FMT_YUV444P16;
+#ifdef NVDEC_HAVE_422_SUPPORT
+        } else if (cuvid_chroma_format == cudaVideoChromaFormat_422) {
+            frames_ctx->sw_format = AV_PIX_FMT_P210;
+#endif
+        } else {
+            frames_ctx->sw_format = AV_PIX_FMT_P010;
+        }
+        break;
     case 12:
         if (chroma_444) {
             frames_ctx->sw_format = AV_PIX_FMT_YUV444P16;
 #ifdef NVDEC_HAVE_422_SUPPORT
         } else if (cuvid_chroma_format == cudaVideoChromaFormat_422) {
-            frames_ctx->sw_format = AV_PIX_FMT_P216LE;
+            frames_ctx->sw_format = AV_PIX_FMT_P216;
 #endif
         } else {
-            frames_ctx->sw_format = AV_PIX_FMT_P016LE;
+            frames_ctx->sw_format = AV_PIX_FMT_P016;
         }
         break;
     default:
diff --git a/libavutil/hwcontext_cuda.c b/libavutil/hwcontext_cuda.c
index 5721612225..ce2ded597f 100644
--- a/libavutil/hwcontext_cuda.c
+++ b/libavutil/hwcontext_cuda.c
@@ -48,7 +48,8 @@ static const enum AVPixelFormat supported_formats[] = {
     AV_PIX_FMT_YUV444P,
     AV_PIX_FMT_P010,
     AV_PIX_FMT_P016,
-    AV_PIX_FMT_P216LE,
+    AV_PIX_FMT_P210,
+    AV_PIX_FMT_P216,
     AV_PIX_FMT_YUV444P16,
     AV_PIX_FMT_0RGB32,
     AV_PIX_FMT_0BGR32,