[FFmpeg-devel] adding RGBA and BGRA to nvenc.c

[Sven C. Dack]

On 07/09/16 12:27, Carl Eugen Hoyos wrote:
> If your patch adds "RGBA" as supported colour space,
> the encoder (by definition) announces that it supports
> transparency.
> nvenc does not support transparency (if it would, it would
> be a - grave - bug to use "RGB0" in the patch). So please
> do not add it.

Done and tested.

Now please explain to me what RGB0/BGR0 exactly means in FFmpeg terms. It says 
in the headers "0" stands for undefined. Does this mean anything can be present 
in the "0"-field or can one rely on the "0"-field to always contain 0x00?

Sven


--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -81,6 +81,8 @@ const enum AVPixelFormat ff_nvenc_pix_fmts[] = {
      AV_PIX_FMT_P010,
      AV_PIX_FMT_YUV444P,
      AV_PIX_FMT_YUV444P16,
+    AV_PIX_FMT_RGB0,
+    AV_PIX_FMT_BGR0,
  #if CONFIG_CUDA
      AV_PIX_FMT_CUDA,
  #endif
@@ -1032,6 +1034,14 @@ static av_cold int nvenc_alloc_surface(AVCodecContext 
*avctx, int idx)
          ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_YUV444_10BIT;
          break;

+    case AV_PIX_FMT_RGB0:
+        ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_ABGR;
+        break;
+
+    case AV_PIX_FMT_BGR0:
+        ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_ARGB;
+        break;
+
      default:
          av_log(avctx, AV_LOG_FATAL, "Invalid input pixel format\n");
          return AVERROR(EINVAL);
@@ -1350,6 +1360,10 @@ static int nvenc_copy_frame(AVCodecContext *avctx, 
NvencSurface *inSurf,
          av_image_copy_plane(buf, lockBufferParams->pitch,
              frame->data[2], frame->linesize[2],
              avctx->width << 1, avctx->height);
+    } else if (frame->format == AV_PIX_FMT_RGB0 || frame->format == 
AV_PIX_FMT_BGR0) {
+      av_image_copy_plane(buf, lockBufferParams->pitch,
+           frame->data[0], frame->linesize[0],
+           avctx->width << 2, avctx->height);
      } else {
          av_log(avctx, AV_LOG_FATAL, "Invalid pixel format!\n");
          return AVERROR(EINVAL);