[FFmpeg-cvslog] avfilter/scale_cuda: fix pitch calculation for >8 bit formats

[Timo Rothenpieler]

ffmpeg | branch: master | Timo Rothenpieler <timo at rothenpieler.org> | Thu Nov  5 00:46:54 2020 +0100| [a1553b0cfbff92223caaed4103b92373b2339173] | committer: Timo Rothenpieler

avfilter/scale_cuda: fix pitch calculation for >8 bit formats

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a1553b0cfbff92223caaed4103b92373b2339173
---

 libavfilter/vf_scale_cuda.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 4de5cc29e8..99fef6b22b 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -406,8 +406,6 @@ static int call_resize_kernel(AVFilterContext *ctx, CUfunction func, int channel
                            &src_width, &src_height, &bit_depth, &s->param };
     int ret;
 
-    dst_pitch /= channels;
-
     CUDA_TEXTURE_DESC tex_desc = {
         .filterMode = s->interp_use_linear ?
                       CU_TR_FILTER_MODE_LINEAR :
@@ -427,6 +425,10 @@ static int call_resize_kernel(AVFilterContext *ctx, CUfunction func, int channel
         .res.pitch2D.devPtr = (CUdeviceptr)src_dptr,
     };
 
+    // Handling of channels is done via vector-types in cuda, so their size is implicitly part of the pitch
+    // Same for pixel_size, which is represented via datatypes on the cuda side of things.
+    dst_pitch /= channels * pixel_size;
+
     ret = CHECK_CU(cu->cuTexObjectCreate(&tex, &res_desc, &tex_desc, NULL));
     if (ret < 0)
         goto exit;