[FFmpeg-devel] [PATCH v06.01] hwcontext_drm: check pixformats before detile

[hanishkvc]

Verify that the source and destination pixformats are supported by
fbtile before detiling the source into destination.

NOTE: This patch is relative to my v06 patch set.

---
 libavutil/hwcontext_drm.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/libavutil/hwcontext_drm.c b/libavutil/hwcontext_drm.c
index bd74b3f13d..b80ebc5b2d 100644
--- a/libavutil/hwcontext_drm.c
+++ b/libavutil/hwcontext_drm.c
@@ -196,17 +196,20 @@ static int drm_transfer_with_detile(const AVFrame *hwAVFrame, AVFrame *dst, cons
     int err = 0;
 
     if (hwAVFrame->format  == AV_PIX_FMT_DRM_PRIME) {
-        AVDRMFrameDescriptor *drmFrame = (AVDRMFrameDescriptor*)hwAVFrame->data[0];
-        uint64_t formatModifier = drmFrame->objects[0].format_modifier;
-        if (formatModifier != DRM_FORMAT_MOD_LINEAR) {
-            err = detile_this(TILE_AUTO, formatModifier, dst->width, dst->height,
-                              dst->data[0], dst->linesize[0],
-                              src->data[0], src->linesize[0], 4);
-            if (!err) {
+        int ok = fbtile_checkpixformats(src->format, dst->format);
+        if (ok) {
+            AVDRMFrameDescriptor *drmFrame = (AVDRMFrameDescriptor*)hwAVFrame->data[0];
+            uint64_t formatModifier = drmFrame->objects[0].format_modifier;
+            if (formatModifier != DRM_FORMAT_MOD_LINEAR) {
+                err = detile_this(TILE_AUTO, formatModifier, dst->width, dst->height,
+                                  dst->data[0], dst->linesize[0],
+                                  src->data[0], src->linesize[0], 4);
+                if (!err) {
 #if HWCTXDRM_SYNCRELATED_FORMATMODIFIER
-                drmFrame->objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR;
+                    drmFrame->objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR;
 #endif
-                return 0;
+                    return 0;
+                }
             }
         }
     }
-- 
2.25.1