diff mbox series

[FFmpeg-devel,v06.01] hwcontext_drm: check pixformats before detile

Message ID 20200704170225.125426-1-hanishkvc@gmail.com
State New
Headers show
Series [FFmpeg-devel,v06.01] hwcontext_drm: check pixformats before detile | expand

Checks

Context Check Description
andriy/default pending
andriy/configure warning Failed to apply patch

Commit Message

hanishkvc July 4, 2020, 5:02 p.m. UTC 
Verify that the source and destination pixformats are supported by
fbtile before detiling the source into destination.

NOTE: This patch is relative to my v06 patch set.

---
 libavutil/hwcontext_drm.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)
diff mbox series

Patch

diff --git a/libavutil/hwcontext_drm.c b/libavutil/hwcontext_drm.c
index bd74b3f13d..b80ebc5b2d 100644
--- a/libavutil/hwcontext_drm.c
+++ b/libavutil/hwcontext_drm.c
@@ -196,17 +196,20 @@  static int drm_transfer_with_detile(const AVFrame *hwAVFrame, AVFrame *dst, cons
     int err = 0;
 
     if (hwAVFrame->format  == AV_PIX_FMT_DRM_PRIME) {
-        AVDRMFrameDescriptor *drmFrame = (AVDRMFrameDescriptor*)hwAVFrame->data[0];
-        uint64_t formatModifier = drmFrame->objects[0].format_modifier;
-        if (formatModifier != DRM_FORMAT_MOD_LINEAR) {
-            err = detile_this(TILE_AUTO, formatModifier, dst->width, dst->height,
-                              dst->data[0], dst->linesize[0],
-                              src->data[0], src->linesize[0], 4);
-            if (!err) {
+        int ok = fbtile_checkpixformats(src->format, dst->format);
+        if (ok) {
+            AVDRMFrameDescriptor *drmFrame = (AVDRMFrameDescriptor*)hwAVFrame->data[0];
+            uint64_t formatModifier = drmFrame->objects[0].format_modifier;
+            if (formatModifier != DRM_FORMAT_MOD_LINEAR) {
+                err = detile_this(TILE_AUTO, formatModifier, dst->width, dst->height,
+                                  dst->data[0], dst->linesize[0],
+                                  src->data[0], src->linesize[0], 4);
+                if (!err) {
 #if HWCTXDRM_SYNCRELATED_FORMATMODIFIER
-                drmFrame->objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR;
+                    drmFrame->objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR;
 #endif
-                return 0;
+                    return 0;
+                }
             }
         }
     }