[FFmpeg-devel,1/2] lavu: Add DRM hwcontext

Submitted by Mark Thompson on June 18, 2017, 5:44 p.m.

Details

Message ID 4ccd61f6-860b-d473-5ec6-061525972d7b@jkqxz.net
State New
Headers show

Commit Message

Mark Thompson June 18, 2017, 5:44 p.m.
---
This grew out of the RockChip decoder discussion following from <https://lists.ffmpeg.org/pipermail/ffmpeg-devel/2017-June/212339.html>.

The intent of this is to have a common structure which can be used in all cases where DRM objects need to be shared between components.  It would be helpful if anyone familiar with specific drivers or use-cases could ensure that the structure (see the hwcontext_drm.h header) is sufficiently general to cover them - we would like this to the one answer and never require any more formats in future.

The immediate use-case will be with the RockChip decoder as producer and EGL or DRM framebuffers as the consumer.  Others will hopefully follow - mobile cases with V4L2 M2M and DRM are likely targets, as are some desktop interop cases which currently use DRM objects underneath other APIs (like VAAPI and EGL).

Included here as patch 2/2 is some test code to see it working on on a desktop platform (Intel / VAAPI).  This is not intended to be applied - it doesn't actually provide any useful features, and is hard coding a lot of detail which should be configurable.

Thanks,

- Mark


 configure                      |   3 +
 libavutil/Makefile             |   2 +
 libavutil/hwcontext.c          |   4 +
 libavutil/hwcontext.h          |   1 +
 libavutil/hwcontext_drm.c      | 280 +++++++++++++++++++++++++++++++++++++++++
 libavutil/hwcontext_drm.h      | 145 +++++++++++++++++++++
 libavutil/hwcontext_internal.h |   1 +
 libavutil/pixdesc.c            |   4 +
 libavutil/pixfmt.h             |   4 +
 9 files changed, 444 insertions(+)
 create mode 100644 libavutil/hwcontext_drm.c
 create mode 100644 libavutil/hwcontext_drm.h

Patch hide | download patch | download mbox

diff --git a/configure b/configure
index 9509bdf09c..82d15f338c 100755
--- a/configure
+++ b/configure
@@ -297,6 +297,7 @@  External library support:
   --enable-cuda-sdk        enable CUDA features that require the CUDA SDK [no]
   --disable-cuvid          disable Nvidia CUVID support [autodetect]
   --disable-d3d11va        disable Microsoft Direct3D 11 video acceleration code [autodetect]
+  --enable-drm             enable DRM code (Linux) [no]
   --disable-dxva2          disable Microsoft DirectX 9 video acceleration code [autodetect]
   --enable-libmfx          enable Intel MediaSDK (AKA Quick Sync Video) code via libmfx [no]
   --enable-libnpp          enable Nvidia Performance Primitives-based code [no]
@@ -1529,6 +1530,7 @@  EXTERNAL_LIBRARY_LIST="
     $EXTERNAL_LIBRARY_GPLV3_LIST
     chromaprint
     crystalhd
+    drm
     gcrypt
     gnutls
     jni
@@ -5783,6 +5785,7 @@  enabled coreimage_filter  && { check_header_objcc QuartzCore/CoreImage.h || disa
 enabled coreimagesrc_filter && { check_header_objcc QuartzCore/CoreImage.h || disable coreimagesrc_filter; }
 enabled decklink          && { { check_header DeckLinkAPI.h || die "ERROR: DeckLinkAPI.h header not found"; } &&
                                { check_cpp_condition DeckLinkAPIVersion.h "BLACKMAGIC_DECKLINK_API_VERSION >= 0x0a060100" || die "ERROR: Decklink API version must be >= 10.6.1."; } }
+enabled drm               && require_pkg_config libdrm xf86drm.h drmGetVersion
 enabled frei0r            && { check_header frei0r.h || die "ERROR: frei0r.h header not found"; }
 enabled gmp               && require gmp gmp.h mpz_export -lgmp
 enabled gnutls            && require_pkg_config gnutls gnutls/gnutls.h gnutls_global_init
diff --git a/libavutil/Makefile b/libavutil/Makefile
index 4fe81fdd07..2e15e7d207 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -156,6 +156,7 @@  OBJS = adler32.o                                                        \
 OBJS-$(!HAVE_ATOMICS_NATIVE)            += atomic.o                     \
 
 OBJS-$(CONFIG_CUDA)                     += hwcontext_cuda.o
+OBJS-$(CONFIG_DRM)                      += hwcontext_drm.o
 OBJS-$(CONFIG_DXVA2)                    += hwcontext_dxva2.o
 OBJS-$(CONFIG_QSV)                   += hwcontext_qsv.o
 OBJS-$(CONFIG_LZO)                      += lzo.o
@@ -171,6 +172,7 @@  SLIBOBJS-$(HAVE_GNU_WINDRES)            += avutilres.o
 
 SKIPHEADERS-$(HAVE_CUDA_H)             += hwcontext_cuda.h
 SKIPHEADERS-$(CONFIG_CUDA)             += hwcontext_cuda_internal.h
+SKIPHEADERS-$(CONFIG_DRM)              += hwcontext_drm.h
 SKIPHEADERS-$(CONFIG_DXVA2)            += hwcontext_dxva2.h
 SKIPHEADERS-$(CONFIG_QSV)           += hwcontext_qsv.h
 SKIPHEADERS-$(CONFIG_VAAPI)            += hwcontext_vaapi.h
diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index 5a22194716..d58f89d2eb 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -32,6 +32,9 @@  static const HWContextType *const hw_table[] = {
 #if CONFIG_CUDA
     &ff_hwcontext_type_cuda,
 #endif
+#if CONFIG_DRM
+    &ff_hwcontext_type_drm,
+#endif
 #if CONFIG_DXVA2
     &ff_hwcontext_type_dxva2,
 #endif
@@ -52,6 +55,7 @@  static const HWContextType *const hw_table[] = {
 
 static const char *const hw_type_names[] = {
     [AV_HWDEVICE_TYPE_CUDA]   = "cuda",
+    [AV_HWDEVICE_TYPE_DRM]    = "drm",
     [AV_HWDEVICE_TYPE_DXVA2]  = "dxva2",
     [AV_HWDEVICE_TYPE_QSV]    = "qsv",
     [AV_HWDEVICE_TYPE_VAAPI]  = "vaapi",
diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
index edf12cc631..fe7613b379 100644
--- a/libavutil/hwcontext.h
+++ b/libavutil/hwcontext.h
@@ -32,6 +32,7 @@  enum AVHWDeviceType {
     AV_HWDEVICE_TYPE_QSV,
     AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
     AV_HWDEVICE_TYPE_NONE,
+    AV_HWDEVICE_TYPE_DRM,
 };
 
 typedef struct AVHWDeviceInternal AVHWDeviceInternal;
diff --git a/libavutil/hwcontext_drm.c b/libavutil/hwcontext_drm.c
new file mode 100644
index 0000000000..0a355568ed
--- /dev/null
+++ b/libavutil/hwcontext_drm.c
@@ -0,0 +1,280 @@ 
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#if HAVE_UNISTD_H
+#   include <unistd.h>
+#endif
+
+#include <drm.h>
+#include <xf86drm.h>
+
+#include "hwcontext.h"
+#include "hwcontext_drm.h"
+#include "hwcontext_internal.h"
+#include "imgutils.h"
+
+
+static void drm_device_free(AVHWDeviceContext *hwdev)
+{
+    AVDRMDeviceContext *hwctx = hwdev->hwctx;
+
+    close(hwctx->fd);
+}
+
+static int drm_device_create(AVHWDeviceContext *hwdev, const char *device,
+                             AVDictionary *opts, int flags)
+{
+    AVDRMDeviceContext *hwctx = hwdev->hwctx;
+    drmVersionPtr version;
+
+    hwctx->fd = open(device, O_RDWR);
+    if (hwctx->fd < 0)
+        return AVERROR_UNKNOWN;
+
+    version = drmGetVersion(hwctx->fd);
+    if (!version) {
+        av_log(hwdev, AV_LOG_ERROR, "Failed to DRM version information "
+               "from %s: probably not a DRM device?\n", device);
+        close(hwctx->fd);
+        return AVERROR(EINVAL);
+    }
+
+    av_log(hwdev, AV_LOG_VERBOSE, "Opened DRM device %s: driver %s "
+           "version %d.%d.%d.\n", device, version->name,
+           version->version_major, version->version_minor,
+           version->version_patchlevel);
+
+    drmFreeVersion(version);
+
+    hwdev->free = &drm_device_free;
+
+    return 0;
+}
+
+static int drm_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+    frame->buf[0] = av_buffer_pool_get(hwfc->pool);
+    if (!frame->buf[0])
+        return AVERROR(ENOMEM);
+
+    frame->data[0] = (uint8_t*)frame->buf[0]->data;
+
+    frame->format = AV_PIX_FMT_DRM_PRIME;
+    frame->width  = hwfc->width;
+    frame->height = hwfc->height;
+
+    return 0;
+}
+
+typedef struct DRMMapping {
+    // Address and length of each mmap()ed region.
+    void *address[AV_NUM_DATA_POINTERS];
+    size_t length[AV_NUM_DATA_POINTERS];
+} DRMMapping;
+
+static void drm_unmap_frame(AVHWFramesContext *hwfc,
+                            HWMapDescriptor *hwmap)
+{
+    DRMMapping *map = hwmap->priv;
+    int plane;
+
+    for (plane = 0; plane < FF_ARRAY_ELEMS(map->address); plane++) {
+        if (map->address[plane])
+            munmap(map->address[plane], map->length[plane]);
+    }
+
+    av_free(map);
+}
+
+static int drm_map_frame(AVHWFramesContext *hwfc,
+                         AVFrame *dst, const AVFrame *src, int flags)
+{
+    const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor*)src->data[0];
+    DRMMapping *map;
+    int err, i, p;
+    int mmap_prot;
+    void *addr;
+
+    map = av_mallocz(sizeof(*map));
+    if (!map)
+        return AVERROR(ENOMEM);
+
+    mmap_prot = 0;
+    if (flags & AV_HWFRAME_MAP_READ)
+        mmap_prot |= PROT_READ;
+    if (flags & AV_HWFRAME_MAP_WRITE)
+        mmap_prot |= PROT_WRITE;
+
+    for (i = 0; i < desc->nb_objects; i++) {
+        addr = mmap(NULL, desc->objects[i].size, mmap_prot, MAP_SHARED,
+                    desc->objects[i].fd, 0);
+        if (addr == MAP_FAILED) {
+            err = AVERROR(errno);
+            av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to "
+                   "memory: %d.\n", desc->objects[i].fd, errno);
+            // TODO: cleanup.
+            goto fail;
+        }
+
+        map->address[i] = addr;
+        map->length[i]  = desc->objects[i].size;
+    }
+
+    for (p = 0; p < desc->nb_planes; p++) {
+        dst->data[p] = (uint8_t*)map->address[desc->planes[p].object_index] +
+                           desc->planes[p].offset;
+        dst->linesize[p] = desc->planes[p].pitch;
+    }
+
+    dst->width  = src->width;
+    dst->height = src->height;
+
+    err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
+                                &drm_unmap_frame, map);
+    if (err < 0)
+        return err;
+
+    return 0;
+
+fail:
+    av_free(map);
+    return err;
+}
+
+static int drm_transfer_get_formats(AVHWFramesContext *ctx,
+                                    enum AVHWFrameTransferDirection dir,
+                                    enum AVPixelFormat **formats)
+{
+    enum AVPixelFormat *pix_fmts;
+
+    pix_fmts = av_malloc_array(2, sizeof(*pix_fmts));
+    if (!pix_fmts)
+        return AVERROR(ENOMEM);
+
+    pix_fmts[0] = ctx->sw_format;
+    pix_fmts[1] = AV_PIX_FMT_NONE;
+
+    *formats = pix_fmts;
+    return 0;
+}
+
+static int drm_transfer_data_from(AVHWFramesContext *hwfc,
+                                  AVFrame *dst, const AVFrame *src)
+{
+    AVFrame *map;
+    int err;
+
+    if (dst->width > hwfc->width || dst->height > hwfc->height)
+        return AVERROR(EINVAL);
+
+    map = av_frame_alloc();
+    if (!map)
+        return AVERROR(ENOMEM);
+    map->format = dst->format;
+
+    err = drm_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ);
+    if (err)
+        goto fail;
+
+    map->width  = dst->width;
+    map->height = dst->height;
+
+    err = av_frame_copy(dst, map);
+    if (err)
+        goto fail;
+
+    err = 0;
+fail:
+    av_frame_free(&map);
+    return err;
+}
+
+static int drm_transfer_data_to(AVHWFramesContext *hwfc,
+                                AVFrame *dst, const AVFrame *src)
+{
+    AVFrame *map;
+    int err;
+
+    if (src->width > hwfc->width || src->height > hwfc->height)
+        return AVERROR(EINVAL);
+
+    map = av_frame_alloc();
+    if (!map)
+        return AVERROR(ENOMEM);
+    map->format = src->format;
+
+    err = drm_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE);
+    if (err)
+        goto fail;
+
+    map->width  = src->width;
+    map->height = src->height;
+
+    err = av_frame_copy(map, src);
+    if (err)
+        goto fail;
+
+    err = 0;
+fail:
+    av_frame_free(&map);
+    return err;
+}
+
+static int drm_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
+                        const AVFrame *src, int flags)
+{
+    int err;
+
+    if (hwfc->sw_format != dst->format)
+        return AVERROR(ENOSYS);
+
+    err = drm_map_frame(hwfc, dst, src, flags);
+    if (err)
+        return err;
+
+    err = av_frame_copy_props(dst, src);
+    if (err)
+        return err;
+
+    return 0;
+}
+
+const HWContextType ff_hwcontext_type_drm = {
+    .type                   = AV_HWDEVICE_TYPE_DRM,
+    .name                   = "DRM",
+
+    .device_hwctx_size      = sizeof(AVDRMDeviceContext),
+
+    .device_create          = &drm_device_create,
+
+    .frames_get_buffer      = &drm_get_buffer,
+
+    .transfer_get_formats   = &drm_transfer_get_formats,
+    .transfer_data_to       = &drm_transfer_data_to,
+    .transfer_data_from     = &drm_transfer_data_from,
+    .map_from               = &drm_map_from,
+
+    .pix_fmts = (const enum AVPixelFormat[]) {
+        AV_PIX_FMT_DRM_PRIME,
+        AV_PIX_FMT_NONE
+    },
+};
diff --git a/libavutil/hwcontext_drm.h b/libavutil/hwcontext_drm.h
new file mode 100644
index 0000000000..62f8d882bc
--- /dev/null
+++ b/libavutil/hwcontext_drm.h
@@ -0,0 +1,145 @@ 
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_HWCONTEXT_DRM_H
+#define AVUTIL_HWCONTEXT_DRM_H
+
+#include "frame.h"
+
+/**
+ * @file
+ * API-specific header for AV_HWDEVICE_TYPE_DRM.
+ *
+ * Internal frame allocation is not currently supported - all frames
+ * must be allocated by the user.  Thus AVHWFramesContext is always
+ * NULL, though this may change if support for frame allocation is
+ * added in future.
+ */
+
+
+/**
+ * DRM object descriptor.
+ *
+ * Describes a single DRM object, addressing it as a PRIME file
+ * descriptor.
+ */
+typedef struct AVDRMObjectDescriptor {
+    /**
+     * DRM PRIME fd for the object.
+     */
+    int fd;
+    /**
+     * Format modifier applied to the object (DRM_FORMAT_MOD_*).
+     */
+    uint64_t format_modifier;
+    /**
+     * Total size of the object.
+     *
+     * (This includes any parts not which do not contain image data.)
+     */
+    size_t size;
+} AVDRMObjectDescriptor;
+
+/**
+ * DRM plane descriptor.
+ *
+ * Describes a single plane of an image, which is contained within
+ * single object.
+ */
+typedef struct AVDRMPlaneDescriptor {
+    /**
+     * Format of the plane (DRM_FORMAT_*).
+     *
+     * Note that this can be a multiple-plane format such as NV12:
+     * if so, this structure is only describes one plane of the
+     * image.
+     */
+    uint32_t format;
+    /**
+     * Index of this plane in within the current image.
+     *
+     * For single plane formats, this is always zero.
+     */
+    int plane_index;
+    /**
+     * Index of the object containing this plane in the objects
+     * array of the enclosing frame descriptor.
+     */
+    int object_index;
+    /**
+     * Offset within that object of this plane.
+     */
+    ptrdiff_t offset;
+    /**
+     * Pitch (linesize) of this plane.
+     */
+    ptrdiff_t pitch;
+} AVDRMPlaneDescriptor;
+
+/**
+ * DRM frame descriptor.
+ *
+ * This is used as the data pointer for AV_PIX_FMT_DRM_PRIME frames.
+ * It is also used by user-allocated frame pools - allocating in
+ * AVHWFramesContext.pool must return AVBufferRefs which contain
+ * an object of this type.
+ *
+ * The fields of this structure should be set such it can be
+ * imported directly by EGL using the EGL_EXT_image_dma_buf_import
+ * and EGL_EXT_image_dma_buf_import_modifiers extensions.
+ * (Note that the exact layout of a particular format may vary between
+ * platforms - we only specify that the same platform should be able
+ * to import it.)
+ */
+typedef struct AVDRMFrameDescriptor {
+    /**
+     * Number of DRM objects making up this frame.
+     */
+    int nb_objects;
+    /**
+     * Array of objects making up the frame.
+     */
+    AVDRMObjectDescriptor objects[AV_NUM_DATA_POINTERS];
+    /**
+     * Number of planes in the frame.
+     */
+    int nb_planes;
+    /**
+     * Array of planes in the frame.
+     */
+    AVDRMPlaneDescriptor planes[AV_NUM_DATA_POINTERS];
+} AVDRMFrameDescriptor;
+
+/**
+ * DRM device.
+ *
+ * Allocated as AVHWDeviceContext.hwctx.
+ */
+typedef struct AVDRMDeviceContext {
+    /**
+     * File descriptor of DRM device.
+     *
+     * This is used as the device to create frames on, and may also be
+     * used in some derivation and mapping operations.
+     *
+     * If no device is required, set to -1.
+     */
+    int fd;
+} AVDRMDeviceContext;
+
+#endif /* AVUTIL_HWCONTEXT_DRM_H */
diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
index 68f78c0a1f..314c3656a7 100644
--- a/libavutil/hwcontext_internal.h
+++ b/libavutil/hwcontext_internal.h
@@ -158,6 +158,7 @@  int ff_hwframe_map_create(AVBufferRef *hwframe_ref,
 
 
 extern const HWContextType ff_hwcontext_type_cuda;
+extern const HWContextType ff_hwcontext_type_drm;
 extern const HWContextType ff_hwcontext_type_dxva2;
 extern const HWContextType ff_hwcontext_type_qsv;
 extern const HWContextType ff_hwcontext_type_vaapi;
diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index d4a7a8ba3b..bc1142d571 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -2158,6 +2158,10 @@  static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = {
         .flags = AV_PIX_FMT_FLAG_BE | AV_PIX_FMT_FLAG_PLANAR |
                  AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_ALPHA,
     },
+    [AV_PIX_FMT_DRM_PRIME] = {
+        .name = "drm_prime",
+        .flags = AV_PIX_FMT_FLAG_HWACCEL,
+    },
 };
 #if FF_API_PLUS1_MINUS1
 FF_ENABLE_DEPRECATION_WARNINGS
diff --git a/libavutil/pixfmt.h b/libavutil/pixfmt.h
index 6b7eea8c4e..5a6a732061 100644
--- a/libavutil/pixfmt.h
+++ b/libavutil/pixfmt.h
@@ -314,6 +314,10 @@  enum AVPixelFormat {
     AV_PIX_FMT_P016LE, ///< like NV12, with 16bpp per component, little-endian
     AV_PIX_FMT_P016BE, ///< like NV12, with 16bpp per component, big-endian
 
+    // DRM-managed buffers exposed through PRIME buffer sharing.
+    // data[0] points to an AVDRMFrameDescriptor.
+    AV_PIX_FMT_DRM_PRIME,
+
     AV_PIX_FMT_NB         ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions
 };