diff mbox series

[FFmpeg-devel,3/4] avformat/mov: add referenced thumbnail streams to tile stream groups

Message ID 20240925225219.3060-3-jamrial@gmail.com
State New
Headers show
Series [FFmpeg-devel,1/4] avformat/avformat: add a new disposition to signal the stream is an HDR gainmap | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

James Almer Sept. 25, 2024, 10:52 p.m. UTC
Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavformat/dump.c |  9 ++++--
 libavformat/isom.h |  3 +-
 libavformat/mov.c  | 71 +++++++++++++++++++++++++++++++++++-----------
 3 files changed, 64 insertions(+), 19 deletions(-)

Comments

Anton Khirnov Sept. 26, 2024, 6:01 a.m. UTC | #1
> Subject: avformat/mov: add referenced thumbnail streams to tile stream groups

This could use some more explanation.
Anton Khirnov Sept. 26, 2024, 2:43 p.m. UTC | #2
Quoting James Almer (2024-09-26 16:33:48)
> Use the reference information present in iref boxes of type thmb to include the
> relevant streams into a Tile Grid stream group.
> This does not yet export the relation of a thumbnail and an independent stream
> (not a grid). For this, a new Stream group type would probably be needed.

So the stream group describing the tile grid would now also contain a
single stream that is a thumbnail rather than a tile in the grid? That's
not something I would expect.
diff mbox series

Patch

diff --git a/libavformat/dump.c b/libavformat/dump.c
index 5178f19685..ba30b92aaf 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -784,11 +784,16 @@  static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed,
         dump_disposition(stg->disposition, AV_LOG_INFO);
         av_log(NULL, AV_LOG_INFO, "\n");
         dump_metadata(NULL, stg->metadata, "    ", AV_LOG_INFO);
-        for (int i = 0; i < stg->nb_streams; i++) {
-            const AVStream *st = stg->streams[i];
+        for (int i = 0; i < tile_grid->nb_tiles; i++) {
+            const AVStream *st = stg->streams[tile_grid->offsets[i].idx];
             dump_stream_format(ic, st->index, i, index, is_output, AV_LOG_VERBOSE);
             printed[st->index] = 1;
         }
+        for (int i = 0; i < stg->nb_streams; i++) {
+            const AVStream *st = stg->streams[i];
+            if (!printed[st->index])
+                dump_stream_format(ic, st->index, i, index, is_output, AV_LOG_VERBOSE);
+        }
         break;
     }
     case AV_STREAM_GROUP_PARAMS_LCEVC: {
diff --git a/libavformat/isom.h b/libavformat/isom.h
index 5076bc5da7..1cf69ed042 100644
--- a/libavformat/isom.h
+++ b/libavformat/isom.h
@@ -279,6 +279,8 @@  typedef struct HEIFItem {
     AVStream *st;
     char *name;
     int item_id;
+    struct HEIFItem **ref_item_list;
+    int nb_ref_item_list;
     int64_t extent_length;
     int64_t extent_offset;
     int width;
@@ -360,7 +362,6 @@  typedef struct MOVContext {
     int nb_heif_item;
     HEIFGrid *heif_grid;
     int nb_heif_grid;
-    int thmb_item_id;
     int64_t idat_offset;
     int interleaved_read;
 } MOVContext;
diff --git a/libavformat/mov.c b/libavformat/mov.c
index bd502d489a..8a257ba535 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -8844,23 +8844,55 @@  static int mov_read_iref_dimg(MOVContext *c, AVIOContext *pb, int version)
     return 0;
 }
 
+static int mov_add_ref_to_item(MOVContext *c, AVIOContext *pb, int version,
+                               HEIFItem *from_item)
+{
+    HEIFItem **ref_item_list, *to_item = NULL;
+    int to_item_id = version ? avio_rb32(pb) : avio_rb16(pb);
+
+    for (int j = 0; j < c->nb_heif_item; j++) {
+        if (c->heif_item[j].item_id != to_item_id)
+            continue;
+        to_item = &c->heif_item[j];
+    }
+    if (!to_item) {
+        av_log(c->fc, AV_LOG_ERROR, "thmb in iref references a non-existent item\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    ref_item_list = av_realloc_array(to_item->ref_item_list, to_item->nb_ref_item_list + 1U,
+                                     sizeof(*to_item->ref_item_list));
+    if (!ref_item_list)
+        return AVERROR(ENOMEM);
+    to_item->ref_item_list = ref_item_list;
+    to_item->ref_item_list[to_item->nb_ref_item_list++] = from_item;
+
+    return 0;
+}
+
 static int mov_read_iref_thmb(MOVContext *c, AVIOContext *pb, int version)
 {
+    HEIFItem *from_item = NULL;
     int entries;
-    int to_item_id, from_item_id = version ? avio_rb32(pb) : avio_rb16(pb);
+    int from_item_id = version ? avio_rb32(pb) : avio_rb16(pb);
 
-    entries = avio_rb16(pb);
-    if (entries > 1) {
-        avpriv_request_sample(c->fc, "thmb in iref referencing several items");
-        return AVERROR_PATCHWELCOME;
+    for (int i = 0; i < c->nb_heif_item; i++) {
+        if (c->heif_item[i].item_id != from_item_id)
+            continue;
+        from_item = &c->heif_item[i];
+    }
+    if (!from_item) {
+        av_log(c->fc, AV_LOG_ERROR, "thmb in iref references a non-existent item\n");
+        return AVERROR_INVALIDDATA;
     }
-    /* 'to' item ids */
-    to_item_id = version ? avio_rb32(pb) : avio_rb16(pb);
-
-    if (to_item_id != c->primary_item_id)
-        return 0;
 
-    c->thmb_item_id = from_item_id;
+    entries = avio_rb16(pb);
+    /* 'to' item ids */
+    for (int i = 0; i < entries; i++) {
+        int ret = mov_add_ref_to_item(c, pb, version, from_item);
+        if (ret < 0)
+            return ret;
+    }
 
     av_log(c->fc, AV_LOG_TRACE, "thmb: from_item_id %d, entries %d\n",
            from_item_id, entries);
@@ -9718,6 +9750,7 @@  static int mov_read_close(AVFormatContext *s)
     for (i = 0; i < mov->nb_heif_item; i++) {
         av_freep(&mov->heif_item[i].name);
         av_freep(&mov->heif_item[i].aux_type);
+        av_freep(&mov->heif_item[i].ref_item_list);
     }
     av_freep(&mov->heif_item);
     for (i = 0; i < mov->nb_heif_grid; i++) {
@@ -10074,6 +10107,17 @@  static int mov_parse_tiles(AVFormatContext *s)
         if (!loop)
             continue;
 
+        for (int j = 0; j < grid->item->nb_ref_item_list; j++) {
+            AVStream *st = grid->item->ref_item_list[j]->st;
+
+            if (!st)
+                continue;
+
+            err = avformat_stream_group_add_stream(stg, st);
+            if (err < 0 && err != AVERROR(EEXIST))
+                return err;
+        }
+
         switch (grid->item->type) {
         case MKTAG('g','r','i','d'):
             err = read_image_grid(s, grid, tile_grid);
@@ -10128,7 +10172,6 @@  static int mov_read_header(AVFormatContext *s)
 
     mov->fc = s;
     mov->trak_index = -1;
-    mov->thmb_item_id = -1;
     mov->primary_item_id = -1;
     mov->cur_item_id = -1;
     /* .mov and .mp4 aren't streamable anyway (only progressive download if moov is before mdat) */
@@ -10161,10 +10204,6 @@  static int mov_read_header(AVFormatContext *s)
             int64_t offset = 0;
 
             if (!item->st) {
-                if (item->item_id == mov->thmb_item_id) {
-                    av_log(s, AV_LOG_ERROR, "HEIF thumbnail doesn't reference a stream\n");
-                    return AVERROR_INVALIDDATA;
-                }
                 continue;
             }
             if (item->is_idat_relative) {