[FFmpeg-devel,2/3] avformat/mov: add support for tile HEIF still images

diff --git a/libavcodec/packet.h b/libavcodec/packet.h
index 2c57d262c6..48ca799334 100644
--- a/libavcodec/packet.h
+++ b/libavcodec/packet.h
@@ -323,6 +323,15 @@  enum AVPacketSideDataType {
      */
     AV_PKT_DATA_IAMF_RECON_GAIN_INFO_PARAM,
 
+    /**
+     * Tile info for image reconstruction, e.g HEIF.
+     * @code
+     * u32le tile number in row major order [0..nb_tiles-1]
+     * u32le nb_tiles
+     * @endcode
+     */
+    AV_PKT_DATA_TILE_INFO,
+
     /**
      * The number of side data types.
      * This is not part of the public API/ABI in the sense that it may
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index ab9a3fc6be..cf4e72e11d 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -811,6 +811,12 @@  typedef struct AVIndexEntry {
  * The video stream contains still images.
  */
 #define AV_DISPOSITION_STILL_IMAGE          (1 << 20)
+/**
+ * The video stream is intended to be merged with another stream before
+ * presentation.
+ * Used for example to signal the stream contains a tile from a HEIF grid.
+ */
+#define AV_DISPOSITION_TILE                 (1 << 21)
 
 /**
  * @return The AV_DISPOSITION_* flag corresponding to disp or a negative error
diff --git a/libavformat/isom.h b/libavformat/isom.h
index 2cf456fee1..cd5478b61c 100644
--- a/libavformat/isom.h
+++ b/libavformat/isom.h
@@ -267,10 +267,10 @@  typedef struct HEIFItem {
     int item_id;
     int64_t extent_length;
     int64_t extent_offset;
-    int64_t size;
     int width;
     int height;
     int type;
+    int is_idat_relative;
 } HEIFItem;
 
 typedef struct MOVContext {
@@ -335,6 +335,12 @@  typedef struct MOVContext {
     int cur_item_id;
     HEIFItem *heif_info;
     int heif_info_size;
+    int grid_item_id;
+    int thmb_item_id;
+    int16_t *tile_id_list;
+    int nb_tiles;
+    uint8_t *idat_buf;
+    int64_t idat_size;
     int interleaved_read;
 } MOVContext;
 
diff --git a/libavformat/mov.c b/libavformat/mov.c
index 5cb907e120..5e253877b8 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -184,6 +184,30 @@  static int mov_read_mac_string(MOVContext *c, AVIOContext *pb, int len,
     return p - dst;
 }
 
+static AVStream *get_curr_st(MOVContext *c)
+{
+    AVStream *st = NULL;
+
+    if (c->fc->nb_streams < 1)
+        return NULL;
+
+    for (int i = 0; i < c->heif_info_size; i++) {
+        HEIFItem *item = &c->heif_info[i];
+
+        if (!item->st)
+            continue;
+        if (item->st->id != c->cur_item_id)
+            continue;
+
+        st = item->st;
+        break;
+    }
+    if (!st)
+        st = c->fc->streams[c->fc->nb_streams-1];
+
+    return st;
+}
+
 static int mov_read_covr(MOVContext *c, AVIOContext *pb, int type, int len)
 {
     AVStream *st;
@@ -1766,9 +1790,9 @@  static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     uint16_t color_primaries, color_trc, color_matrix;
     int ret;
 
-    if (c->fc->nb_streams < 1)
+    st = get_curr_st(c);
+    if (!st)
         return 0;
-    st = c->fc->streams[c->fc->nb_streams - 1];
 
     ret = ffio_read_size(pb, color_parameter_type, 4);
     if (ret < 0)
@@ -2116,9 +2140,9 @@  static int mov_read_glbl(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     AVStream *st;
     int ret;
 
-    if (c->fc->nb_streams < 1)
+    st = get_curr_st(c);
+    if (!st)
         return 0;
-    st = c->fc->streams[c->fc->nb_streams-1];
 
     if ((uint64_t)atom.size > (1<<30))
         return AVERROR_INVALIDDATA;
@@ -4928,12 +4952,10 @@  static int heif_add_stream(MOVContext *c, HEIFItem *item)
     st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codecpar->codec_id = mov_codec_id(st, item->type);
     sc->ffindex = st->index;
-    c->trak_index = st->index;
     st->avg_frame_rate.num = st->avg_frame_rate.den = 1;
     st->time_base.num = st->time_base.den = 1;
     st->nb_frames = 1;
     sc->time_scale = 1;
-    sc = st->priv_data;
     sc->pb = c->fc->pb;
     sc->pb_is_copied = 1;
 
@@ -7761,15 +7783,60 @@  static int mov_read_pitm(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     return atom.size;
 }
 
+static int mov_read_idat(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    av_log(c->fc, AV_LOG_TRACE, "idat: size %"PRId64"\n", atom.size);
+
+    c->idat_buf = av_malloc(atom.size);
+    if (!c->idat_buf)
+        return AVERROR(ENOMEM);
+
+    if (avio_read(pb, c->idat_buf, atom.size) != atom.size) {
+        av_freep(&c->idat_buf);
+        return AVERROR_INVALIDDATA;
+    }
+    c->idat_size = atom.size;
+
+    return 0;
+}
+
+static int read_image_grid(MOVContext *c, AVTileGrid *tile_grid) {
+    FFIOContext b;
+    AVIOContext *pb = &b.pub;
+    uint8_t flags;
+
+    ffio_init_read_context(&b, c->idat_buf, c->idat_size);
+    avio_r8(pb);    /* version */
+    flags = avio_r8(pb);
+
+    tile_grid->tile_rows = avio_r8(pb) + 1;
+    tile_grid->tile_cols = avio_r8(pb) + 1;
+    /* actual width and height of output image */
+    tile_grid->output_width  = (flags & 1) ? avio_rb32(pb) : avio_rb16(pb);
+    tile_grid->output_height = (flags & 1) ? avio_rb32(pb) : avio_rb16(pb);
+
+    tile_grid->tile_width  = av_calloc(tile_grid->tile_rows * tile_grid->tile_cols,
+                                       sizeof(*tile_grid->tile_width));
+    tile_grid->tile_height = av_calloc(tile_grid->tile_rows * tile_grid->tile_cols,
+                                       sizeof(*tile_grid->tile_height));
+    if (!tile_grid->tile_width || !tile_grid->tile_width)
+        return AVERROR(ENOMEM);
+
+    av_log(c->fc, AV_LOG_TRACE, "grid: grid_rows %d grid_cols %d output_width %d output_height %d\n",
+           tile_grid->tile_rows, tile_grid->tile_cols, tile_grid->output_width, tile_grid->output_height);
+
+    return 0;
+}
+
 static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     int version, offset_size, length_size, base_offset_size, index_size;
     int item_count, extent_count;
-    uint64_t base_offset, extent_offset, extent_length;
+    int64_t base_offset, extent_offset, extent_length;
     uint8_t value;
 
     if (c->found_iloc) {
-        av_log(c->fc, AV_LOG_INFO, "Duplicate iloc box found\n");
+        av_log(c->fc, AV_LOG_WARNING, "Duplicate iloc box found\n");
         return 0;
     }
 
@@ -7811,6 +7878,7 @@  static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         avio_rb16(pb);  // data_reference_index.
         if (rb_size(pb, &base_offset, base_offset_size) < 0)
             return AVERROR_INVALIDDATA;
+        av_log(c->fc, AV_LOG_TRACE, "iloc: base_offset %"PRId64"\n", base_offset);
         extent_count = avio_rb16(pb);
         if (extent_count > 1) {
             // For still AVIF images, we only support one extent item.
@@ -7821,6 +7889,8 @@  static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
             if (rb_size(pb, &extent_offset, offset_size) < 0 ||
                 rb_size(pb, &extent_length, length_size) < 0)
                 return AVERROR_INVALIDDATA;
+            if (offset_type == 1)
+                c->heif_info[i].is_idat_relative = 1;
             c->heif_info[i].extent_length = extent_length;
             c->heif_info[i].extent_offset = base_offset + extent_offset;
             av_log(c->fc, AV_LOG_TRACE, "iloc: item_idx %d, offset_type %d, "
@@ -7859,10 +7929,6 @@  static int mov_read_infe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     av_log(c->fc, AV_LOG_TRACE, "infe: item_id %d, item_type %s, item_name %s\n",
            item_id, av_fourcc2str(item_type), item_name);
 
-    // Skip all but the primary item until support is added
-    if (item_id != c->primary_item_id)
-        return 0;
-
     if (size > 0)
         avio_skip(pb, size);
 
@@ -7876,6 +7942,9 @@  static int mov_read_infe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         if (ret < 0)
             return ret;
         break;
+    case MKTAG('g','r','i','d'):
+        c->grid_item_id = item_id;
+        break;
     default:
         av_log(c->fc, AV_LOG_TRACE, "infe: ignoring item_type %s\n", av_fourcc2str(item_type));
         break;
@@ -7924,6 +7993,59 @@  static int mov_read_iref(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     return mov_read_default(c, pb, atom);
 }
 
+static int mov_read_dimg(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    int entries, i;
+    int from_item_id = avio_rb16(pb);
+
+    if (c->grid_item_id < 0) {
+        av_log(c->fc, AV_LOG_ERROR, "Missing grid information\n");
+        return AVERROR_INVALIDDATA;
+    }
+    if (from_item_id != c->grid_item_id) {
+        avpriv_request_sample(c->fc, "Derived item of type other than 'grid'");
+        return AVERROR_PATCHWELCOME;
+    }
+    entries = avio_rb16(pb);
+    c->tile_id_list = av_malloc_array(entries, sizeof(*c->tile_id_list));
+    if (!c->tile_id_list)
+        return AVERROR(ENOMEM);
+    /* 'to' item ids */
+    for (i = 0; i < entries; i++)
+        c->tile_id_list[i] = avio_rb16(pb);
+    c->nb_tiles = entries;
+
+    av_log(c->fc, AV_LOG_TRACE, "dimg: from_item_id %d, entries %d\n",
+           from_item_id, entries);
+
+    return 0;
+}
+
+static int mov_read_thmb(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+    int entries;
+    int to_item_id, from_item_id = avio_rb16(pb);
+
+    entries = avio_rb16(pb);
+    if (entries > 1) {
+        avpriv_request_sample(c->fc, "More than one thmb entry");
+        return AVERROR_PATCHWELCOME;
+    }
+    /* 'to' item ids */
+    to_item_id = avio_rb16(pb);
+
+    if (to_item_id != c->primary_item_id ||
+        to_item_id != c->grid_item_id)
+        return 0;
+
+    c->thmb_item_id = from_item_id;
+
+    av_log(c->fc, AV_LOG_TRACE, "thmb: from_item_id %d, entries %d\n",
+           from_item_id, entries);
+
+    return 0;
+}
+
 static int mov_read_ispe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     uint32_t width, height;
@@ -7932,15 +8054,17 @@  static int mov_read_ispe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     width  = avio_rb32(pb);
     height = avio_rb32(pb);
 
-    av_log(c->fc, AV_LOG_TRACE, "ispe: item_id %d, width %u, height %u\n",
+    av_log(c->fc, AV_LOG_TRACE, "ispe: cur_item_id %d, width %u, height %u\n",
            c->cur_item_id, width, height);
 
     for (int i = 0; i < c->heif_info_size; i++) {
-        if (c->heif_info[i].item_id == c->cur_item_id) {
-            c->heif_info[i].width  = width;
-            c->heif_info[i].height = height;
-            break;
-        }
+        HEIFItem *item = &c->heif_info[i];
+        if (item->item_id != c->cur_item_id)
+            continue;
+
+        item->width  = width;
+        item->height = height;
+        break;
     }
 
     return 0;
@@ -8031,10 +8155,6 @@  static int mov_read_iprp(MOVContext *c, AVIOContext *pb, MOVAtom atom)
             av_log(c->fc, AV_LOG_TRACE, "ipma: property_index %d, item_id %d, item_type %s\n",
                    index + 1, item_id, av_fourcc2str(ref->type));
 
-            // Skip properties referencing items other than the primary item until support is added
-            if (item_id != c->primary_item_id)
-                continue;
-
             c->cur_item_id = item_id;
 
             ret = mov_read_default(c, &ref->b.pub,
@@ -8162,6 +8282,9 @@  static const MOVParseTableEntry mov_default_parse_table[] = {
 { MKTAG('p','c','m','C'), mov_read_pcmc }, /* PCM configuration box */
 { MKTAG('p','i','t','m'), mov_read_pitm },
 { MKTAG('e','v','c','C'), mov_read_glbl },
+{ MKTAG('d','i','m','g'), mov_read_dimg },
+{ MKTAG('t','h','m','b'), mov_read_thmb },
+{ MKTAG('i','d','a','t'), mov_read_idat },
 { MKTAG('i','r','e','f'), mov_read_iref },
 { MKTAG('i','s','p','e'), mov_read_ispe },
 { MKTAG('i','p','r','p'), mov_read_iprp },
@@ -8664,6 +8787,8 @@  static int mov_read_close(AVFormatContext *s)
     av_freep(&mov->aes_decrypt);
     av_freep(&mov->chapter_tracks);
     av_freep(&mov->heif_info);
+    av_freep(&mov->idat_buf);
+    av_freep(&mov->tile_id_list);
 
     return 0;
 }
@@ -8803,6 +8928,109 @@  fail:
     return ret;
 }
 
+static int mov_parse_tiles(AVFormatContext *s)
+{
+    MOVContext *mov = s->priv_data;
+    AVStreamGroup *stg = avformat_stream_group_create(s, AV_STREAM_GROUP_PARAMS_TILE_GRID, NULL);
+    AVTileGrid *tile_grid;
+    int err;
+
+    if (!stg)
+        return AVERROR(ENOMEM);
+
+    tile_grid = stg->params.tile_grid;
+
+    av_assert0(mov->grid_item_id >= 0);
+    for (int i = 0; i < mov->heif_info_size; i++) {
+        HEIFItem *item = &mov->heif_info[i];
+
+        if (item->item_id != mov->grid_item_id)
+            continue;
+        if (!item->is_idat_relative)
+            return AVERROR_PATCHWELCOME;
+        err = read_image_grid(mov, tile_grid);
+        if (err < 0)
+            return err;
+        stg->id = item->item_id;
+        break;
+    }
+
+    for (int i = 0; i < mov->nb_tiles; i++) {
+        int tile_id = mov->tile_id_list[i];
+
+        for (int j = 0; j < mov->heif_info_size; j++) {
+            HEIFItem *item = &mov->heif_info[j];
+            AVStream *st = item->st;
+            AVPacketSideData *sd;
+            MOVStreamContext *sc;
+
+            if (item->item_id != tile_id)
+                continue;
+            if (!st) {
+                av_log(s, AV_LOG_ERROR, "HEIF tile %d doesn't reference a stream\n", tile_id);
+                return AVERROR_INVALIDDATA;
+            }
+
+            if (j && ((tile_grid->tile_width[j - 1]  != item->width) ||
+                      (tile_grid->tile_height[j - 1] != item->height))) {
+                avpriv_request_sample(s, "Non uniform HEIF tiles");
+                return AVERROR_PATCHWELCOME;
+            }
+            tile_grid->tile_width[j]  = item->width;
+            tile_grid->tile_height[j] = item->height;
+            st->codecpar->width  = item->width;
+            st->codecpar->height = item->height;
+
+            err = avformat_stream_group_add_stream(stg, st);
+            if (err == AVERROR(EEXIST))
+                return AVERROR_INVALIDDATA;
+            else if (err < 0)
+                return err;
+
+            sc = st->priv_data;
+            sc->sample_sizes[0]  = item->extent_length;
+            sc->chunk_offsets[0] = item->extent_offset;
+
+            st->disposition |= AV_DISPOSITION_TILE;
+
+            mov_build_index(mov, st);
+
+            sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
+                                         &st->codecpar->nb_coded_side_data,
+                                         AV_PKT_DATA_TILE_INFO,
+                                         sizeof(uint32_t) * 2, 0);
+            if (!sd)
+                return AVERROR(ENOMEM);
+
+            AV_WL32(sd->data, i);
+            AV_WL32(sd->data + 4, mov->nb_tiles);
+            break;
+        }
+    }
+
+    for (int i = 0; i < mov->heif_info_size; i++) {
+        HEIFItem *item = &mov->heif_info[i];
+        AVStream *st = item->st;
+        MOVStreamContext *sc;
+
+        if (item->item_id != mov->thmb_item_id)
+            continue;
+
+        if (!st) {
+            av_log(s, AV_LOG_ERROR, "HEIF thumbnail doesn't reference a stream\n");
+            return AVERROR_INVALIDDATA;
+        }
+
+        sc = st->priv_data;
+        sc->sample_sizes[0]  = item->extent_length;
+        sc->chunk_offsets[0] = item->extent_offset;
+
+        mov_build_index(mov, st);
+    }
+
+    return 0;
+}
+
 static int mov_read_header(AVFormatContext *s)
 {
     MOVContext *mov = s->priv_data;
@@ -8819,6 +9047,9 @@  static int mov_read_header(AVFormatContext *s)
 
     mov->fc = s;
     mov->trak_index = -1;
+    mov->grid_item_id = -1;
+    mov->thmb_item_id = -1;
+    mov->primary_item_id = -1;
     /* .mov and .mp4 aren't streamable anyway (only progressive download if moov is before mdat) */
     if (pb->seekable & AVIO_SEEKABLE_NORMAL)
         atom.size = avio_size(pb);
@@ -8841,23 +9072,30 @@  static int mov_read_header(AVFormatContext *s)
     av_log(mov->fc, AV_LOG_TRACE, "on_parse_exit_offset=%"PRId64"\n", avio_tell(pb));
 
     if (mov->found_iloc) {
-        for (i = 0; i < mov->heif_info_size; i++) {
-            HEIFItem *item = &mov->heif_info[i];
-            MOVStreamContext *sc;
-            AVStream *st;
+        if (mov->nb_tiles) {
+            err = mov_parse_tiles(s);
+            if (err < 0)
+                return err;
+        } else
+            for (i = 0; i < mov->heif_info_size; i++) {
+                HEIFItem *item = &mov->heif_info[i];
+                AVStream *st = item->st;
+                MOVStreamContext *sc;
 
-            if (!item->st)
-                continue;
+                if (!st)
+                    continue;
 
-            st = item->st;
-            sc = st->priv_data;
-            st->codecpar->width  = item->width;
-            st->codecpar->height = item->height;
-            sc->sample_sizes[0]  = item->extent_length;
-            sc->chunk_offsets[0] = item->extent_offset;
+                sc = st->priv_data;
+                st->codecpar->width  = item->width;
+                st->codecpar->height = item->height;
+                sc->sample_sizes[0]  = item->extent_length;
+                sc->chunk_offsets[0] = item->extent_offset;
 
-            mov_build_index(mov, st);
-        }
+                if (item->item_id == mov->primary_item_id)
+                    st->disposition |= AV_DISPOSITION_DEFAULT;
+
+                mov_build_index(mov, st);
+            }
     }
 
     if (pb->seekable & AVIO_SEEKABLE_NORMAL) {