[FFmpeg-devel] libavcodec/pgssubdec: Implement cropping

Implement missing cropping option of subtitle bitmap, enabling complex 
cropping effects.
A test sample as been submitted to https://streams.videolan.org/upload/.

Signed-off-by: Massimo Eynard <massimo.eynard@edu.univ-eiffel.fr>
---
  libavcodec/pgssubdec.c | 193 ++++++++++++++++++++++++++++-------------
  1 file changed, 135 insertions(+), 58 deletions(-)

      int          id;
      int          w;
      int          h;
-    uint8_t      *rle;
-    unsigned int rle_buffer_size, rle_data_len;
+    uint8_t      *rle;                /**< Run Length Encoded bitmap. */
+    unsigned int rle_buffer_size;
+    unsigned int rle_data_len;
      unsigned int rle_remaining_len;
+    uint8_t      *bitmap;             /**< Decoded bitmap. */
+    unsigned int bitmap_buffer_size;
+    unsigned int bitmap_size;
  } PGSSubObject;
   typedef struct PGSSubObjects {
@@ -105,8 +111,11 @@ static void flush_cache(AVCodecContext *avctx)
       for (i = 0; i < ctx->objects.count; i++) {
          av_freep(&ctx->objects.object[i].rle);
-        ctx->objects.object[i].rle_buffer_size  = 0;
+        ctx->objects.object[i].rle_buffer_size    = 0;
          ctx->objects.object[i].rle_remaining_len  = 0;
+        av_freep(&ctx->objects.object[i].bitmap);
+        ctx->objects.object[i].bitmap_buffer_size = 0;
+        ctx->objects.object[i].bitmap_size        = 0;
      }
      ctx->objects.count = 0;
      ctx->palettes.count = 0;
@@ -149,57 +158,57 @@ static av_cold int close_decoder(AVCodecContext 
*avctx)
  }
   /**
- * Decode the RLE data.
+ * Decode the RLE data of a subtitle object.
   *
- * The subtitle is stored as a Run Length Encoded image.
+ * The subtitle is stored as a Run Length Encoded bitmap image.
   *
- * @param avctx contains the current codec context
- * @param sub pointer to the processed subtitle data
- * @param buf pointer to the RLE data to process
- * @param buf_size size of the RLE data to process
+ * @param avctx Contains the current codec context.
+ * @param object Pointer to the processed subtitle object.
   */
-static int decode_rle(AVCodecContext *avctx, AVSubtitleRect *rect,
-                      const uint8_t *buf, unsigned int buf_size)
+static int decode_object_rle(AVCodecContext *avctx, PGSSubObject *object)
  {
-    const uint8_t *rle_bitmap_end;
+    const uint8_t *rle_buf;
+    const uint8_t *rle_end;
      int pixel_count, line_count;
  -    rle_bitmap_end = buf + buf_size;
+    rle_buf = object->rle;
+    rle_end = object->rle + object->rle_data_len;
  -    rect->data[0] = av_malloc_array(rect->w, rect->h);
-
-    if (!rect->data[0])
+    object->bitmap_size = object->w * object->h;
+    av_fast_padded_malloc(&object->bitmap, &object->bitmap_buffer_size, 
object->bitmap_size);
+    if (!object->bitmap)
          return AVERROR(ENOMEM);
       pixel_count = 0;
      line_count  = 0;
  -    while (buf < rle_bitmap_end && line_count < rect->h) {
+    while (rle_buf < rle_end && line_count < object->h) {
          uint8_t flags, color;
          int run;
  -        color = bytestream_get_byte(&buf);
+        color = bytestream_get_byte(&rle_buf);
          run   = 1;
           if (color == 0x00) {
-            flags = bytestream_get_byte(&buf);
+            flags = bytestream_get_byte(&rle_buf);
              run   = flags & 0x3f;
              if (flags & 0x40)
-                run = (run << 8) + bytestream_get_byte(&buf);
-            color = flags & 0x80 ? bytestream_get_byte(&buf) : 0;
+                run = (run << 8) + bytestream_get_byte(&rle_buf);
+            color = flags & 0x80 ? bytestream_get_byte(&rle_buf) : 0;
          }
  -        if (run > 0 && pixel_count + run <= rect->w * rect->h) {
-            memset(rect->data[0] + pixel_count, color, run);
+        if (run > 0 && pixel_count + run <= object->w * object->h) {
+            memset(object->bitmap + pixel_count, color, run);
              pixel_count += run;
          } else if (!run) {
              /*
               * New Line. Check if correct pixels decoded, if not 
display warning
               * and adjust bitmap pointer to correct new line position.
               */
-            if (pixel_count % rect->w > 0) {
-                av_log(avctx, AV_LOG_ERROR, "Decoded %d pixels, when 
line should be %d pixels\n",
-                       pixel_count % rect->w, rect->w);
+            if (pixel_count % object->w > 0) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Decoded %d pixels, when object line should be 
%d pixels\n",
+                       pixel_count % object->w, object->w);
                  if (avctx->err_recognition & AV_EF_EXPLODE) {
                      return AVERROR_INVALIDDATA;
                  }
@@ -208,12 +217,12 @@ static int decode_rle(AVCodecContext *avctx, 
AVSubtitleRect *rect,
          }
      }
  -    if (pixel_count < rect->w * rect->h) {
-        av_log(avctx, AV_LOG_ERROR, "Insufficient RLE data for 
subtitle\n");
+    if (pixel_count < object->w * object->h) {
+        av_log(avctx, AV_LOG_ERROR, "Insufficient RLE data for object\n");
          return AVERROR_INVALIDDATA;
      }
  -    ff_dlog(avctx, "Pixel Count = %d, Area = %d\n", pixel_count, 
rect->w * rect->h);
+    ff_dlog(avctx, "Pixel Count = %d, Area = %d\n", pixel_count, 
object->w * object->h);
       return 0;
  }
@@ -290,13 +299,15 @@ static int parse_object_segment(AVCodecContext *avctx,
      height = bytestream_get_be16(&buf);
       /* Make sure the bitmap is not too large */
-    if (avctx->width < width || avctx->height < height || !width || 
!height) {
-        av_log(avctx, AV_LOG_ERROR, "Bitmap dimensions (%dx%d) 
invalid.\n", width, height);
+    if (MAX_OBJECT_WH < width || MAX_OBJECT_WH < height || !width || 
!height) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Bitmap dimensions (%dx%d) invalid.\n", width, height);
          return AVERROR_INVALIDDATA;
      }
       object->w = width;
      object->h = height;
+    /* Dimensions against video are checked at decode after cropping. */
       av_fast_padded_malloc(&object->rle, &object->rle_buffer_size, 
rle_bitmap_len);
  @@ -468,16 +479,7 @@ static int 
parse_presentation_segment(AVCodecContext *avctx,
           ff_dlog(avctx, "Subtitle Placement x=%d, y=%d\n",
                  object->x, object->y);
-
-        if (object->x > avctx->width || object->y > avctx->height) {
-            av_log(avctx, AV_LOG_ERROR, "Subtitle out of video bounds. 
x = %d, y = %d, video width = %d, video height = %d.\n",
-                   object->x, object->y,
-                    avctx->width, avctx->height);
-            object->y = object->x = 0;
-            if (avctx->err_recognition & AV_EF_EXPLODE) {
-                return AVERROR_INVALIDDATA;
-            }
-        }
+        /* Placement is checked at decode after cropping. */
      }
       return 0;
@@ -528,6 +530,7 @@ static int display_end_segment(AVCodecContext 
*avctx, AVSubtitle *sub,
          return AVERROR_INVALIDDATA;
      }
      for (i = 0; i < ctx->presentation.object_count; i++) {
+        const PGSSubObjectRef sub_object = ctx->presentation.objects[i];
          AVSubtitleRect *const rect = av_mallocz(sizeof(*rect));
          PGSSubObject *object;
  @@ -537,45 +540,119 @@ static int display_end_segment(AVCodecContext 
*avctx, AVSubtitle *sub,
          rect->type = SUBTITLE_BITMAP;
           /* Process bitmap */
-        object = find_object(ctx->presentation.objects[i].id, 
&ctx->objects);
+        object = find_object(sub_object.id, &ctx->objects);
          if (!object) {
              // Missing object.  Should only happen with damaged streams.
              av_log(avctx, AV_LOG_ERROR, "Invalid object id %d\n",
-                   ctx->presentation.objects[i].id);
+                   sub_object.id);
              if (avctx->err_recognition & AV_EF_EXPLODE)
                  return AVERROR_INVALIDDATA;
              // Leaves rect empty with 0 width and height.
              continue;
          }
-        if (ctx->presentation.objects[i].composition_flag & 0x40)
+        if (sub_object.composition_flag & 0x40)
              rect->flags |= AV_SUBTITLE_FLAG_FORCED;
  -        rect->x    = ctx->presentation.objects[i].x;
-        rect->y    = ctx->presentation.objects[i].y;
+        rect->x    = sub_object.x;
+        rect->y    = sub_object.y;
           if (object->rle) {
-            rect->w    = object->w;
-            rect->h    = object->h;
+            int out_of_picture = 0;
  +            rect->w = object->w;
+            rect->h = object->h;
              rect->linesize[0] = object->w;
  -            if (object->rle_remaining_len) {
-                av_log(avctx, AV_LOG_ERROR, "RLE data length %u is %u 
bytes shorter than expected\n",
-                       object->rle_data_len, object->rle_remaining_len);
+            // Check for cropping.
+            if (sub_object.composition_flag & 0x80) {
+                int out_of_object = 0;
+
+                if (object->w < sub_object.crop_x + sub_object.crop_w)
+                    out_of_object = 1;
+                if (object->h < sub_object.crop_y + sub_object.crop_h)
+                    out_of_object = 1;
+
+                if (out_of_object) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Subtitle cropping values are out of object. "
+                           "obj_w = %d, obj_h = %d, crop_x = %d, crop_y 
= %d, "
+                           "crop_w = %d, crop_h = %d.\n",
+                           object->w,
+                           object->h,
+                           sub_object.crop_x,
+                           sub_object.crop_y,
+                           sub_object.crop_w,
+                           sub_object.crop_h);
+                    if (avctx->err_recognition & AV_EF_EXPLODE)
+                        return AVERROR_INVALIDDATA;
+                }
+                else {
+                    // Replace subtitle dimensions with cropping ones.
+                    rect->w = sub_object.crop_w;
+                    rect->h = sub_object.crop_h;
+                    rect->linesize[0] = sub_object.crop_w;
+                }
+            }
+
+            /* Make sure the subtitle is not out of picture. */
+            if (avctx->width < rect->x + rect->w || !rect->w)
+                out_of_picture = 1;
+            if (avctx->height < rect->y + rect->h || !rect->h)
+                out_of_picture = 1;
+            if (out_of_picture) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Subtitle out of video bounds. "
+                       "x = %d, y = %d, width = %d, height = %d.\n",
+                       rect->x, rect->y, rect->w, rect->h);
                  if (avctx->err_recognition & AV_EF_EXPLODE)
                      return AVERROR_INVALIDDATA;
-            }
-            ret = decode_rle(avctx, rect, object->rle, 
object->rle_data_len);
-            if (ret < 0) {
-                if ((avctx->err_recognition & AV_EF_EXPLODE) ||
-                    ret == AVERROR(ENOMEM)) {
-                    return ret;
-                }
                  rect->w = 0;
                  rect->h = 0;
                  continue;
              }
+
+            if (!object->bitmap_size) {
+                /* Decode bitmap from RLE. */
+                if (object->rle_remaining_len) {
+                    av_log(avctx, AV_LOG_ERROR,
+                        "RLE data length %u is %u bytes shorter than 
expected\n",
+                        object->rle_data_len, object->rle_remaining_len);
+                    if (avctx->err_recognition & AV_EF_EXPLODE)
+                        return AVERROR_INVALIDDATA;
+                }
+
+                ret = decode_object_rle(avctx, object);
+                if (ret < 0) {
+                    if ((avctx->err_recognition & AV_EF_EXPLODE) ||
+                        ret == AVERROR(ENOMEM)) {
+                        return ret;
+                    }
+                    rect->w = 0;
+                    rect->h = 0;
+                    continue;
+                }
+            }
+
+            rect->data[0] = av_malloc_array(rect->w, rect->h);
+            if (!rect->data[0])
+                return AVERROR(ENOMEM);
+
+            if (sub_object.composition_flag & 0x80) {
+                /* Copy cropped bitmap. */
+                int y;
+
+                for (y = 0; y < sub_object.crop_h; y++) {
+                    memcpy(&rect->data[0][y * sub_object.crop_w],
+                           &object->bitmap[(sub_object.crop_y + y) *
+                           object->w + sub_object.crop_x],
+                           sub_object.crop_w);
+                }
+            }
+            else {
+                memcpy(rect->data[0], object->bitmap, object->bitmap_size);
+            }
          }
+
          /* Allocate memory for colors */
          rect->nb_colors = 256;
          rect->data[1]   = av_mallocz(AVPALETTE_SIZE);