diff mbox

[FFmpeg-devel,1/2] vaapi_encode_h264: Reduce SAR to valid range

Message ID 20181028171354.15794-1-sw@jkqxz.net
State Accepted
Commit a830056b32feb2db463e062d4ec2b6fb47ec70ee
Headers show

Commit Message

Mark Thompson Oct. 28, 2018, 5:13 p.m. UTC 
The SAR of the input could have a numerator or denominator greater than
2^16 which would then be truncated to a 16-bit integer when written to
the VUI parameters, giving a random result.  Instead, reduce the SAR to
the nearest representable fraction.
---
 libavcodec/vaapi_encode_h264.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

Comments

Mark Thompson Oct. 31, 2018, 11:01 p.m. UTC | #1 
On 28/10/18 17:13, Mark Thompson wrote:
> The SAR of the input could have a numerator or denominator greater than
> 2^16 which would then be truncated to a 16-bit integer when written to
> the VUI parameters, giving a random result.  Instead, reduce the SAR to
> the nearest representable fraction.

Fixes #7502.

> ---
>  libavcodec/vaapi_encode_h264.c | 12 +++++++-----
>  1 file changed, 7 insertions(+), 5 deletions(-)
> 
> diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
> index 7bb77cfba2..f9402992b8 100644
> --- a/libavcodec/vaapi_encode_h264.c
> +++ b/libavcodec/vaapi_encode_h264.c
> @@ -389,18 +389,20 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
>              {  80, 33 }, {  18, 11 }, {  15, 11 }, {  64, 33 },
>              { 160, 99 }, {   4,  3 }, {   3,  2 }, {   2,  1 },
>          };
> -        int i;
> +        int num, den, i;
> +        av_reduce(&num, &den, avctx->sample_aspect_ratio.num,
> +                  avctx->sample_aspect_ratio.den, 65535);
>          for (i = 0; i < FF_ARRAY_ELEMS(sar_idc); i++) {
> -            if (avctx->sample_aspect_ratio.num == sar_idc[i].num &&
> -                avctx->sample_aspect_ratio.den == sar_idc[i].den) {
> +            if (num == sar_idc[i].num &&
> +                den == sar_idc[i].den) {
>                  sps->vui.aspect_ratio_idc = i;
>                  break;
>              }
>          }
>          if (i >= FF_ARRAY_ELEMS(sar_idc)) {
>              sps->vui.aspect_ratio_idc = 255;
> -            sps->vui.sar_width  = avctx->sample_aspect_ratio.num;
> -            sps->vui.sar_height = avctx->sample_aspect_ratio.den;
> +            sps->vui.sar_width  = num;
> +            sps->vui.sar_height = den;
>          }
>          sps->vui.aspect_ratio_info_present_flag = 1;
>      }
>
diff mbox

Patch

diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index 7bb77cfba2..f9402992b8 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -389,18 +389,20 @@  static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
             {  80, 33 }, {  18, 11 }, {  15, 11 }, {  64, 33 },
             { 160, 99 }, {   4,  3 }, {   3,  2 }, {   2,  1 },
         };
-        int i;
+        int num, den, i;
+        av_reduce(&num, &den, avctx->sample_aspect_ratio.num,
+                  avctx->sample_aspect_ratio.den, 65535);
         for (i = 0; i < FF_ARRAY_ELEMS(sar_idc); i++) {
-            if (avctx->sample_aspect_ratio.num == sar_idc[i].num &&
-                avctx->sample_aspect_ratio.den == sar_idc[i].den) {
+            if (num == sar_idc[i].num &&
+                den == sar_idc[i].den) {
                 sps->vui.aspect_ratio_idc = i;
                 break;
             }
         }
         if (i >= FF_ARRAY_ELEMS(sar_idc)) {
             sps->vui.aspect_ratio_idc = 255;
-            sps->vui.sar_width  = avctx->sample_aspect_ratio.num;
-            sps->vui.sar_height = avctx->sample_aspect_ratio.den;
+            sps->vui.sar_width  = num;
+            sps->vui.sar_height = den;
         }
         sps->vui.aspect_ratio_info_present_flag = 1;
     }