diff mbox series

[FFmpeg-devel] avfilter/vf_scale: validate values before converting to integer

Message ID 20240710152401.1192-1-kasper93@gmail.com
State New
Headers show
Series [FFmpeg-devel] avfilter/vf_scale: validate values before converting to integer | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Kacper Michajlow July 10, 2024, 3:24 p.m. UTC 
Fixes the conversion of double values to integer, which may be out of
the representable range.

Also, bail out on overflow check instead of printing an error only.

Found by OSS-Fuzz.

Signed-off-by: Kacper Michajłow <kasper93@gmail.com>
---
 libavfilter/vf_scale.c | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)
diff mbox series

Patch

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index ae7356fd7b..66bb81dd1f 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -537,7 +537,6 @@  static int scale_eval_dimensions(AVFilterContext *ctx)
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
     const AVPixFmtDescriptor *out_desc = av_pix_fmt_desc_get(outlink->format);
     char *expr;
-    int eval_w, eval_h;
     int ret;
     double res;
     const AVPixFmtDescriptor *main_desc;
@@ -588,26 +587,20 @@  static int scale_eval_dimensions(AVFilterContext *ctx)
     }
 
     res = av_expr_eval(scale->w_pexpr, scale->var_values, NULL);
-    eval_w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = (int) res == 0 ? inlink->w : (int) res;
-
-    res = av_expr_eval(scale->h_pexpr, scale->var_values, NULL);
-    if (isnan(res)) {
-        expr = scale->h_expr;
+    if (isnan(res) || res < INT_MIN || res > INT_MAX) {
+        expr = scale->w_expr;
         ret = AVERROR(EINVAL);
         goto fail;
     }
-    eval_h = scale->var_values[VAR_OUT_H] = scale->var_values[VAR_OH] = (int) res == 0 ? inlink->h : (int) res;
+    scale->w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = res == 0 ? inlink->w : res;
 
-    res = av_expr_eval(scale->w_pexpr, scale->var_values, NULL);
-    if (isnan(res)) {
-        expr = scale->w_expr;
+    res = av_expr_eval(scale->h_pexpr, scale->var_values, NULL);
+    if (isnan(res) || res < INT_MIN || res > INT_MAX) {
+        expr = scale->h_expr;
         ret = AVERROR(EINVAL);
         goto fail;
     }
-    eval_w = scale->var_values[VAR_OUT_W] = scale->var_values[VAR_OW] = (int) res == 0 ? inlink->w : (int) res;
-
-    scale->w = eval_w;
-    scale->h = eval_h;
+    scale->h = scale->var_values[VAR_OUT_H] = scale->var_values[VAR_OH] = res == 0 ? inlink->h : res;
 
     return 0;
 
@@ -642,11 +635,15 @@  static int config_props(AVFilterLink *outlink)
                                scale->force_original_aspect_ratio,
                                scale->force_divisible_by);
 
-    if (outlink->w > INT_MAX ||
-        outlink->h > INT_MAX ||
-        (outlink->h * inlink->w) > INT_MAX ||
-        (outlink->w * inlink->h) > INT_MAX)
+    if (outlink->w <= 0 ||
+        outlink->h <= 0 ||
+        outlink->h > INT_MAX / inlink->w ||
+        outlink->w > INT_MAX / inlink->h)
+    {
         av_log(ctx, AV_LOG_ERROR, "Rescaled value for width or height is too big.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
 
     /* TODO: make algorithm configurable */