From 597d1f4c03071dc1af6399a68b2373953e8a820a Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Tue, 9 May 2017 10:49:59 -0700 Subject: [PATCH] vp9: Fix ubsan failure in denoiser. Fix the overflow for subtraction between two unsigned integers. BUG=webm:1432 Change-Id: I7b665e93ba5850548810eff23258782c4f5ee15a --- vp9/encoder/vp9_denoiser.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c index 5a5ca1f94..975a39b66 100644 --- a/vp9/encoder/vp9_denoiser.c +++ b/vp9/encoder/vp9_denoiser.c @@ -191,7 +191,9 @@ static VP9_DENOISER_DECISION perform_motion_compensation( int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx, int motion_magnitude, int is_skin, int *zeromv_filter, int consec_zeromv, int num_spatial_layers, int width) { - int sse_diff = ctx->zeromv_sse - ctx->newmv_sse; + const int sse_diff = (ctx->newmv_sse == UINT_MAX) + ? 0 + : ((int)ctx->zeromv_sse - (int)ctx->newmv_sse); MV_REFERENCE_FRAME frame; MACROBLOCKD *filter_mbd = &mb->e_mbd; MODE_INFO *mi = filter_mbd->mi[0]; @@ -217,7 +219,6 @@ static VP9_DENOISER_DECISION perform_motion_compensation( // difference in sum-squared-error, use it. if (frame != INTRA_FRAME && (frame != GOLDEN_FRAME || num_spatial_layers == 1) && - ctx->newmv_sse != UINT_MAX && sse_diff > sse_diff_thresh(bs, increase_denoising, motion_magnitude)) { mi->ref_frame[0] = ctx->best_reference_frame; mi->mode = ctx->best_sse_inter_mode;