LineageOS/android_external_libvpx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix unsigned integer overflow in sse computation

Browse Source 
Basically port the fix from libaom:
https://aomedia-review.googlesource.com/c/aom/+/169361

Change-Id: Id06a5db91372037832399200ded75d514e096726
(cherry picked from commit a94cdd57ffd95ee7beb48d2794dae538f25da46c)
This commit is contained in:
Cheng Chen
2023-01-30 11:51:58 -08:00
committed by James Zern
parent b5a2b3a929
commit d6eb9696aa
1 changed files with 24 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
vpx_dsp/psnr.c
View File

@@ -26,57 +26,44 @@ double vpx_sse_to_psnr(double samples, double peak, double sse) {
/* TODO(yaowu): The block_variance calls the unoptimized versions of variance()
* and highbd_8_variance(). It should not.
*/
static void encoder_variance(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int w, int h, unsigned int *sse,
int *sum) {
static int64_t encoder_sse(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int w, int h) {
int i, j;
*sum = 0;
*sse = 0;
int64_t sse = 0;
for (i = 0; i < h; i++) {
for (j = 0; j < w; j++) {
const int diff = a[j] - b[j];
*sum += diff;
*sse += diff * diff;
sse += diff * diff;
}
a += a_stride;
b += b_stride;
}
return sse;
}
#if CONFIG_VP9_HIGHBITDEPTH
static void encoder_highbd_variance64(const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, int w,
int h, uint64_t *sse, int64_t *sum) {
static int64_t encoder_highbd_8_sse(const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, int w,
int h) {
int i, j;
int64_t sse = 0;
uint16_t *a = CONVERT_TO_SHORTPTR(a8);
uint16_t *b = CONVERT_TO_SHORTPTR(b8);
*sum = 0;
*sse = 0;
for (i = 0; i < h; i++) {
for (j = 0; j < w; j++) {
const int diff = a[j] - b[j];
*sum += diff;
*sse += diff * diff;
sse += diff * diff;
}
a += a_stride;
b += b_stride;
}
}
static void encoder_highbd_8_variance(const uint8_t *a8, int a_stride,
const uint8_t *b8, int b_stride, int w,
int h, unsigned int *sse, int *sum) {
uint64_t sse_long = 0;
int64_t sum_long = 0;
encoder_highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long,
&sum_long);
*sse = (unsigned int)sse_long;
*sum = (int)sum_long;
return sse;
}
#endif // CONFIG_VP9_HIGHBITDEPTH
@@ -85,26 +72,23 @@ static int64_t get_sse(const uint8_t *a, int a_stride, const uint8_t *b,
const int dw = width % 16;
const int dh = height % 16;
int64_t total_sse = 0;
unsigned int sse = 0;
int sum = 0;
int x, y;
if (dw > 0) {
encoder_variance(&a[width - dw], a_stride, &b[width - dw], b_stride, dw,
height, &sse, &sum);
total_sse += sse;
total_sse += encoder_sse(&a[width - dw], a_stride, &b[width - dw], b_stride,
dw, height);
}
if (dh > 0) {
encoder_variance(&a[(height - dh) * a_stride], a_stride,
&b[(height - dh) * b_stride], b_stride, width - dw, dh,
&sse, &sum);
total_sse += sse;
total_sse +=
encoder_sse(&a[(height - dh) * a_stride], a_stride,
&b[(height - dh) * b_stride], b_stride, width - dw, dh);
}
for (y = 0; y < height / 16; ++y) {
const uint8_t *pa = a;
const uint8_t *pb = b;
unsigned int sse;
for (x = 0; x < width / 16; ++x) {
vpx_mse16x16(pa, a_stride, pb, b_stride, &sse);
total_sse += sse;
@@ -146,22 +130,19 @@ static int64_t highbd_get_sse(const uint8_t *a, int a_stride, const uint8_t *b,
int x, y;
const int dw = width % 16;
const int dh = height % 16;
unsigned int sse = 0;
int sum = 0;
if (dw > 0) {
encoder_highbd_8_variance(&a[width - dw], a_stride, &b[width - dw],
b_stride, dw, height, &sse, &sum);
total_sse += sse;
total_sse += encoder_highbd_8_sse(&a[width - dw], a_stride, &b[width - dw],
b_stride, dw, height);
}
if (dh > 0) {
encoder_highbd_8_variance(&a[(height - dh) * a_stride], a_stride,
&b[(height - dh) * b_stride], b_stride,
width - dw, dh, &sse, &sum);
total_sse += sse;
total_sse += encoder_highbd_8_sse(&a[(height - dh) * a_stride], a_stride,
&b[(height - dh) * b_stride], b_stride,
width - dw, dh);
}
for (y = 0; y < height / 16; ++y) {
const uint8_t *pa = a;
const uint8_t *pb = b;
unsigned int sse;
for (x = 0; x < width / 16; ++x) {
vpx_highbd_8_mse16x16(pa, a_stride, pb, b_stride, &sse);
total_sse += sse;