From 4b0cad6596bcbce5833251c9ba2a60617186a39c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 12 Jan 2014 02:27:00 +0100 Subject: [PATCH] avcodec/simple_idct_template: fix rounding of the special DC case for 10bit MSE doesnt change for the test as the code only triggers for very sparse matrixes Signed-off-by: Michael Niedermayer --- libavcodec/simple_idct_template.c | 8 ++++---- tests/ref/fate/prores-422_lt | 4 ++-- tests/ref/vsynth/vsynth2-prores | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index 838cea4951..ac04923d96 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -107,10 +107,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) #define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN) if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) { uint64_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; temp += temp << 32; @@ -124,10 +124,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) ((uint32_t*)row)[3] | row[1])) { uint32_t temp; - if (DC_SHIFT - extra_shift > 0) { + if (DC_SHIFT - extra_shift >= 0) { temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff; } else { - temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff; + temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff; } temp += temp << 16; ((uint32_t*)row)[0]=((uint32_t*)row)[1] = diff --git a/tests/ref/fate/prores-422_lt b/tests/ref/fate/prores-422_lt index 33976d0a44..3113c8a522 100644 --- a/tests/ref/fate/prores-422_lt +++ b/tests/ref/fate/prores-422_lt @@ -1,3 +1,3 @@ #tb 0: 100/2997 -0, 0, 0, 1, 8294400, 0xdaff4019 -0, 1, 1, 1, 8294400, 0xdaff4019 +0, 0, 0, 1, 8294400, 0x4ca110c7 +0, 1, 1, 1, 8294400, 0x4ca110c7 diff --git a/tests/ref/vsynth/vsynth2-prores b/tests/ref/vsynth/vsynth2-prores index b2169d827a..82311b4dfc 100644 --- a/tests/ref/vsynth/vsynth2-prores +++ b/tests/ref/vsynth/vsynth2-prores @@ -1,4 +1,4 @@ af885572d77d28e5ede27932bd68cdde *tests/data/fate/vsynth2-prores.mov 2844076 tests/data/fate/vsynth2-prores.mov -387a3c23586c679b411c3eff2c9d9d06 *tests/data/fate/vsynth2-prores.out.rawvideo +03fd29e3963716a09d232b6f817ecb57 *tests/data/fate/vsynth2-prores.out.rawvideo stddev: 1.31 PSNR: 45.77 MAXDIFF: 11 bytes: 7603200/ 7603200