From 6a9dbefc7c7ac299dc9f311f197a72af954289c5 Mon Sep 17 00:00:00 2001 From: Jingning Han <jingning@google.com> Date: Mon, 10 Apr 2017 10:25:14 -0700 Subject: [PATCH] Fix high bit-depth distortion computation Resolve the segmentation fault issue in nightly run unit test AV1/AltRefForcedKeyTestLarge.Frame1IsKey/0 Change-Id: Ib27ac0d41767f39114905b7edb6c220bf62214a1 --- av1/encoder/rdopt.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index f091afbc9..a76eb0906 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -1497,8 +1497,13 @@ static void dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane, #endif // CONFIG_DAALA_DIST } else { #if CONFIG_AOM_HIGHBITDEPTH + uint8_t *recon; DECLARE_ALIGNED(16, uint16_t, recon16[MAX_TX_SQUARE]); - uint8_t *recon = CONVERT_TO_BYTEPTR(recon16); + + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + recon = CONVERT_TO_BYTEPTR(recon16); + else + recon = (uint8_t *)recon16; #else DECLARE_ALIGNED(16, uint8_t, recon[MAX_TX_SQUARE]); #endif // CONFIG_AOM_HIGHBITDEPTH