Merge "Fix ext-tx experiment for highbitdepth" into nextgenv2

vp10/common/idct.c

@@ -433,6 +433,47 @@ static void highbd_inv_idtx_add_c(const tran_low_t *input, uint8_t *dest8,
     input += bs;
   }
 }
+
+static void maybe_flip_strides16(uint16_t **dst, int *dstride,
+                                 tran_low_t **src, int *sstride,
+                                 int tx_type, int size) {
+  // Note that the transpose of src will be added to dst. In order to LR
+  // flip the addends (in dst coordinates), we UD flip the src. To UD flip
+  // the addends, we UD flip the dst.
+  switch (tx_type) {
+    case DCT_DCT:
+    case ADST_DCT:
+    case DCT_ADST:
+    case ADST_ADST:
+    case DST_DST:
+    case DCT_DST:
+    case DST_DCT:
+    case DST_ADST:
+    case ADST_DST:
+      break;
+    case FLIPADST_DCT:
+    case FLIPADST_ADST:
+    case FLIPADST_DST:
+      // flip UD
+      FLIPUD_PTR(*dst, *dstride, size);
+      break;
+    case DCT_FLIPADST:
+    case ADST_FLIPADST:
+    case DST_FLIPADST:
+      // flip LR
+      FLIPUD_PTR(*src, *sstride, size);
+      break;
+    case FLIPADST_FLIPADST:
+      // flip UD
+      FLIPUD_PTR(*dst, *dstride, size);
+      // flip LR
+      FLIPUD_PTR(*src, *sstride, size);
+      break;
+    default:
+      assert(0);
+      break;
+  }
+}
 #endif  // CONFIG_VP9_HIGHBITDEPTH
 #endif  // CONFIG_EXT_TX
 
@@ -883,8 +924,7 @@ void vp10_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
   }
 
 #if CONFIG_EXT_TX
-  maybe_flip_strides((uint8_t**)&dest, &stride,
+  maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 4);
-                     &outp, &outstride, tx_type, 4 * 2);
 #endif
 
   // Sum with the destination
@@ -950,8 +990,7 @@ void vp10_highbd_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest8,
   }
 
 #if CONFIG_EXT_TX
-  maybe_flip_strides((uint8_t**)&dest,
+  maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 8);
-                     &stride, &outp, &outstride, tx_type, 8 * 2);
 #endif
 
   // Sum with the destination
@@ -1017,8 +1056,7 @@ void vp10_highbd_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest8,
   }
 
 #if CONFIG_EXT_TX
-  maybe_flip_strides((uint8_t**)&dest, &stride,
+  maybe_flip_strides16(&dest, &stride, &outp, &outstride, tx_type, 16);
-                     &outp, &outstride, tx_type, 16 * 2);
 #endif
 
   // Sum with the destination