Some fixes related to 64x64 tx, ext-tx, supertx

Change-Id: I95455ed7e44f3a163c751bd8a89f6119c0055c71

av1/common/blockd.h

@@ -512,6 +512,7 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
 #endif  // CONFIG_SUPERTX
 
 #if CONFIG_EXT_TX
+
 #define ALLOW_INTRA_EXT_TX 1
 
 static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
@@ -524,18 +525,20 @@ static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter) {
   return (tx_size == TX_16X16 ? 2 : 1);
 }
 
-static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
-  { 0, 0, 0, 0 },  // unused
-  { 1, 1, 0, 0 },
-  { 0, 0, 1, 0 },
-};
+static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA]
+                                            [EXT_TX_SIZES] = {
+                                              { 1, 1, 1, 1 },  // unused
+                                              { 1, 1, 0, 0 },
+                                              { 0, 0, 1, 0 },
+                                            };
 
-static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
-  { 0, 0, 0, 0 },  // unused
-  { 1, 1, 0, 0 },
-  { 0, 0, 1, 0 },
-  { 0, 0, 0, 1 },
-};
+static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER]
+                                            [EXT_TX_SIZES] = {
+                                              { 1, 1, 1, 1 },  // unused
+                                              { 1, 1, 0, 0 },
+                                              { 0, 0, 1, 0 },
+                                              { 0, 0, 0, 1 },
+                                            };
 
 // Transform types used in each intra set
 static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
@@ -757,7 +760,7 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
     }
     if (is_inter_block(mbmi))
       // UV Inter only
-      return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] == TX_32X32)
+      return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
                  ? DCT_DCT
                  : mbmi->tx_type;
   }

av1/common/entropymode.c

@@ -1438,8 +1438,13 @@ static const aom_prob default_filter_intra_probs[2] = { 230, 230 };
 #if CONFIG_SUPERTX
 static const aom_prob default_supertx_prob[PARTITION_SUPERTX_CONTEXTS]
                                           [TX_SIZES] = {
+#if CONFIG_TX64X64
+                                            { 1, 160, 160, 170, 180 },
+                                            { 1, 200, 200, 210, 220 },
+#else
                                             { 1, 160, 160, 170 },
                                             { 1, 200, 200, 210 },
+#endif  // CONFIG_TX64X64
                                           };
 #endif  // CONFIG_SUPERTX
 

av1/common/loopfilter.c

@@ -901,7 +901,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
     case BLOCK_64X64: build_masks(lfi_n, mip[0], 0, 0, lfm); break;
     case BLOCK_64X32: build_masks(lfi_n, mip[0], 0, 0, lfm);
 #if CONFIG_SUPERTX && CONFIG_TX64X64
-      if (supertx_enabled(&mip->mbmi)) break;
+      if (supertx_enabled(&mip[0]->mbmi)) break;
 #endif  // CONFIG_SUPERTX && CONFIG_TX64X64
       mip2 = mip + mode_info_stride * 4;
       if (4 >= max_rows) break;
@@ -909,7 +909,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
       break;
     case BLOCK_32X64: build_masks(lfi_n, mip[0], 0, 0, lfm);
 #if CONFIG_SUPERTX && CONFIG_TX64X64
-      if (supertx_enabled(&mip->mbmi)) break;
+      if (supertx_enabled(&mip[0]->mbmi)) break;
 #endif  // CONFIG_SUPERTX && CONFIG_TX64X64
       mip2 = mip + 4;
       if (4 >= max_cols) break;
@@ -917,8 +917,8 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
       break;
     default:
 #if CONFIG_SUPERTX && CONFIG_TX64X64
-      if (mip->mbmi.tx_size == TX_64X64) {
-        build_masks(cm, mip, 0, 0, lfm);
+      if (mip[0]->mbmi.tx_size == TX_64X64) {
+        build_masks(lfi_n, mip[0], 0, 0, lfm);
       } else {
 #endif  // CONFIG_SUPERTX && CONFIG_TX64X64
         for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {

av1/encoder/encodeframe.c

@@ -5108,7 +5108,7 @@ void av1_encode_frame(AV1_COMP *cpi) {
       count16x16_lp += counts->tx_size_implied[4][TX_16X16];
       count32x32_32x32p += counts->tx_size_implied[3][TX_32X32];
       count32x32_lp += counts->tx_size_implied[4][TX_32X32];
-      count64x64_64x64p += counts->tx_size[4][TX_64X64];
+      count64x64_64x64p += counts->tx_size_implied[4][TX_64X64];
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
       if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
           count32x32_lp == 0 && count32x32_32x32p == 0 &&