Fix unit test failure for RECT_TX + VAR_TX

Disable rect_tx because we only support 4x4 Walsh-Hadamard transform
in lossless mode.

Fixes failure in ./test_libaom --gtest_filter=*Large*ScreencastQ0/1
Configuration: --enable-experimental --enable-var-tx --enable-rect-tx
 --enable-ref-mv --enable-ext_intra --enable-ext_tx --enable-debug
 --disable-optimizations

Change-Id: Ib6b3494c7dcf7182f1cab9b138388d054851a23d

av1/common/blockd.h

@@ -516,8 +516,10 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
   return LUT[bsize];
 }
 
-static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) {
+static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
-  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type);
+                                     const MB_MODE_INFO *mbmi) {
+  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
+         !xd->lossless[mbmi->segment_id];
 }
 
 static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }

av1/decoder/decodemv.c

@@ -1633,7 +1633,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
       int idx, idy;
       int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
+      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize) &&
+                               !xd->lossless[mbmi->segment_id];
       int use_rect_tx = 0;
 
       if (is_rect_tx_allowed) {

av1/encoder/bitstream.c

@@ -406,7 +406,7 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(mbmi)));
+    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
     assert(
         IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -1132,7 +1132,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
       int idx, idy;
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      if (is_rect_tx_allowed(mbmi)) {
+      if (is_rect_tx_allowed(xd, mbmi)) {
         int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 
         aom_write(w, is_rect_tx(mbmi->tx_size),

av1/encoder/encodeframe.c

@@ -5141,14 +5141,14 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
                                        : intra_tx_size_cat_lookup[bsize];
       const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
+      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      if (is_rect_tx_allowed(mbmi)) {
+      if (is_rect_tx_allowed(xd, mbmi)) {
         td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
       }
-      if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+      if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         if (is_inter)
           tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);

av1/encoder/rdopt.c

@@ -1385,14 +1385,14 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
   const int is_inter = is_inter_block(mbmi);
 #if CONFIG_EXT_TX
 #if CONFIG_RECT_TX
-  int evaulate_rect_tx = 0;
+  int evaluate_rect_tx = 0;
 #endif  // CONFIG_RECT_TX
   int ext_tx_set;
 #endif  // CONFIG_EXT_TX
 
   if (tx_select) {
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    evaulate_rect_tx = is_rect_tx_allowed(mbmi);
+    evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     start_tx = max_tx_size;
     end_tx = 0;
@@ -1400,8 +1400,8 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
     const TX_SIZE chosen_tx_size =
         tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    evaulate_rect_tx = is_rect_tx(chosen_tx_size);
+    evaluate_rect_tx = is_rect_tx(chosen_tx_size);
-    assert(IMPLIES(evaulate_rect_tx, is_rect_tx_allowed(mbmi)));
+    assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     start_tx = chosen_tx_size;
     end_tx = chosen_tx_size;
@@ -1415,7 +1415,7 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
   mbmi->tx_type = tx_type;
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-  if (evaulate_rect_tx) {
+  if (evaluate_rect_tx) {
     const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
     const int ext_tx_set = get_ext_tx_set(rect_tx_size, bs, 1);
     if (ext_tx_used_inter[ext_tx_set][tx_type]) {
@@ -3214,7 +3214,7 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
   mbmi->tx_type = tx_type;
   inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-  if (is_rect_tx_allowed(mbmi)) {
+  if (is_rect_tx_allowed(xd, mbmi)) {
     int rate_rect_tx, skippable_rect_tx = 0;
     int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
     int tx_size_cat = inter_tx_size_cat_lookup[bsize];