Restrict dering thresholds and add damping to RDO

High latency, cpu-used=0:
   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.0947 |  0.0968 |  0.1203 |  -0.0325 | -0.0648 | -0.0290 |    -0.0099

Low latency, cpu-used=0:
PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.0635 |  0.1315 | -0.0771 |  -0.0122 | -0.0598 |  0.0111 |    -0.0362

High latency, cpu-used=4:
   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.1178 |  0.0026 |  0.1003 |  -0.0609 | -0.1287 | -0.1119 |    -0.1249

Low latency, cpu-used=4:
   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
-0.0404 |  0.0547 |  0.0976 |   0.0042 | -0.0585 | -0.0234 |    -0.0245

Change-Id: I48bcdb4d3d27512160ae1e1a36308dd62cf54c59

av1/common/cdef.h

@@ -13,7 +13,7 @@
 
 #define CDEF_STRENGTH_BITS 7
 
-#define DERING_STRENGTHS 21
+#define DERING_STRENGTHS 32
 #define CLPF_STRENGTHS 4
 
 #include "./aom_config.h"

av1/common/od_dering.c

@@ -281,20 +281,13 @@ void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec,
   int by;
   int bsize;
 
-  // TODO(stemidts): We might be good with fewer strengths and different
-  // strengths for chroma.  Perhaps reduce CDEF_STRENGTH_BITS to 5 and
-  // DERING_STRENGTHS to 8 and use the following tables:
-  // static int level_table[DERING_STRENGTHS] = {0, 1, 3, 7, 14, 24, 39, 63};
-  // static int level_table_uv[DERING_STRENGTHS] = {0, 1, 2, 5, 8, 12, 18, 25};
-  // For now, use 21 strengths and the same for luma and chroma.
-  static int level_table[DERING_STRENGTHS] = {
-    0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 24, 28, 33, 39, 46, 54, 63
-  };
-  static int level_table_uv[DERING_STRENGTHS] = {
-    0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 24, 28, 33, 39, 46, 54, 63
-  };
+  int threshold = (level >> 1) << coeff_shift;
+  int dering_damping = 4 + !pli + (level & 1) + coeff_shift;
+  if (level == 1) {
+    threshold = 1 << coeff_shift;
+    dering_damping = 3 + !pli + coeff_shift;
+  }
 
-  int threshold = (pli ? level_table_uv : level_table)[level] << coeff_shift;
   od_filter_dering_direction_func filter_dering_direction[OD_DERINGSIZES] = {
     od_filter_dering_direction_4x4, od_filter_dering_direction_8x8
   };
@@ -326,7 +319,8 @@ void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec,
         (filter_dering_direction[bsize - OD_LOG_BSIZE0])(
             &y[bi << 2 * bsize], 1 << bsize,
             &in[(by * OD_FILT_BSTRIDE << bsize) + (bx << bsize)],
-            od_adjust_thresh(threshold, var[by][bx]), dir[by][bx], 6);
+            od_adjust_thresh(threshold, var[by][bx]), dir[by][bx],
+            dering_damping);
       }
     } else {
       for (bi = 0; bi < dering_count; bi++) {
@@ -335,7 +329,7 @@ void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec,
         (filter_dering_direction[bsize - OD_LOG_BSIZE0])(
             &y[bi << 2 * bsize], 1 << bsize,
             &in[(by * OD_FILT_BSTRIDE << bsize) + (bx << bsize)], threshold,
-            dir[by][bx], threshold == 0 ? 0 : get_msb(threshold) + 1);
+            dir[by][bx], dering_damping);
       }
     }
   }