Signal CDEF damping in the frame header

Change-Id: I57f232623e647f029b007de8ddb203c47ca3c11c

av1/common/cdef.c

@@ -276,7 +276,8 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
         uint16_t dst[MAX_SB_SIZE * MAX_SB_SIZE];
         int coffset;
         int rend, cend;
-        int clpf_damping = 3 - (pli != AOM_PLANE_Y) + (cm->base_qindex >> 6);
+        int clpf_damping = cm->cdef_clpf_damping;
+        int dering_damping = cm->cdef_dering_damping;
         int hsize = nhb << mi_wide_l2[pli];
         int vsize = nvb << mi_high_l2[pli];
 
@@ -405,7 +406,8 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
               xd->plane[pli].dst.stride, dst,
               &src[OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER],
               xdec[pli], ydec[pli], dir, NULL, var, pli, dlist, dering_count,
-              level, clpf_strength, clpf_damping, coeff_shift, 0, 1);
+              level, clpf_strength, clpf_damping, dering_damping, coeff_shift,
+              0, 1);
         } else {
 #endif
           od_dering(&xd->plane[pli]
@@ -416,7 +418,7 @@ void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
                     &src[OD_FILT_VBORDER * OD_FILT_BSTRIDE + OD_FILT_HBORDER],
                     xdec[pli], ydec[pli], dir, NULL, var, pli, dlist,
                     dering_count, level, clpf_strength, clpf_damping,
-                    coeff_shift, 0, 0);
+                    dering_damping, coeff_shift, 0, 0);
 
 #if CONFIG_HIGHBITDEPTH
         }

av1/common/od_dering.c

@@ -318,22 +318,22 @@ void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec,
                int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS],
                int *dirinit, int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS],
                int pli, dering_list *dlist, int dering_count, int level,
-               int clpf_strength, int clpf_damping, int coeff_shift,
-               int skip_dering, int hbd) {
+               int clpf_strength, int clpf_damping, int dering_damping,
+               int coeff_shift, int skip_dering, int hbd) {
   int bi;
   int bx;
   int by;
   int bsize, bsizex, bsizey;
 
   int threshold = (level >> 1) << coeff_shift;
-  int dering_damping = 5 + !pli + coeff_shift;
   int filter_skip = get_filter_skip(level);
   if (level == 1) threshold = 31 << coeff_shift;
 
   od_filter_dering_direction_func filter_dering_direction[] = {
     od_filter_dering_direction_4x4, od_filter_dering_direction_8x8
   };
-  clpf_damping += coeff_shift;
+  clpf_damping += coeff_shift - (pli != AOM_PLANE_Y);
+  dering_damping += coeff_shift - (pli != AOM_PLANE_Y);
   bsize =
       ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
   bsizex = 3 - xdec;

av1/common/od_dering.h

@@ -49,6 +49,6 @@ void od_dering(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in, int xdec,
                int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS],
                int *dirinit, int var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS],
                int pli, dering_list *dlist, int dering_count, int level,
-               int clpf_strength, int clpf_damping, int coeff_shift,
-               int skip_dering, int hbd);
+               int clpf_strength, int clpf_damping, int dering_damping,
+               int coeff_shift, int skip_dering, int hbd);
 #endif

av1/common/onyxc_int.h

@@ -402,6 +402,8 @@ typedef struct AV1Common {
   int mib_size;        // Size of the superblock in units of MI blocks
   int mib_size_log2;   // Log 2 of above.
 #if CONFIG_CDEF
+  int cdef_dering_damping;
+  int cdef_clpf_damping;
   int nb_cdef_strengths;
   int cdef_strengths[CDEF_MAX_STRENGTHS];
   int cdef_uv_strengths[CDEF_MAX_STRENGTHS];

av1/decoder/decodeframe.c

@@ -2605,6 +2605,8 @@ static void setup_loopfilter(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
 #if CONFIG_CDEF
 static void setup_cdef(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
   int i;
+  cm->cdef_dering_damping = aom_rb_read_literal(rb, 1) + 5;
+  cm->cdef_clpf_damping = aom_rb_read_literal(rb, 2) + 3;
   cm->cdef_bits = aom_rb_read_literal(rb, 2);
   cm->nb_cdef_strengths = 1 << cm->cdef_bits;
   for (i = 0; i < cm->nb_cdef_strengths; i++) {

av1/encoder/bitstream.c

@@ -3521,6 +3521,8 @@ static void encode_loopfilter(AV1_COMMON *cm, struct aom_write_bit_buffer *wb) {
 #if CONFIG_CDEF
 static void encode_cdef(const AV1_COMMON *cm, struct aom_write_bit_buffer *wb) {
   int i;
+  aom_wb_write_literal(wb, cm->cdef_dering_damping - 5, 1);
+  aom_wb_write_literal(wb, cm->cdef_clpf_damping - 3, 2);
   aom_wb_write_literal(wb, cm->cdef_bits, 2);
   for (i = 0; i < cm->nb_cdef_strengths; i++) {
     aom_wb_write_literal(wb, cm->cdef_strengths[i], CDEF_STRENGTH_BITS);

av1/encoder/pickcdef.c

@@ -295,6 +295,7 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
   int *selected_strength = aom_malloc(nvsb * nhsb * sizeof(*sb_index));
   uint64_t(*mse[2])[TOTAL_STRENGTHS];
   int clpf_damping = 3 + (cm->base_qindex >> 6);
+  int dering_damping = 6;
   int i;
   int nb_strengths;
   int nb_strength_bits;
@@ -412,7 +413,7 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
                     tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit, var, pli,
                     dlist, dering_count, threshold,
                     clpf_strength + (clpf_strength == 3), clpf_damping,
-                    coeff_shift, clpf_strength != 0, 1);
+                    dering_damping, coeff_shift, clpf_strength != 0, 1);
           curr_mse = compute_dering_dist(
               ref_coeff[pli] +
                   (sbr * MAX_MIB_SIZE << mi_high_l2[pli]) * stride[pli] +
@@ -476,6 +477,8 @@ void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
     selected_strength[i] = best_gi;
     cm->mi_grid_visible[sb_index[i]]->mbmi.cdef_strength = best_gi;
   }
+  cm->cdef_dering_damping = dering_damping;
+  cm->cdef_clpf_damping = clpf_damping;
   aom_free(mse[0]);
   aom_free(mse[1]);
   for (pli = 0; pli < nplanes; pli++) {