Fix deringing level choice for 10-bit and 12-bit

Making sure we never exceed a base level of 63

Change-Id: I821254b8d970446bd40fdd6e4d7073c69760a86d

av1/encoder/pickdering.c

@@ -90,8 +90,12 @@ int av1_dering_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
      adjusted on a 64x64 basis. We use a threshold of the form T = a*Q^b,
      where a and b are derived empirically trying to optimize rate-distortion
      at different quantizer settings. */
-  best_level = (int)floor(
-      .5 + .45 * pow(av1_ac_quant(cm->base_qindex, 0, cm->bit_depth), 0.6));
+  best_level = AOMMIN(
+      MAX_DERING_LEVEL - 1,
+      (int)floor(.5 +
+                 .45 * pow(av1_ac_quant(cm->base_qindex, 0, cm->bit_depth) >>
+                               (cm->bit_depth - 8),
+                           0.6)));
   for (sbr = 0; sbr < nvsb; sbr++) {
     for (sbc = 0; sbc < nhsb; sbc++) {
       int nvb, nhb;