Revert "Clamp inverse transform coefficients"

This reverts commit 79b78b7d47.

The transform coefficient range needs some more tuning.
Before we finalize on that front, directly applying clamping
would cause multiple unit test failure issues. Hence revert
this Cl temporarily.

BUG=aomedia:612

Change-Id: I1dd8680dee17289801c4a209275f05a498355c8e

aom_dsp/aom_dsp_common.h

@@ -80,10 +80,6 @@ static INLINE int clamp(int value, int low, int high) {
   return value < low ? low : (value > high ? high : value);
 }
 
-static INLINE int64_t clamp_64(int64_t value, int64_t low, int64_t high) {
-  return value < low ? low : (value > high ? high : (int)value);
-}
-
 static INLINE double fclamp(double value, double low, double high) {
   return value < low ? low : (value > high ? high : value);
 }

aom_dsp/inv_txfm.h

@@ -45,10 +45,6 @@ static INLINE tran_high_t check_range(tran_high_t input, int bd) {
   assert(int_min <= input);
   assert(input <= int_max);
   (void)int_min;
-#else
-  const int32_t int_max = (1 << (7 + bd)) - 1;
-  const int32_t int_min = -int_max - 1;
-  input = (int32_t)clamp_64(input, int_min, int_max);
 #endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
   (void)bd;
   return input;

av1/common/av1_fwd_txfm1d.c

@@ -21,9 +21,13 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
   range_check_func(stage, input, buf, size, bit)
 #else
 #define range_check(stage, input, buf, size, bit) \
-  clamp_buf((int32_t *)buf, size, bit)
-
-void clamp_buf(const int32_t *buf, int32_t size, int8_t bit);
+  {                                               \
+    (void)stage;                                  \
+    (void)input;                                  \
+    (void)buf;                                    \
+    (void)size;                                   \
+    (void)bit;                                    \
+  }
 #endif
 
 // TODO(angiebird): Make 1-d txfm functions static

av1/common/av1_inv_txfm1d.c

@@ -43,15 +43,13 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
   range_check_func(stage, input, buf, size, bit)
 #else
 #define range_check(stage, input, buf, size, bit) \
-  clamp_buf((int32_t *)buf, size, bit)
-
-void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
-  const int64_t maxValue = (1LL << (bit - 1)) - 1;
-  const int64_t minValue = -(1LL << (bit - 1));
-
-  for (int i = 0; i < size; ++i)
-    buf[i] = (int32_t)clamp_64(buf[i], minValue, maxValue);
-}
+  {                                               \
+    (void)stage;                                  \
+    (void)input;                                  \
+    (void)buf;                                    \
+    (void)size;                                   \
+    (void)bit;                                    \
+  }
 #endif
 
 // TODO(angiebird): Make 1-d txfm functions static

av1/common/scan.c

@@ -6600,6 +6600,10 @@ static uint32_t *get_non_zero_counts(FRAME_COUNTS *counts, TX_SIZE tx_size,
   }
 }
 
+static INLINE int clamp_64(int64_t value, int low, int high) {
+  return value < low ? low : (value > high ? high : (int)value);
+}
+
 static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
                              int rate_16) {
   FRAME_CONTEXT *pre_fc = cm->pre_fc;
@@ -6617,7 +6621,7 @@ static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
         (curr_prob * rate_16 + prev_prob * ((1 << 16) - rate_16)) >> 16;
     // TODO(angiebird): reduce the bit usage of probabilities and remove
     // clamp_64()
-    non_zero_prob[i] = (int32_t)clamp_64(pred_prob, 0, UINT16_MAX);
+    non_zero_prob[i] = clamp_64(pred_prob, 0, UINT16_MAX);
   }
 }