Fix RunAccuracyCheck failure

Measure the accuracy of each transform in terms of per coefficient basis.
Set up a accuracy limit corresponding to current transform
implementation.

Change-Id: Ib7db9680c963427e94e728bf453b66180ce30b89

test/av1_fht16x16_test.cc

@@ -87,7 +87,7 @@ class AV1Trans16x16HT : public libaom_test::TransformTestBase,
 };
 
 TEST_P(AV1Trans16x16HT, MemCheck) { RunMemCheck(); }
-TEST_P(AV1Trans16x16HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(AV1Trans16x16HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); }
 TEST_P(AV1Trans16x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); }
 TEST_P(AV1Trans16x16HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans16x16HT, InvCoeffCheck) { RunInvCoeffCheck(); }

test/av1_fht16x32_test.cc

@@ -70,11 +70,11 @@ class AV1Trans16x32HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(AV1Trans16x32HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(AV1Trans16x32HT, AccuracyCheck) { RunAccuracyCheck(4, 0.2); }
 TEST_P(AV1Trans16x32HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans16x32HT, MemCheck) { RunMemCheck(); }
 TEST_P(AV1Trans16x32HT, InvCoeffCheck) { RunInvCoeffCheck(); }
-TEST_P(AV1Trans16x32HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); }
+TEST_P(AV1Trans16x32HT, InvAccuracyCheck) { RunInvAccuracyCheck(4); }
 
 using std::tr1::make_tuple;
 const Ht16x32Param kArrayHt16x32Param_c[] = {

test/av1_fht16x8_test.cc

@@ -70,7 +70,7 @@ class AV1Trans16x8HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(AV1Trans16x8HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(AV1Trans16x8HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); }
 TEST_P(AV1Trans16x8HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans16x8HT, MemCheck) { RunMemCheck(); }
 TEST_P(AV1Trans16x8HT, InvCoeffCheck) { RunInvCoeffCheck(); }

test/av1_fht32x16_test.cc

@@ -71,10 +71,10 @@ class AV1Trans32x16HT : public libaom_test::TransformTestBase,
 };
 
 TEST_P(AV1Trans32x16HT, MemCheck) { RunMemCheck(); }
-TEST_P(AV1Trans32x16HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(AV1Trans32x16HT, AccuracyCheck) { RunAccuracyCheck(4, 0.2); }
 TEST_P(AV1Trans32x16HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans32x16HT, InvCoeffCheck) { RunInvCoeffCheck(); }
-TEST_P(AV1Trans32x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(1); }
+TEST_P(AV1Trans32x16HT, InvAccuracyCheck) { RunInvAccuracyCheck(4); }
 
 using std::tr1::make_tuple;
 const Ht32x16Param kArrayHt32x16Param_c[] = {

test/av1_fht4x8_test.cc

@@ -70,7 +70,7 @@ class AV1Trans4x8HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(AV1Trans4x8HT, AccuracyCheck) { RunAccuracyCheck(0); }
+TEST_P(AV1Trans4x8HT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); }
 TEST_P(AV1Trans4x8HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans4x8HT, MemCheck) { RunMemCheck(); }
 TEST_P(AV1Trans4x8HT, InvCoeffCheck) { RunInvCoeffCheck(); }

test/av1_fht8x16_test.cc

@@ -69,7 +69,7 @@ class AV1Trans8x16HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(AV1Trans8x16HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(AV1Trans8x16HT, AccuracyCheck) { RunAccuracyCheck(1, 0.001); }
 TEST_P(AV1Trans8x16HT, MemCheck) { RunMemCheck(); }
 TEST_P(AV1Trans8x16HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans8x16HT, InvCoeffCheck) { RunInvCoeffCheck(); }

test/av1_fht8x4_test.cc

@@ -69,7 +69,7 @@ class AV1Trans8x4HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(AV1Trans8x4HT, AccuracyCheck) { RunAccuracyCheck(0); }
+TEST_P(AV1Trans8x4HT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); }
 TEST_P(AV1Trans8x4HT, CoeffCheck) { RunCoeffCheck(); }
 TEST_P(AV1Trans8x4HT, MemCheck) { RunMemCheck(); }
 TEST_P(AV1Trans8x4HT, InvCoeffCheck) { RunInvCoeffCheck(); }

test/fdct4x4_test.cc

@@ -120,7 +120,7 @@ class Trans4x4DCT : public libaom_test::TransformTestBase,
   IdctFunc inv_txfm_;
 };
 
-TEST_P(Trans4x4DCT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(Trans4x4DCT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); }
 
 TEST_P(Trans4x4DCT, CoeffCheck) { RunCoeffCheck(); }
 
@@ -159,7 +159,7 @@ class Trans4x4HT : public libaom_test::TransformTestBase,
   IhtFunc inv_txfm_;
 };
 
-TEST_P(Trans4x4HT, AccuracyCheck) { RunAccuracyCheck(1); }
+TEST_P(Trans4x4HT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); }
 
 TEST_P(Trans4x4HT, CoeffCheck) { RunCoeffCheck(); }
 
@@ -197,7 +197,7 @@ class Trans4x4WHT : public libaom_test::TransformTestBase,
   IdctFunc inv_txfm_;
 };
 
-TEST_P(Trans4x4WHT, AccuracyCheck) { RunAccuracyCheck(0); }
+TEST_P(Trans4x4WHT, AccuracyCheck) { RunAccuracyCheck(0, 0.00001); }
 
 TEST_P(Trans4x4WHT, CoeffCheck) { RunCoeffCheck(); }
 

test/transform_test_base.h

@@ -42,7 +42,7 @@ class TransformTestBase {
 
   virtual void RunInvTxfm(const tran_low_t *out, uint8_t *dst, int stride) = 0;
 
-  void RunAccuracyCheck(int limit) {
+  void RunAccuracyCheck(uint32_t ref_max_error, double ref_avg_error) {
     ACMRandom rnd(ACMRandom::DeterministicSeed());
     uint32_t max_error = 0;
     int64_t total_error = 0;
@@ -104,11 +104,14 @@ class TransformTestBase {
       }
     }
 
-    EXPECT_GE(static_cast<uint32_t>(limit), max_error)
-        << "Error: FHT/IHT has an individual round trip error > " << limit;
+    double avg_error = total_error * 1. / count_test_block / num_coeffs_;
 
-    EXPECT_GE(count_test_block * limit, total_error)
-        << "Error: FHT/IHT has average round trip error > " << limit
+    EXPECT_GE(ref_max_error, max_error)
+        << "Error: FHT/IHT has an individual round trip error > "
+        << ref_max_error;
+
+    EXPECT_GE(ref_avg_error, avg_error)
+        << "Error: FHT/IHT has average round trip error > " << ref_avg_error
         << " per block";
 
     aom_free(test_input_block);