From db0846a47569db9382ab07cc41bc4d2e5adfc0e7 Mon Sep 17 00:00:00 2001
From: Aaron Meihm <ameihm@mozilla.com>
Date: Mon, 13 Jul 2015 21:36:54 -0500
Subject: [PATCH] add validation phase for tests

---
 src/scribe/document.go | 10 ++++++++++
 src/scribe/parser.go   |  6 ++++++
 src/scribe/test.go     | 27 +++++++++++++++++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/src/scribe/document.go b/src/scribe/document.go
index 67047e5..a1ccd81 100644
--- a/src/scribe/document.go
+++ b/src/scribe/document.go
@@ -17,6 +17,16 @@ type Document struct {
 	Tests     []Test     `json:"tests"`
 }
 
+func (d *Document) Validate() error {
+	for _, x := range d.Tests {
+		err := x.validate(d)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (d *Document) runTests() error {
 	// Note that prepare() and runTest() will return an error if something
 	// goes wrong, but we don't propagate this back. Errors within a test
diff --git a/src/scribe/parser.go b/src/scribe/parser.go
index 84aa98b..1525d5c 100644
--- a/src/scribe/parser.go
+++ b/src/scribe/parser.go
@@ -28,6 +28,12 @@ func LoadDocument(r io.Reader) (Document, error) {
 	debugPrint("new document has %v test(s)\n", len(ret.Tests))
 	debugPrint("new document has %v variable(s)\n", len(ret.Variables))
 
+	debugPrint("validating document...\n")
+	err = ret.Validate()
+	if err != nil {
+		return ret, err
+	}
+
 	return ret, nil
 }
 
diff --git a/src/scribe/test.go b/src/scribe/test.go
index cddf993..64605f3 100644
--- a/src/scribe/test.go
+++ b/src/scribe/test.go
@@ -69,6 +69,33 @@ type genericEvaluator interface {
 	evaluate(EvaluationCriteria) (EvaluationResult, error)
 }
 
+func (t *Test) validate(d *Document) error {
+	if len(t.Name) == 0 {
+		return fmt.Errorf("a test in document has no name")
+	}
+	if len(t.Identifier) == 0 {
+		return fmt.Errorf("%v: no identifier", t.Name)
+	}
+	if t.getSourceInterface() == nil {
+		return fmt.Errorf("%v: no valid source interface", t.Name)
+	}
+	if t.getEvaluationInterface() == nil {
+		return fmt.Errorf("%v: no valid evaluation interface", t.Name)
+	}
+	for _, x := range t.Aliases {
+		if len(x) == 0 {
+			return fmt.Errorf("%v: bad alias within test", t.Name)
+		}
+	}
+	for _, x := range t.If {
+		_, err := d.getTest(x)
+		if err != nil {
+			return fmt.Errorf("%v: %v", t.Name, err)
+		}
+	}
+	return nil
+}
+
 func (t *Test) GetResults() ([]EvaluationResult, error) {
 	if t.Err != nil {
 		return nil, t.Err