Merge pull request #74 from hasLeland/pipeline-migration

Pipeline migration -- Part 1

deftree/svcparse/scanner.go

@@ -19,7 +19,7 @@ type RuneReader struct {
 
 func (self *RuneReader) ReadRune() (rune, error) {
 	var toret rune = 0
-	var err error = nil
+	var err error
 
 	if self.RunePos < self.ContentLen {
 		toret = self.Contents[self.RunePos]
@@ -301,9 +301,12 @@ func (self *SvcScanner) FastForward() error {
 	return nil
 }
 
+// ReadUnit returns the next "group" of runes found in the input stream. If the
+// end of the stream is reached, io.EOF will be returned as error. No other
+// errors will be returned.
 func (self *SvcScanner) ReadUnit() ([]rune, error) {
 	var rv []rune
-	var err error = nil
+	var err error
 	if self.UnitPos < len(self.Buf) {
 		unit := self.Buf[self.UnitPos]
 

truss/main.go

@@ -12,11 +12,13 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/TuneLab/go-truss/truss/execprotoc"
+	"github.com/TuneLab/go-truss/truss/parsepkgname"
 	"github.com/TuneLab/go-truss/truss/truss"
 
 	"github.com/TuneLab/go-truss/deftree"
 	"github.com/TuneLab/go-truss/gendoc"
 	"github.com/TuneLab/go-truss/gengokit"
+	"github.com/TuneLab/go-truss/svcdef"
 )
 
 var (
@@ -42,12 +44,10 @@ func main() {
 	cfg, err := parseInput()
 	exitIfError(errors.Wrap(err, "cannot parse input"))
 
-	dt, err := parseServiceDefinition(cfg.DefPaths)
+	//dt, sd, err := parseServiceDefinition(cfg)
+	dt, _, err := parseServiceDefinition(cfg)
 	exitIfError(errors.Wrap(err, "cannot parse input definition proto files"))
 
-	err = updateConfigWithService(cfg, dt)
-	exitIfError(err)
-
 	genFiles, err := generateCode(cfg, dt)
 	exitIfError(errors.Wrap(err, "cannot generate service"))
 
@@ -79,82 +79,118 @@ func parseInput() (*truss.Config, error) {
 		return nil, errors.Wrap(err, "cannot parse input arguments")
 	}
 
-	// PBGoPackage
-	if *pbPackageFlag == "" {
-		return &cfg, nil
-	}
-
-	cfg.PBPackage = *pbPackageFlag
-	if !fileExists(
-		filepath.Join(cfg.GOPATH, "src", cfg.PBPackage)) {
-		return nil, errors.Errorf(".pb.go output package directory does not exist: %q", cfg.PBPackage)
-	}
-
-	return &cfg, nil
-}
-
-// parseServiceDefinition returns a deftree which contains all needed for all
-// generating a truss service and documentation
-func parseServiceDefinition(definitionPaths []string) (deftree.Deftree, error) {
-	protocOut, err := execprotoc.CodeGeneratorRequest(definitionPaths)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot use parse input files with protoc")
-	}
-
-	svcFile, err := execprotoc.ServiceFile(protocOut, filepath.Dir(definitionPaths[0]))
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot find service definition file")
-	}
-
-	dt, err := deftree.New(protocOut, svcFile)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot to construct service definition")
-	}
-
-	return dt, nil
-}
-
-// updateConfigWithService updates the config with all information needed to
-// generate a truss service using the parsedServiceDefinition deftree
-func updateConfigWithService(cfg *truss.Config, dt deftree.Deftree) error {
-	var err error
-
 	// Service Path
-	svcName := dt.GetName() + "-service"
+	defFile, err := os.Open(cfg.DefPaths[0])
-	svcPath := filepath.Join(filepath.Dir(cfg.DefPaths[0]), svcName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Could not open package file %q", cfg.DefPaths[0])
+	}
+	svcName, err := parsepkgname.FromReader(defFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot parse package name from file %q", cfg.DefPaths[0])
+	}
+	svcFolderName := svcName + "-service"
+	svcPath := filepath.Join(filepath.Dir(cfg.DefPaths[0]), svcFolderName)
 	cfg.ServicePackage, err = filepath.Rel(filepath.Join(cfg.GOPATH, "src"), svcPath)
 	if err != nil {
-		return errors.Wrap(err, "service path is not in GOPATH")
+		return nil, errors.Wrap(err, "service path is not in GOPATH")
 	}
 
 	// PrevGen
 	cfg.PrevGen, err = readPreviousGeneration(cfg.ServicePath())
 	if err != nil {
-		return errors.Wrap(err, "cannot read previously generated files")
+		return nil, errors.Wrap(err, "cannot read previously generated files")
 	}
 
-	// PBGoPath
+	// PBGoPackage
-	if cfg.PBPackage == "" {
+	if *pbPackageFlag == "" {
 		cfg.PBPackage = cfg.ServicePackage
+	} else {
+		cfg.PBPackage = *pbPackageFlag
+		if !fileExists(
+			filepath.Join(cfg.GOPATH, "src", cfg.PBPackage)) {
+			return nil, errors.Errorf(".pb.go output package directory does not exist: %q", cfg.PBPackage)
+		}
 	}
 
-	return nil
+	return &cfg, nil
+}
+
+// parseServiceDefinition returns a deftree which contains all necessary
+// information for generating a truss service and its documentation.
+func parseServiceDefinition(cfg *truss.Config) (deftree.Deftree, *svcdef.Svcdef, error) {
+	svcPath := cfg.ServicePath()
+	protoDefPaths := cfg.DefPaths
+	// Create the ServicePath so the .pb.go files may be place within it
+	if cfg.PrevGen == nil {
+		err := os.Mkdir(svcPath, 0777)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "cannot create service directory")
+		}
+	}
+
+	err := execprotoc.GeneratePBDotGo(cfg.DefPaths, svcPath, cfg.PBPath())
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot create .pb.go files")
+	}
+
+	// Open all .pb.go files and store in slice to be passed to svcdef.New()
+	//var openFiles func([]string) ([]io.Reader, error)
+	openFiles := func(paths []string) ([]io.Reader, error) {
+		rv := []io.Reader{}
+		for _, p := range paths {
+			reader, err := os.Open(p)
+			if err != nil {
+				return nil, errors.Wrapf(err, "couldn't open file %q", p)
+			}
+			rv = append(rv, reader)
+		}
+		return rv, nil
+	}
+	// Get path names of .pb.go files
+	pbgoPaths := []string{}
+	for _, p := range protoDefPaths {
+		base := filepath.Base(p)
+		barename := strings.TrimSuffix(base, filepath.Ext(p))
+		pbgp := filepath.Join(cfg.PBPath(), barename+".pb.go")
+		pbgoPaths = append(pbgoPaths, pbgp)
+	}
+	pbgoFiles, err := openFiles(pbgoPaths)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed to open a .pb.go file")
+	}
+	pbFiles, err := openFiles(protoDefPaths)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed to open a .proto file")
+	}
+
+	// Create the svcdef
+	sd, err := svcdef.New(pbgoFiles, pbFiles)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed to create svcdef")
+	}
+
+	// Create the Deftree
+	protocOut, err := execprotoc.CodeGeneratorRequest(protoDefPaths)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot parse input files with protoc")
+	}
+
+	svcFile, err := execprotoc.ServiceFile(protocOut, filepath.Dir(protoDefPaths[0]))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot find service definition file")
+	}
+
+	dt, err := deftree.New(protocOut, svcFile)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot to construct service definition")
+	}
+
+	return dt, sd, nil
 }
 
 // generateCode returns a []truss.NamedReadWriter that represents a gokit
 // service with documentation
 func generateCode(cfg *truss.Config, dt deftree.Deftree) ([]truss.NamedReadWriter, error) {
-	if cfg.PrevGen == nil {
-		err := os.Mkdir(cfg.ServicePath(), 0777)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot create service directory")
-		}
-	}
-
-	err := execprotoc.GeneratePBDotGo(cfg.DefPaths, cfg.ServicePath(), cfg.PBPath())
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot create .pb.go files")
-	}
 
 	genGokitFiles, err := gengokit.GenerateGokit(dt, cfg.ServicePackage, cfg.PBPackage, cfg.PrevGen)
 	if err != nil {

truss/parsepkgname/parsepkgname.go

@@ -0,0 +1,113 @@
+/*
+Package parsepkgname provides functions for extracting the name of a package
+from a protocol buffer definition file. For example, given a protocol buffer 3
+file like this:
+
+	// A comment about this proto file
+	package  examplepackage;
+
+	// and the rest of the file goes here
+
+The functions in this package would extract the name "examplepackage" as the
+name of the protobuf package.
+*/
+package parsepkgname
+
+import (
+	"io"
+	"unicode"
+
+	"github.com/TuneLab/go-truss/deftree/svcparse"
+)
+
+type token int
+
+const (
+	ident token = iota
+	whitespaceToken
+	comment
+	other
+)
+
+type Scanner interface {
+	// ReadUnit must return groups of runes representing at least the following
+	// lexical groups:
+	//
+	//     ident
+	//     comments (c++ style single line comments and block comments)
+	//     whitespace
+	//
+	// If you need a scanner which provides these out of the box, see the
+	// SvcScanner struct in github.com/TuneLab/go-truss/deftree/svcparse
+	ReadUnit() ([]rune, error)
+}
+
+func categorize(unit []rune) token {
+	rv := other
+	r := unit[0]
+	switch {
+	case unicode.IsLetter(r):
+		rv = ident
+	case unicode.IsDigit(r):
+		rv = ident
+	case r == '_':
+		rv = ident
+	case unicode.IsSpace(r):
+		rv = whitespaceToken
+	case r == '/' && len(unit) > 1:
+		rv = comment
+	}
+	return rv
+}
+
+// FromReader accepts an io.Reader, the contents of which should be a
+// valid proto3 file, and returns the name of the protobuf package which that
+// file belongs to.
+func FromReader(protofile io.Reader) (string, error) {
+	scanner := svcparse.NewSvcScanner(protofile)
+	return FromScanner(scanner)
+}
+
+// FromScanner accepts a Scanner for a protobuf file and returns the name of
+// the protobuf package that the file belongs to.
+func FromScanner(scanner Scanner) (string, error) {
+	foundpackage := false
+
+	// A nice way to ignore comments. Recursively calls itself until it
+	// recieves a unit from the scanner which is not a comment.
+	var readIgnoreComment func(Scanner) (token, []rune, error)
+	readIgnoreComment = func(scn Scanner) (token, []rune, error) {
+		unit, err := scn.ReadUnit()
+		if err != nil {
+			return other, nil, err
+		}
+		tkn := categorize(unit)
+		if tkn == comment {
+			return readIgnoreComment(scn)
+		}
+		return tkn, unit, err
+	}
+
+	// A tiny state machine to find two sequential idents: the ident "package"
+	// and the ident immediately following. That second ident will be the name
+	// of the package.
+	for {
+		tkn, unit, err := readIgnoreComment(scanner)
+		if err != nil {
+			return "", err
+		}
+		if foundpackage {
+			if tkn == ident {
+				return string(unit), nil
+			} else if tkn == whitespaceToken {
+				continue
+			} else {
+				foundpackage = false
+			}
+		} else {
+			if tkn == ident && string(unit) == "package" {
+				foundpackage = true
+			}
+		}
+	}
+}

truss/parsepkgname/parsepkgname_test.go

@@ -0,0 +1,88 @@
+package parsepkgname
+
+import (
+	"io"
+	"strings"
+	"testing"
+)
+
+type testScanner struct {
+	contents [][]rune
+	position int
+}
+
+func (t *testScanner) ReadUnit() ([]rune, error) {
+	if t.position < len(t.contents) {
+		rv := t.contents[t.position]
+		t.position += 1
+		return rv, nil
+	}
+	return nil, io.EOF
+}
+
+func NewTestScanner(units []string) *testScanner {
+	rv := testScanner{position: 0}
+	for _, u := range units {
+		rv.contents = append(rv.contents, []rune(u))
+	}
+	return &rv
+}
+
+func TestFromScanner_simple(t *testing.T) {
+	basicContents := []string{
+		"\n",
+		"package",
+		" ",
+		"examplename",
+		";",
+		"\n",
+	}
+	scn := NewTestScanner(basicContents)
+	want := "examplename"
+	got, err := FromScanner(scn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Fatalf("Got %q for package name, want %q", got, want)
+	}
+}
+
+func TestFromScanner_mid_comment(t *testing.T) {
+	contents := []string{
+		"\n",
+		"package",
+		" ",
+		"/* comment in the middle of the declaration */",
+		"examplename",
+		";",
+		"\n",
+	}
+	scn := NewTestScanner(contents)
+	want := "examplename"
+	got, err := FromScanner(scn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Fatalf("Got %q for package name, want %q", got, want)
+	}
+}
+
+func TestFromReader(t *testing.T) {
+	code := `
+// A comment about this proto file
+package /* some mid-definition comment */ examplepackage;
+
+// and the rest of the file goes here
+`
+	name, err := FromReader(strings.NewReader(code))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := name
+	want := "examplepackage"
+	if got != want {
+		t.Fatalf("Got %q for package name, want %q", got, want)
+	}
+}

truss/truss/config.go

@@ -27,7 +27,7 @@ func (c *Config) ServicePath() string {
 	return goSvcPath
 }
 
-// PBPath returns the full paht to Config.PBPackage
+// PBPath returns the full path to Config.PBPackage
 func (c *Config) PBPath() string {
 	pbPath := filepath.Join(c.GOPATH, "src", c.PBPackage)