From 47eaf4bb8f3bf63d4b451ddf26dd909e127c1a1f Mon Sep 17 00:00:00 2001 From: Leland Batey Date: Sat, 22 Oct 2016 10:47:05 -0700 Subject: [PATCH 1/3] Implement protobuf package name extractor --- deftree/svcparse/scanner.go | 3 + truss/parsepkgname/parsepkgname.go | 111 ++++++++++++++++++++++++ truss/parsepkgname/parsepkgname_test.go | 89 +++++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100644 truss/parsepkgname/parsepkgname.go create mode 100644 truss/parsepkgname/parsepkgname_test.go diff --git a/deftree/svcparse/scanner.go b/deftree/svcparse/scanner.go index 98582b6..0d35431 100644 --- a/deftree/svcparse/scanner.go +++ b/deftree/svcparse/scanner.go @@ -301,6 +301,9 @@ func (self *SvcScanner) FastForward() error { return nil } +// ReadUnit returns the next "group" of runes found in the input stream. If the +// end of the stream is reached, io.EOF will be returned as error. No other +// errors will be returned. func (self *SvcScanner) ReadUnit() ([]rune, error) { var rv []rune var err error = nil diff --git a/truss/parsepkgname/parsepkgname.go b/truss/parsepkgname/parsepkgname.go new file mode 100644 index 0000000..5734c3a --- /dev/null +++ b/truss/parsepkgname/parsepkgname.go @@ -0,0 +1,111 @@ +/* +Package parsepkgname provides functions for extracting the name of a package +from a protocol buffer definition file. For example, given a protocol buffer 3 +file like this: + + // A comment about this proto file + package examplepackage; + + // and the rest of the file goes here + +The functions in this package would extract the name "examplepackage" as the +name of the protobuf package. +*/ +package parsepkgname + +import ( + "io" + "unicode" + + "github.com/TuneLab/go-truss/deftree/svcparse" +) + +type Token int + +const ( + IDENT Token = iota + WHITESPACE + COMMENT + OTHER +) + +type Scanner interface { + // ReadUnit must return groups of runes representing at least the following + // lexical groups: + // + // ident + // comments (c++ style single line comments and block comments) + // whitespace + // + // If you need a scanner which provides these out of the box, see the + // SvcScanner struct in github.com/TuneLab/go-truss/deftree/svcparse + ReadUnit() ([]rune, error) +} + +func categorize(unit []rune) Token { + rv := OTHER + r := unit[0] + switch { + case unicode.IsLetter(r): + rv = IDENT + case unicode.IsDigit(r): + rv = IDENT + case r == '_': + rv = IDENT + case unicode.IsSpace(r): + rv = WHITESPACE + case r == '/' && len(unit) > 1: + rv = COMMENT + } + return rv +} + +// PackageNameFromFile accepts an io.Reader, the contents of which should be a +// valid proto3 file, and returns the name of the protobuf package for that +// file. +func PackageNameFromFile(protofile io.Reader) (string, error) { + scanner := svcparse.NewSvcScanner(protofile) + return GetPackageName(scanner) +} + +// GetPackageName accepts a Scanner for a protobuf file and returns the name of +// the protobuf package that the file lives within. +func GetPackageName(scanner Scanner) (string, error) { + foundpackage := false + + // A nice way to ignore comments. Recursively calls itself until it + // recieves a unit from the scanner which is not a comment. + var readIgnoreComment func(Scanner) (Token, []rune, error) + readIgnoreComment = func(scn Scanner) (Token, []rune, error) { + unit, err := scanner.ReadUnit() + if err != nil { + return OTHER, nil, err + } + tkn := categorize(unit) + if tkn == COMMENT { + return readIgnoreComment(scn) + } + return tkn, unit, err + } + + for { + tkn, unit, err := readIgnoreComment(scanner) + // Err may only be io.EOF + if err != nil { + return "", err + } + if foundpackage { + if tkn == IDENT { + return string(unit), nil + } else if tkn == WHITESPACE { + continue + } else { + foundpackage = false + } + } else { + if tkn == IDENT && string(unit) == "package" { + foundpackage = true + } + } + } +} diff --git a/truss/parsepkgname/parsepkgname_test.go b/truss/parsepkgname/parsepkgname_test.go new file mode 100644 index 0000000..ea77cf0 --- /dev/null +++ b/truss/parsepkgname/parsepkgname_test.go @@ -0,0 +1,89 @@ +package parsepkgname + +import ( + "io" + "strings" + "testing" +) + +type testScanner struct { + contents [][]rune + position int +} + +func (t *testScanner) ReadUnit() ([]rune, error) { + if t.position < len(t.contents) { + rv := t.contents[t.position] + t.position += 1 + return rv, nil + } else { + return nil, io.EOF + } +} + +func NewTestScanner(units []string) *testScanner { + rv := testScanner{position: 0} + for _, u := range units { + rv.contents = append(rv.contents, []rune(u)) + } + return &rv +} + +func TestGetPackageName_simple(t *testing.T) { + basicContents := []string{ + "\n", + "package", + " ", + "examplename", + ";", + "\n", + } + scn := NewTestScanner(basicContents) + want := "examplename" + got, err := GetPackageName(scn) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("Got %q for package name, want %q", got, want) + } +} + +func TestGetPackageName_mid_comment(t *testing.T) { + contents := []string{ + "\n", + "package", + " ", + "/* comment in the middle of the declaration */", + "examplename", + ";", + "\n", + } + scn := NewTestScanner(contents) + want := "examplename" + got, err := GetPackageName(scn) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("Got %q for package name, want %q", got, want) + } +} + +func TestPackageNameFromFile(t *testing.T) { + code := ` +// A comment about this proto file +package /* some mid-definition comment */ examplepackage; + +// and the rest of the file goes here +` + name, err := PackageNameFromFile(strings.NewReader(code)) + if err != nil { + t.Fatal(err) + } + got := name + want := "examplepackage" + if got != want { + t.Fatalf("Got %q for package name, want %q", got, want) + } +} From cb84ced170d52e9073b7ae820333b8f9c3ef88ab Mon Sep 17 00:00:00 2001 From: Leland Batey Date: Sat, 22 Oct 2016 11:21:21 -0700 Subject: [PATCH 2/3] Consolidate two-step config creation into one step --- truss/main.go | 67 +++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/truss/main.go b/truss/main.go index 3dab491..f95e1b3 100644 --- a/truss/main.go +++ b/truss/main.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/TuneLab/go-truss/truss/execprotoc" + "github.com/TuneLab/go-truss/truss/parsepkgname" "github.com/TuneLab/go-truss/truss/truss" "github.com/TuneLab/go-truss/deftree" @@ -45,9 +46,6 @@ func main() { dt, err := parseServiceDefinition(cfg.DefPaths) exitIfError(errors.Wrap(err, "cannot parse input definition proto files")) - err = updateConfigWithService(cfg, dt) - exitIfError(err) - genFiles, err := generateCode(cfg, dt) exitIfError(errors.Wrap(err, "cannot generate service")) @@ -79,15 +77,37 @@ func parseInput() (*truss.Config, error) { return nil, errors.Wrap(err, "cannot parse input arguments") } - // PBGoPackage - if *pbPackageFlag == "" { - return &cfg, nil + // Service Path + defFile, err := os.Open(cfg.DefPaths[0]) + if err != nil { + return nil, errors.Wrapf(err, "Could not open package file %q", cfg.DefPaths[0]) + } + svcName, err := parsepkgname.PackageNameFromFile(defFile) + if err != nil { + return nil, errors.Wrapf(err, "cannot parse package name from file %q", cfg.DefPaths[0]) + } + svcFolderName := svcName + "-service" + svcPath := filepath.Join(filepath.Dir(cfg.DefPaths[0]), svcFolderName) + cfg.ServicePackage, err = filepath.Rel(filepath.Join(cfg.GOPATH, "src"), svcPath) + if err != nil { + return nil, errors.Wrap(err, "service path is not in GOPATH") } - cfg.PBPackage = *pbPackageFlag - if !fileExists( - filepath.Join(cfg.GOPATH, "src", cfg.PBPackage)) { - return nil, errors.Errorf(".pb.go output package directory does not exist: %q", cfg.PBPackage) + // PrevGen + cfg.PrevGen, err = readPreviousGeneration(cfg.ServicePath()) + if err != nil { + return nil, errors.Wrap(err, "cannot read previously generated files") + } + + // PBGoPackage + if *pbPackageFlag == "" { + cfg.PBPackage = cfg.ServicePackage + } else { + cfg.PBPackage = *pbPackageFlag + if !fileExists( + filepath.Join(cfg.GOPATH, "src", cfg.PBPackage)) { + return nil, errors.Errorf(".pb.go output package directory does not exist: %q", cfg.PBPackage) + } } return &cfg, nil @@ -114,33 +134,6 @@ func parseServiceDefinition(definitionPaths []string) (deftree.Deftree, error) { return dt, nil } -// updateConfigWithService updates the config with all information needed to -// generate a truss service using the parsedServiceDefinition deftree -func updateConfigWithService(cfg *truss.Config, dt deftree.Deftree) error { - var err error - - // Service Path - svcName := dt.GetName() + "-service" - svcPath := filepath.Join(filepath.Dir(cfg.DefPaths[0]), svcName) - cfg.ServicePackage, err = filepath.Rel(filepath.Join(cfg.GOPATH, "src"), svcPath) - if err != nil { - return errors.Wrap(err, "service path is not in GOPATH") - } - - // PrevGen - cfg.PrevGen, err = readPreviousGeneration(cfg.ServicePath()) - if err != nil { - return errors.Wrap(err, "cannot read previously generated files") - } - - // PBGoPath - if cfg.PBPackage == "" { - cfg.PBPackage = cfg.ServicePackage - } - - return nil -} - // generateCode returns a []truss.NamedReadWriter that represents a gokit // service with documentation func generateCode(cfg *truss.Config, dt deftree.Deftree) ([]truss.NamedReadWriter, error) { From dc580b847d6d505d125071a672efdd79878f19f9 Mon Sep 17 00:00:00 2001 From: Leland Batey Date: Sun, 23 Oct 2016 11:18:11 -0700 Subject: [PATCH 3/3] Implement creation of svcdef in truss --- deftree/svcparse/scanner.go | 4 +- truss/main.go | 89 ++++++++++++++++++------- truss/parsepkgname/parsepkgname.go | 60 +++++++++-------- truss/parsepkgname/parsepkgname_test.go | 15 ++--- truss/truss/config.go | 2 +- 5 files changed, 107 insertions(+), 63 deletions(-) diff --git a/deftree/svcparse/scanner.go b/deftree/svcparse/scanner.go index 0d35431..86597c3 100644 --- a/deftree/svcparse/scanner.go +++ b/deftree/svcparse/scanner.go @@ -19,7 +19,7 @@ type RuneReader struct { func (self *RuneReader) ReadRune() (rune, error) { var toret rune = 0 - var err error = nil + var err error if self.RunePos < self.ContentLen { toret = self.Contents[self.RunePos] @@ -306,7 +306,7 @@ func (self *SvcScanner) FastForward() error { // errors will be returned. func (self *SvcScanner) ReadUnit() ([]rune, error) { var rv []rune - var err error = nil + var err error if self.UnitPos < len(self.Buf) { unit := self.Buf[self.UnitPos] diff --git a/truss/main.go b/truss/main.go index f95e1b3..063bac0 100644 --- a/truss/main.go +++ b/truss/main.go @@ -18,6 +18,7 @@ import ( "github.com/TuneLab/go-truss/deftree" "github.com/TuneLab/go-truss/gendoc" "github.com/TuneLab/go-truss/gengokit" + "github.com/TuneLab/go-truss/svcdef" ) var ( @@ -43,7 +44,8 @@ func main() { cfg, err := parseInput() exitIfError(errors.Wrap(err, "cannot parse input")) - dt, err := parseServiceDefinition(cfg.DefPaths) + //dt, sd, err := parseServiceDefinition(cfg) + dt, _, err := parseServiceDefinition(cfg) exitIfError(errors.Wrap(err, "cannot parse input definition proto files")) genFiles, err := generateCode(cfg, dt) @@ -82,7 +84,7 @@ func parseInput() (*truss.Config, error) { if err != nil { return nil, errors.Wrapf(err, "Could not open package file %q", cfg.DefPaths[0]) } - svcName, err := parsepkgname.PackageNameFromFile(defFile) + svcName, err := parsepkgname.FromReader(defFile) if err != nil { return nil, errors.Wrapf(err, "cannot parse package name from file %q", cfg.DefPaths[0]) } @@ -113,41 +115,82 @@ func parseInput() (*truss.Config, error) { return &cfg, nil } -// parseServiceDefinition returns a deftree which contains all needed for all -// generating a truss service and documentation -func parseServiceDefinition(definitionPaths []string) (deftree.Deftree, error) { - protocOut, err := execprotoc.CodeGeneratorRequest(definitionPaths) - if err != nil { - return nil, errors.Wrap(err, "cannot use parse input files with protoc") +// parseServiceDefinition returns a deftree which contains all necessary +// information for generating a truss service and its documentation. +func parseServiceDefinition(cfg *truss.Config) (deftree.Deftree, *svcdef.Svcdef, error) { + svcPath := cfg.ServicePath() + protoDefPaths := cfg.DefPaths + // Create the ServicePath so the .pb.go files may be place within it + if cfg.PrevGen == nil { + err := os.Mkdir(svcPath, 0777) + if err != nil { + return nil, nil, errors.Wrap(err, "cannot create service directory") + } } - svcFile, err := execprotoc.ServiceFile(protocOut, filepath.Dir(definitionPaths[0])) + err := execprotoc.GeneratePBDotGo(cfg.DefPaths, svcPath, cfg.PBPath()) if err != nil { - return nil, errors.Wrap(err, "cannot find service definition file") + return nil, nil, errors.Wrap(err, "cannot create .pb.go files") + } + + // Open all .pb.go files and store in slice to be passed to svcdef.New() + //var openFiles func([]string) ([]io.Reader, error) + openFiles := func(paths []string) ([]io.Reader, error) { + rv := []io.Reader{} + for _, p := range paths { + reader, err := os.Open(p) + if err != nil { + return nil, errors.Wrapf(err, "couldn't open file %q", p) + } + rv = append(rv, reader) + } + return rv, nil + } + // Get path names of .pb.go files + pbgoPaths := []string{} + for _, p := range protoDefPaths { + base := filepath.Base(p) + barename := strings.TrimSuffix(base, filepath.Ext(p)) + pbgp := filepath.Join(cfg.PBPath(), barename+".pb.go") + pbgoPaths = append(pbgoPaths, pbgp) + } + pbgoFiles, err := openFiles(pbgoPaths) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to open a .pb.go file") + } + pbFiles, err := openFiles(protoDefPaths) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to open a .proto file") + } + + // Create the svcdef + sd, err := svcdef.New(pbgoFiles, pbFiles) + if err != nil { + return nil, nil, errors.Wrap(err, "Failed to create svcdef") + } + + // Create the Deftree + protocOut, err := execprotoc.CodeGeneratorRequest(protoDefPaths) + if err != nil { + return nil, nil, errors.Wrap(err, "cannot parse input files with protoc") + } + + svcFile, err := execprotoc.ServiceFile(protocOut, filepath.Dir(protoDefPaths[0])) + if err != nil { + return nil, nil, errors.Wrap(err, "cannot find service definition file") } dt, err := deftree.New(protocOut, svcFile) if err != nil { - return nil, errors.Wrap(err, "cannot to construct service definition") + return nil, nil, errors.Wrap(err, "cannot to construct service definition") } - return dt, nil + return dt, sd, nil } // generateCode returns a []truss.NamedReadWriter that represents a gokit // service with documentation func generateCode(cfg *truss.Config, dt deftree.Deftree) ([]truss.NamedReadWriter, error) { - if cfg.PrevGen == nil { - err := os.Mkdir(cfg.ServicePath(), 0777) - if err != nil { - return nil, errors.Wrap(err, "cannot create service directory") - } - } - - err := execprotoc.GeneratePBDotGo(cfg.DefPaths, cfg.ServicePath(), cfg.PBPath()) - if err != nil { - return nil, errors.Wrap(err, "cannot create .pb.go files") - } genGokitFiles, err := gengokit.GenerateGokit(dt, cfg.ServicePackage, cfg.PBPackage, cfg.PrevGen) if err != nil { diff --git a/truss/parsepkgname/parsepkgname.go b/truss/parsepkgname/parsepkgname.go index 5734c3a..8d14087 100644 --- a/truss/parsepkgname/parsepkgname.go +++ b/truss/parsepkgname/parsepkgname.go @@ -20,13 +20,13 @@ import ( "github.com/TuneLab/go-truss/deftree/svcparse" ) -type Token int +type token int const ( - IDENT Token = iota - WHITESPACE - COMMENT - OTHER + ident token = iota + whitespaceToken + comment + other ) type Scanner interface { @@ -42,68 +42,70 @@ type Scanner interface { ReadUnit() ([]rune, error) } -func categorize(unit []rune) Token { - rv := OTHER +func categorize(unit []rune) token { + rv := other r := unit[0] switch { case unicode.IsLetter(r): - rv = IDENT + rv = ident case unicode.IsDigit(r): - rv = IDENT + rv = ident case r == '_': - rv = IDENT + rv = ident case unicode.IsSpace(r): - rv = WHITESPACE + rv = whitespaceToken case r == '/' && len(unit) > 1: - rv = COMMENT + rv = comment } return rv } -// PackageNameFromFile accepts an io.Reader, the contents of which should be a -// valid proto3 file, and returns the name of the protobuf package for that -// file. -func PackageNameFromFile(protofile io.Reader) (string, error) { +// FromReader accepts an io.Reader, the contents of which should be a +// valid proto3 file, and returns the name of the protobuf package which that +// file belongs to. +func FromReader(protofile io.Reader) (string, error) { scanner := svcparse.NewSvcScanner(protofile) - return GetPackageName(scanner) + return FromScanner(scanner) } -// GetPackageName accepts a Scanner for a protobuf file and returns the name of -// the protobuf package that the file lives within. -func GetPackageName(scanner Scanner) (string, error) { +// FromScanner accepts a Scanner for a protobuf file and returns the name of +// the protobuf package that the file belongs to. +func FromScanner(scanner Scanner) (string, error) { foundpackage := false // A nice way to ignore comments. Recursively calls itself until it // recieves a unit from the scanner which is not a comment. - var readIgnoreComment func(Scanner) (Token, []rune, error) - readIgnoreComment = func(scn Scanner) (Token, []rune, error) { - unit, err := scanner.ReadUnit() + var readIgnoreComment func(Scanner) (token, []rune, error) + readIgnoreComment = func(scn Scanner) (token, []rune, error) { + unit, err := scn.ReadUnit() if err != nil { - return OTHER, nil, err + return other, nil, err } tkn := categorize(unit) - if tkn == COMMENT { + if tkn == comment { return readIgnoreComment(scn) } return tkn, unit, err } + // A tiny state machine to find two sequential idents: the ident "package" + // and the ident immediately following. That second ident will be the name + // of the package. for { tkn, unit, err := readIgnoreComment(scanner) - // Err may only be io.EOF if err != nil { return "", err } if foundpackage { - if tkn == IDENT { + if tkn == ident { return string(unit), nil - } else if tkn == WHITESPACE { + } else if tkn == whitespaceToken { continue } else { foundpackage = false } } else { - if tkn == IDENT && string(unit) == "package" { + if tkn == ident && string(unit) == "package" { foundpackage = true } } diff --git a/truss/parsepkgname/parsepkgname_test.go b/truss/parsepkgname/parsepkgname_test.go index ea77cf0..65f3c13 100644 --- a/truss/parsepkgname/parsepkgname_test.go +++ b/truss/parsepkgname/parsepkgname_test.go @@ -16,9 +16,8 @@ func (t *testScanner) ReadUnit() ([]rune, error) { rv := t.contents[t.position] t.position += 1 return rv, nil - } else { - return nil, io.EOF } + return nil, io.EOF } func NewTestScanner(units []string) *testScanner { @@ -29,7 +28,7 @@ func NewTestScanner(units []string) *testScanner { return &rv } -func TestGetPackageName_simple(t *testing.T) { +func TestFromScanner_simple(t *testing.T) { basicContents := []string{ "\n", "package", @@ -40,7 +39,7 @@ func TestGetPackageName_simple(t *testing.T) { } scn := NewTestScanner(basicContents) want := "examplename" - got, err := GetPackageName(scn) + got, err := FromScanner(scn) if err != nil { t.Fatal(err) } @@ -49,7 +48,7 @@ func TestGetPackageName_simple(t *testing.T) { } } -func TestGetPackageName_mid_comment(t *testing.T) { +func TestFromScanner_mid_comment(t *testing.T) { contents := []string{ "\n", "package", @@ -61,7 +60,7 @@ func TestGetPackageName_mid_comment(t *testing.T) { } scn := NewTestScanner(contents) want := "examplename" - got, err := GetPackageName(scn) + got, err := FromScanner(scn) if err != nil { t.Fatal(err) } @@ -70,14 +69,14 @@ func TestGetPackageName_mid_comment(t *testing.T) { } } -func TestPackageNameFromFile(t *testing.T) { +func TestFromReader(t *testing.T) { code := ` // A comment about this proto file package /* some mid-definition comment */ examplepackage; // and the rest of the file goes here ` - name, err := PackageNameFromFile(strings.NewReader(code)) + name, err := FromReader(strings.NewReader(code)) if err != nil { t.Fatal(err) } diff --git a/truss/truss/config.go b/truss/truss/config.go index 1cf2b34..55a4309 100644 --- a/truss/truss/config.go +++ b/truss/truss/config.go @@ -27,7 +27,7 @@ func (c *Config) ServicePath() string { return goSvcPath } -// PBPath returns the full paht to Config.PBPackage +// PBPath returns the full path to Config.PBPackage func (c *Config) PBPath() string { pbPath := filepath.Join(c.GOPATH, "src", c.PBPackage)