Merge pull request #815 from kyleconroy/govendor-importer

Add importer for govendor

CHANGELOG.md

@@ -1,5 +1,9 @@
 # v0.3.3 (Unreleased)
 
+NEW FEATURES:
+* Add support for importing from [govendor](https://github.com/kardianos/govendor)
+  based projects (#815).
+
 BUG FIXES:
 
 * Releases targeting Windows now have a `.exe` suffix (#1291).

cmd/dep/init.go

@@ -30,7 +30,7 @@ specified, use the current directory.
 When configuration for another dependency management tool is detected, it is
 imported into the initial manifest and lock. Use the -skip-tools flag to
 disable this behavior. The following external tools are supported:
-glide, godep, vndr, govend, gb, gvt.
+glide, godep, vndr, govend, gb, gvt, govendor.
 
 Any dependencies that are not constrained by external configuration use the
 GOPATH analysis below.

cmd/dep/testdata/harness_tests/init/govendor/case1/README.md

@@ -0,0 +1 @@
+Import govendor config in vendor dir.

cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.lock

@@ -0,0 +1,21 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "github.com/sdboyer/deptest"
+  packages = ["."]
+  revision = "3f4c3bea144e112a69bbe5d8d01c1b09a544253f"
+  version = "v0.8.1"
+
+[[projects]]
+  name = "github.com/sdboyer/deptestdos"
+  packages = ["."]
+  revision = "5c607206be5decd28e6263ffffdcee067266015e"
+  version = "v2.0.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "9cc662f2e1b80c8df205d9d667fe2c47825a06961ceae378f44a8290d01dd359"
+  solver-name = "gps-cdcl"
+  solver-version = 1

cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml

@@ -0,0 +1,5 @@
+ignored = ["github.com/golang/notexist/samples*","github.com/sdboyer/dep-test*"]
+
+[[constraint]]
+  name = "github.com/sdboyer/deptestdos"
+  version = "2.0.0"

cmd/dep/testdata/harness_tests/init/govendor/case1/initial/main.go

@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/sdboyer/deptestdos"
+)
+
+func main() {
+	var x deptestdos.Bar
+	fmt.Println(x)
+}

cmd/dep/testdata/harness_tests/init/govendor/case1/initial/samples/samples.go

@@ -0,0 +1,12 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package samples
+
+import dt "github.com/carolynvs/go-dep-test"
+
+func Sample1() int {
+	var x = dt.Thing
+	return x
+}

cmd/dep/testdata/harness_tests/init/govendor/case1/initial/vendor/vendor.json

@@ -0,0 +1,21 @@
+{
+	"comment": "",
+	"ignore": "test github.com/sdboyer/dep-test samples/",
+	"package": [
+		{
+			"checksumSHA1": "4R6TQcq0/gI/I2kKeUunuO/pEec=",
+			"path": "github.com/sdboyer/deptest",
+			"revision": "3f4c3bea144e112a69bbe5d8d01c1b09a544253f",
+			"revisionTime": "2017-02-22T03:31:47Z"
+		},
+		{
+			"checksumSHA1": "96YwrJjpE07ENey/eDWWnCWKQOw=",
+			"path": "github.com/sdboyer/deptestdos",
+			"revision": "5c607206be5decd28e6263ffffdcee067266015e",
+			"revisionTime": "2017-02-22T03:34:58Z",
+			"version": "v2",
+			"versionExact": "v2.0.0"
+		}
+	],
+	"rootPath": "github.com/golang/notexist"
+}

cmd/dep/testdata/harness_tests/init/govendor/case1/testcase.json

@@ -0,0 +1,14 @@
+{
+  "commands": [
+    ["init", "-no-examples", "-gopath"]
+  ],
+  "error-expected": "",
+  "gopath-initial": {
+    "github.com/sdboyer/deptest": "3f4c3bea144e112a69bbe5d8d01c1b09a544253f",
+    "github.com/sdboyer/deptestdos": "5c607206be5decd28e6263ffffdcee067266015e"
+  },
+  "vendor-final": [
+    "github.com/sdboyer/deptest",
+    "github.com/sdboyer/deptestdos"
+  ]
+}

docs/FAQ.md

@@ -217,7 +217,7 @@ about what's going on.
 During `dep init` configuration from other dependency managers is detected
 and imported, unless `-skip-tools` is specified.
 
-The following tools are supported: `glide`, `godep`, `vndr`, `govend`, `gb` and `gvt`.
+The following tools are supported: `glide`, `godep`, `vndr`, `govend`, `gb`, `gvt` and `govendor`.
 
 See [#186](https://github.com/golang/dep/issues/186#issuecomment-306363441) for
 how to add support for another tool.

internal/importers/base/importer.go

@@ -6,6 +6,7 @@ package base
 
 import (
 	"log"
+	"strings"
 
 	"github.com/golang/dep"
 	"github.com/golang/dep/gps"
@@ -16,28 +17,27 @@ import (
 // Importer provides a common implementation for importing from other
 // dependency managers.
 type Importer struct {
-	sm gps.SourceManager
-
-	Logger   *log.Logger
-	Verbose  bool
-	Manifest *dep.Manifest
-	Lock     *dep.Lock
+	SourceManager gps.SourceManager
+	Logger        *log.Logger
+	Verbose       bool
+	Manifest      *dep.Manifest
+	Lock          *dep.Lock
 }
 
 // NewImporter creates a new Importer for embedding in an importer.
 func NewImporter(logger *log.Logger, verbose bool, sm gps.SourceManager) *Importer {
 	return &Importer{
-		Logger:   logger,
-		Verbose:  verbose,
-		Manifest: dep.NewManifest(),
-		Lock:     &dep.Lock{},
-		sm:       sm,
+		Logger:        logger,
+		Verbose:       verbose,
+		Manifest:      dep.NewManifest(),
+		Lock:          &dep.Lock{},
+		SourceManager: sm,
 	}
 }
 
 // isTag determines if the specified value is a tag (plain or semver).
 func (i *Importer) isTag(pi gps.ProjectIdentifier, value string) (bool, gps.Version, error) {
-	versions, err := i.sm.ListVersions(pi)
+	versions, err := i.SourceManager.ListVersions(pi)
 	if err != nil {
 		return false, nil, errors.Wrapf(err, "unable to list versions for %s(%s)", pi.ProjectRoot, pi.Source)
 	}
@@ -61,7 +61,7 @@ func (i *Importer) isTag(pi gps.ProjectIdentifier, value string) (bool, gps.Vers
 // manifest, then finally the revision.
 func (i *Importer) lookupVersionForLockedProject(pi gps.ProjectIdentifier, c gps.Constraint, rev gps.Revision) (gps.Version, error) {
 	// Find the version that goes with this revision, if any
-	versions, err := i.sm.ListVersions(pi)
+	versions, err := i.SourceManager.ListVersions(pi)
 	if err != nil {
 		return rev, errors.Wrapf(err, "Unable to lookup the version represented by %s in %s(%s). Falling back to locking the revision only.", rev, pi.ProjectRoot, pi.Source)
 	}
@@ -130,7 +130,7 @@ func (i *Importer) loadPackages(packages []ImportedPackage) ([]importedProject,
 
 	projects := make(map[gps.ProjectRoot]*importedProject, len(packages))
 	for _, pkg := range packages {
-		pr, err := i.sm.DeduceProjectRoot(pkg.Name)
+		pr, err := i.SourceManager.DeduceProjectRoot(pkg.Name)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Cannot determine the project root for %s", pkg.Name)
 		}
@@ -188,6 +188,9 @@ func (i *Importer) ImportPackages(packages []ImportedPackage, defaultConstraintF
 				source = ""
 			} else if isDefault {
 				source = ""
+			} else if strings.Contains(source, "/vendor/") {
+				i.Logger.Printf("  Ignoring imported source %s for %s because vendored sources aren't supported", source, prj.Root)
+				source = ""
 			}
 		}
 
@@ -198,7 +201,7 @@ func (i *Importer) ImportPackages(packages []ImportedPackage, defaultConstraintF
 			},
 		}
 
-		pc.Constraint, err = i.sm.InferConstraint(prj.ConstraintHint, pc.Ident)
+		pc.Constraint, err = i.SourceManager.InferConstraint(prj.ConstraintHint, pc.Ident)
 		if err != nil {
 			pc.Constraint = gps.Any()
 		}
@@ -306,12 +309,12 @@ func (i *Importer) convertToConstraint(v gps.Version) gps.Constraint {
 func (i *Importer) isDefaultSource(projectRoot gps.ProjectRoot, sourceURL string) (bool, error) {
 	// this condition is mainly for gopkg.in imports,
 	// as some importers specify the repository url as https://gopkg.in/...,
-	// but sm.SourceURLsForPath() returns https://github.com/... urls for gopkg.in
+	// but SourceManager.SourceURLsForPath() returns https://github.com/... urls for gopkg.in
 	if sourceURL == "https://"+string(projectRoot) {
 		return true, nil
 	}
 
-	sourceURLs, err := i.sm.SourceURLsForPath(string(projectRoot))
+	sourceURLs, err := i.SourceManager.SourceURLsForPath(string(projectRoot))
 	if err != nil {
 		return false, err
 	}

internal/importers/base/importer_test.go

@@ -382,7 +382,7 @@ func TestBaseImporter_ImportProjects(t *testing.T) {
 				},
 			},
 		},
-		"ignoring default source": {
+		"ignore default source": {
 			importertest.TestCase{
 				WantConstraint: "*",
 				WantSourceRepo: "",
@@ -394,6 +394,19 @@ func TestBaseImporter_ImportProjects(t *testing.T) {
 				},
 			},
 		},
+		"ignore vendored source": {
+			importertest.TestCase{
+				WantConstraint: "*",
+				WantSourceRepo: "",
+				WantWarning:    "vendored sources aren't supported",
+			},
+			[]ImportedPackage{
+				{
+					Name:   importertest.Project,
+					Source: "example.com/vendor/" + importertest.Project,
+				},
+			},
+		},
 	}
 
 	for name, tc := range testcases {

internal/importers/govendor/importer.go

@@ -0,0 +1,150 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package govendor
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/dep"
+	"github.com/golang/dep/gps"
+	"github.com/golang/dep/internal/importers/base"
+	"github.com/pkg/errors"
+)
+
+const govendorDir = "vendor"
+const govendorName = "vendor.json"
+
+// Importer imports govendor configuration into the dep configuration format.
+type Importer struct {
+	*base.Importer
+
+	file govendorFile
+}
+
+// NewImporter for govendor.
+func NewImporter(logger *log.Logger, verbose bool, sm gps.SourceManager) *Importer {
+	return &Importer{Importer: base.NewImporter(logger, verbose, sm)}
+}
+
+// File is the structure of the vendor file.
+type govendorFile struct {
+	RootPath string // Import path of vendor folder
+	Ignore   string
+	Package  []*govendorPackage
+}
+
+// Package represents each package.
+type govendorPackage struct {
+	// See the vendor spec for definitions.
+	Origin   string
+	Path     string
+	Revision string
+	Version  string
+}
+
+// Name of the importer.
+func (g *Importer) Name() string {
+	return "govendor"
+}
+
+// HasDepMetadata checks if a directory contains config that the importer can handle.
+func (g *Importer) HasDepMetadata(dir string) bool {
+	y := filepath.Join(dir, govendorDir, govendorName)
+	if _, err := os.Stat(y); err != nil {
+		return false
+	}
+	return true
+}
+
+// Import the config found in the directory.
+func (g *Importer) Import(dir string, pr gps.ProjectRoot) (*dep.Manifest, *dep.Lock, error) {
+	err := g.load(dir)
+	if err != nil {
+		return nil, nil, err
+	}
+	return g.convert(pr)
+}
+
+func (g *Importer) load(projectDir string) error {
+	g.Logger.Println("Detected govendor configuration file...")
+	v := filepath.Join(projectDir, govendorDir, govendorName)
+	if g.Verbose {
+		g.Logger.Printf("  Loading %s", v)
+	}
+	vb, err := ioutil.ReadFile(v)
+	if err != nil {
+		return errors.Wrapf(err, "unable to read %s", v)
+	}
+	err = json.Unmarshal(vb, &g.file)
+	if err != nil {
+		return errors.Wrapf(err, "unable to parse %s", v)
+	}
+	return nil
+}
+
+func (g *Importer) convert(pr gps.ProjectRoot) (*dep.Manifest, *dep.Lock, error) {
+	g.Logger.Println("Converting from vendor.json...")
+
+	packages := make([]base.ImportedPackage, 0, len(g.file.Package))
+	for _, pkg := range g.file.Package {
+		// Path must not be empty
+		if pkg.Path == "" {
+			err := errors.New("invalid govendor configuration, Path is required")
+			return nil, nil, err
+		}
+
+		// There are valid govendor configs in the wild that don't have a revision set
+		// so we are not requiring it to be set during import
+
+		ip := base.ImportedPackage{
+			Name:     pkg.Path,
+			Source:   pkg.Origin,
+			LockHint: pkg.Revision,
+		}
+		packages = append(packages, ip)
+	}
+
+	err := g.ImportPackages(packages, true)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if len(g.file.Ignore) > 0 {
+		// Govendor has three use cases here
+		// 1. 'test' - special case for ignoring test files
+		// 2. build tags - any string without a slash (/) in it
+		// 3. path and path prefix - any string with a slash (/) in it.
+		//   The path case could be a full path or just a prefix.
+		// Dep doesn't support build tags right now: https://github.com/golang/dep/issues/120
+		for _, i := range strings.Split(g.file.Ignore, " ") {
+			if !strings.Contains(i, "/") {
+				g.Logger.Printf("  Govendor was configured to ignore the %s build tag, but that isn't supported by dep yet, and will be ignored. See https://github.com/golang/dep/issues/291.", i)
+				continue
+			}
+
+			var ignorePattern string
+			_, err := g.SourceManager.DeduceProjectRoot(i)
+			if err == nil { // external package
+				ignorePattern = i
+			} else { // relative package path in the current project
+				ignorePattern = path.Join(string(pr), i)
+			}
+
+			// Convert to a a wildcard ignore
+			ignorePattern = strings.TrimRight(ignorePattern, "/")
+			ignorePattern += "*"
+
+			g.Manifest.Ignored = append(g.Manifest.Ignored, ignorePattern)
+		}
+	}
+
+	return g.Manifest, g.Lock, nil
+}

internal/importers/govendor/importer_test.go

@@ -0,0 +1,155 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package govendor
+
+import (
+	"bytes"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"github.com/golang/dep"
+	"github.com/golang/dep/gps"
+	"github.com/golang/dep/internal/importers/importertest"
+	"github.com/golang/dep/internal/test"
+	"github.com/pkg/errors"
+)
+
+const testGovendorProjectRoot = "github.com/golang/notexist"
+
+func TestGovendorConfig_Import(t *testing.T) {
+	h := test.NewHelper(t)
+	defer h.Cleanup()
+
+	ctx := importertest.NewTestContext(h)
+	sm, err := ctx.SourceManager()
+	h.Must(err)
+	defer sm.Release()
+
+	h.TempDir(filepath.Join("src", testGovendorProjectRoot))
+	h.TempCopy(filepath.Join(testGovendorProjectRoot, govendorDir, govendorName), "vendor.json")
+	projectRoot := h.Path(testGovendorProjectRoot)
+
+	// Capture stderr so we can verify output
+	verboseOutput := &bytes.Buffer{}
+	ctx.Err = log.New(verboseOutput, "", 0)
+
+	g := NewImporter(ctx.Err, false, sm) // Disable verbose so that we don't print values that change each test run
+	if !g.HasDepMetadata(projectRoot) {
+		t.Fatal("Expected the importer to detect the govendor configuration files")
+	}
+
+	m, l, err := g.Import(projectRoot, testGovendorProjectRoot)
+	h.Must(err)
+
+	if m == nil {
+		t.Fatal("Expected the manifest to be generated")
+	}
+
+	if l == nil {
+		t.Fatal("Expected the lock to be generated")
+	}
+
+	goldenFile := "golden.txt"
+	got := verboseOutput.String()
+	want := h.GetTestFileString(goldenFile)
+	if want != got {
+		if *test.UpdateGolden {
+			if err := h.WriteTestFile(goldenFile, got); err != nil {
+				t.Fatalf("%+v", errors.Wrapf(err, "Unable to write updated golden file %s", goldenFile))
+			}
+		} else {
+			t.Fatalf("expected %s, got %s", want, got)
+		}
+	}
+}
+
+func TestGovendorConfig_Convert(t *testing.T) {
+	testCases := map[string]struct {
+		file govendorFile
+		importertest.TestCase
+	}{
+		"project": {
+			govendorFile{
+				Package: []*govendorPackage{
+					{
+						Path:     importertest.Project,
+						Origin:   importertest.ProjectSrc,
+						Revision: importertest.V1Rev,
+					},
+				},
+			},
+			importertest.TestCase{
+				WantSourceRepo: importertest.ProjectSrc,
+				WantConstraint: importertest.V1Constraint,
+				WantRevision:   importertest.V1Rev,
+				WantVersion:    importertest.V1Tag,
+			},
+		},
+		"skipped build tags": {
+			govendorFile{
+				Ignore: "test linux_amd64",
+			},
+			importertest.TestCase{
+				WantIgnored: nil,
+			},
+		},
+		"ignored external package": {
+			govendorFile{
+				Ignore: "github.com/sdboyer/deptest k8s.io/apimachinery",
+			},
+			importertest.TestCase{
+				WantIgnored: []string{"github.com/sdboyer/deptest*", "k8s.io/apimachinery*"},
+			},
+		},
+		"ignored internal package": {
+			govendorFile{
+				Ignore: "samples/ foo/bar",
+			},
+			importertest.TestCase{
+				WantIgnored: []string{importertest.RootProject + "/samples*", importertest.RootProject + "/foo/bar*"},
+			},
+		},
+		"missing package path": {
+			govendorFile{
+				Package: []*govendorPackage{
+					{
+						Revision: importertest.V2PatchRev,
+					},
+				},
+			},
+			importertest.TestCase{
+				WantConvertErr: true,
+			},
+		},
+		"missing package revision doesn't cause an error": {
+			govendorFile{
+				Package: []*govendorPackage{
+					{
+						Path: importertest.Project,
+					},
+				},
+			},
+			importertest.TestCase{
+				WantConstraint: "*",
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		name := name
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			err := testCase.Execute(t, func(logger *log.Logger, sm gps.SourceManager) (*dep.Manifest, *dep.Lock, error) {
+				g := NewImporter(logger, true, sm)
+				g.file = testCase.file
+				return g.convert(importertest.RootProject)
+			})
+			if err != nil {
+				t.Fatalf("%#v", err)
+			}
+		})
+	}
+}

internal/importers/govendor/testdata/golden.txt

@@ -0,0 +1,7 @@
+Detected govendor configuration file...
+Converting from vendor.json...
+  Using ^0.8.1 as initial constraint for imported dep github.com/sdboyer/deptest
+  Trying v0.8.1 (3f4c3be) as initial lock for imported dep github.com/sdboyer/deptest
+  Using ^2.0.0 as initial constraint for imported dep github.com/sdboyer/deptestdos
+  Trying v2.0.0 (5c60720) as initial lock for imported dep github.com/sdboyer/deptestdos
+  Govendor was configured to ignore the test build tag, but that isn't supported by dep yet, and will be ignored. See https://github.com/golang/dep/issues/291.

internal/importers/govendor/testdata/vendor.json

@@ -0,0 +1,22 @@
+{
+	"comment": "",
+	"ignore": "test github.com/sdboyer/dep-test",
+	"package": [
+		{
+			"checksumSHA1": "4R6TQcq0/gI/I2kKeUunuO/pEec=",
+			"origin": "github.com/carolynvs/deptest",
+			"path": "github.com/sdboyer/deptest",
+			"revision": "3f4c3bea144e112a69bbe5d8d01c1b09a544253f",
+			"revisionTime": "2017-02-22T03:31:47Z"
+		},
+		{
+			"checksumSHA1": "96YwrJjpE07ENey/eDWWnCWKQOw=",
+			"path": "github.com/sdboyer/deptestdos",
+			"revision": "5c607206be5decd28e6263ffffdcee067266015e",
+			"revisionTime": "2017-02-22T03:34:58Z",
+			"version": "v2",
+			"versionExact": "v2.0.0"
+		}
+	],
+	"rootPath": "github.com/golang/notexist"
+}

internal/importers/importers.go

@@ -12,6 +12,7 @@ import (
 	"github.com/golang/dep/internal/importers/glide"
 	"github.com/golang/dep/internal/importers/godep"
 	"github.com/golang/dep/internal/importers/govend"
+	"github.com/golang/dep/internal/importers/govendor"
 	"github.com/golang/dep/internal/importers/gvt"
 	"github.com/golang/dep/internal/importers/vndr"
 )
@@ -37,5 +38,6 @@ func BuildAll(logger *log.Logger, verbose bool, sm gps.SourceManager) []Importer
 		vndr.NewImporter(logger, verbose, sm),
 		govend.NewImporter(logger, verbose, sm),
 		gvt.NewImporter(logger, verbose, sm),
+		govendor.NewImporter(logger, verbose, sm),
 	}
 }

internal/importers/importertest/testcase.go

@@ -79,7 +79,7 @@ func (tc TestCase) validate(manifest *dep.Manifest, lock *dep.Lock, convertErr e
 	}
 
 	if !equalSlice(manifest.Ignored, tc.WantIgnored) {
-		return errors.Errorf("unexpected set of ignored projects: \n\t(GOT) %v \n\t(WNT) %v",
+		return errors.Errorf("unexpected set of ignored projects: \n\t(GOT) %#v \n\t(WNT) %#v",
 			manifest.Ignored, tc.WantIgnored)
 	}