go
/
dep
зеркало из https://github.com/golang/dep.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

ensure: Absorb dep prune into dep ensure 

This merge incorporates the long-running branch that consolidated all
the work related to absorbing dep prune into dep ensure. As of this
commit, dep prune is now a hidden dummy command, and dep ensure does all
the heavy lifting automatically.
This commit is contained in:
sam boyer 2018-01-23 20:05:25 -05:00
Родитель 52307d468b 142735b594
Коммит 0368da431f
49 изменённых файлов: 1254 добавлений и 1207 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
cmd/dep/ensure.go
Просмотреть файл

@ -279,7 +279,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
// that "verification" is supposed to look like (#121); in the meantime,
// we unconditionally write out vendor/ so that `dep ensure`'s behavior
// is maximally compatible with what it will eventually become.
sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways)
sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
if err != nil {
return err
}
@ -304,7 +304,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
return handleAllTheFailuresOfTheWorld(err)
}
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior())
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions)
if err != nil {
return err
}
@ -329,7 +329,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
}
// Pass the same lock as old and new so that the writer will observe no
// difference and choose not to write it out.
sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways)
sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
if err != nil {
return err
}
@ -394,7 +394,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
return handleAllTheFailuresOfTheWorld(err)
}
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior())
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions)
if err != nil {
return err
}
@ -693,7 +693,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
}
sort.Strings(reqlist)
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), dep.VendorOnChanged)
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), dep.VendorOnChanged, p.Manifest.PruneOptions)
if err != nil {
return err
}

5
cmd/dep/init.go
Просмотреть файл

@ -118,6 +118,9 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
return errors.Wrap(err, "init failed: unable to prepare an initial manifest and lock for the solver")
}
// Set default prune options for go-tests and unused-packages
p.Manifest.PruneOptions.PruneOptions = gps.PruneNestedVendorDirs + gps.PruneGoTestFiles + gps.PruneUnusedPackages
if cmd.gopath {
gs := newGopathScanner(ctx, directDeps, sm)
err = gs.InitializeRootManifestAndLock(p.Manifest, p.Lock)
@ -177,7 +180,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
}
sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways)
sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions)
if err != nil {
return errors.Wrap(err, "init failed: unable to create a SafeWriter")
}

2
cmd/dep/main.go
Просмотреть файл

@ -67,8 +67,8 @@ func (c *Config) Run() int {
&initCommand{},
&statusCommand{},
&ensureCommand{},
&hashinCommand{},
&pruneCommand{},
&hashinCommand{},
&versionCommand{},
}

192
cmd/dep/prune.go
Просмотреть файл

@ -1,206 +1,36 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"flag"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/golang/dep"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/pkgtree"
"github.com/golang/dep/internal/fs"
"github.com/pkg/errors"
)
const pruneShortHelp = `Prune the vendor tree of unused packages`
const pruneShortHelp = `Pruning is now performed automatically by dep ensure.`
const pruneLongHelp = `
Prune is used to remove unused packages from your vendor tree.
STABILITY NOTICE: this command creates problems for vendor/ verification. As
such, it may be removed and/or moved out into a separate project later on.
Prune was merged into the ensure command.
Set prune options in the manifest and it will be applied after every ensure.
dep prune will be removed in a future version of dep, causing this command to exit non-0.
`
type pruneCommand struct {
}
type pruneCommand struct{}
func (cmd *pruneCommand) Name() string { return "prune" }
func (cmd *pruneCommand) Args() string { return "" }
func (cmd *pruneCommand) ShortHelp() string { return pruneShortHelp }
func (cmd *pruneCommand) LongHelp() string { return pruneLongHelp }
func (cmd *pruneCommand) Hidden() bool { return false }
func (cmd *pruneCommand) Hidden() bool { return true }
func (cmd *pruneCommand) Register(fs *flag.FlagSet) {
}
func (cmd *pruneCommand) Register(fs *flag.FlagSet) {}
func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
p, err := ctx.LoadProject()
if err != nil {
return err
}
sm, err := ctx.SourceManager()
if err != nil {
return err
}
sm.UseDefaultSignalHandling()
defer sm.Release()
// While the network churns on ListVersions() requests, statically analyze
// code from the current project.
ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
if err != nil {
return errors.Wrap(err, "analysis of local packages failed: %v")
}
// Set up a solver in order to check the InputHash.
params := p.MakeParams()
params.RootPackageTree = ptree
if ctx.Verbose {
params.TraceLogger = ctx.Err
}
s, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "could not set up solver for input hashing")
}
if p.Lock == nil {
return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
}
if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
}
pruneLogger := ctx.Err
if !ctx.Verbose {
pruneLogger = log.New(ioutil.Discard, "", 0)
}
return pruneProject(p, sm, pruneLogger)
}
// pruneProject removes unused packages from a project.
func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) error {
td, err := ioutil.TempDir(os.TempDir(), "dep")
if err != nil {
return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
}
defer os.RemoveAll(td)
if err := gps.WriteDepTree(td, p.Lock, sm, true, logger); err != nil {
return err
}
var toKeep []string
for _, project := range p.Lock.Projects() {
projectRoot := string(project.Ident().ProjectRoot)
for _, pkg := range project.Packages() {
toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
}
}
toDelete, err := calculatePrune(td, toKeep, logger)
if err != nil {
return err
}
if len(toDelete) > 0 {
logger.Println("Calculated the following directories to prune:")
for _, d := range toDelete {
logger.Printf(" %s\n", d)
}
} else {
logger.Println("No directories found to prune")
}
if err := deleteDirs(toDelete); err != nil {
return err
}
vpath := filepath.Join(p.AbsRoot, "vendor")
vendorbak := vpath + ".orig"
var failerr error
if _, err := os.Stat(vpath); err == nil {
// Move out the old vendor dir. just do it into an adjacent dir, to
// try to mitigate the possibility of a pointless cross-filesystem
// move with a temp directory.
if _, err := os.Stat(vendorbak); err == nil {
// If the adjacent dir already exists, bite the bullet and move
// to a proper tempdir.
vendorbak = filepath.Join(td, "vendor.orig")
}
failerr = fs.RenameWithFallback(vpath, vendorbak)
if failerr != nil {
goto fail
}
}
// Move in the new one.
failerr = fs.RenameWithFallback(td, vpath)
if failerr != nil {
goto fail
}
os.RemoveAll(vendorbak)
ctx.Out.Printf("Pruning is now performed automatically by dep ensure.\n")
ctx.Out.Printf("Set prune settings in %s and it it will be applied when running ensure.\n", dep.ManifestName)
ctx.Out.Printf("\ndep prune will be removed in a future version, and this command will exit non-0.\nPlease update your scripts.\n")
return nil
fail:
fs.RenameWithFallback(vendorbak, vpath)
return failerr
}
func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
logger.Println("Calculating prune. Checking the following packages:")
sort.Strings(keep)
toDelete := []string{}
err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
if _, err := os.Lstat(path); err != nil {
return nil
}
if !info.IsDir() {
return nil
}
if path == vendorDir {
return nil
}
name := strings.TrimPrefix(path, vendorDir+string(filepath.Separator))
logger.Printf(" %s", name)
i := sort.Search(len(keep), func(i int) bool {
return name <= keep[i]
})
if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
toDelete = append(toDelete, path)
}
return nil
})
return toDelete, err
}
func deleteDirs(toDelete []string) error {
// sort by length so we delete sub dirs first
sort.Sort(byLen(toDelete))
for _, path := range toDelete {
if err := os.RemoveAll(path); err != nil {
return err
}
}
return nil
}
type byLen []string
func (a byLen) Len() int { return len(a) }
func (a byLen) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }

50
cmd/dep/prune_test.go
Просмотреть файл

@ -1,50 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"io/ioutil"
"log"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/golang/dep/internal/test"
)
func TestCalculatePrune(t *testing.T) {
h := test.NewHelper(t)
defer h.Cleanup()
vendorDir := "vendor"
h.TempDir(vendorDir)
h.TempDir(filepath.Join(vendorDir, "github.com/keep/pkg/sub"))
h.TempDir(filepath.Join(vendorDir, "github.com/prune/pkg/sub"))
toKeep := []string{
filepath.FromSlash("github.com/keep/pkg"),
filepath.FromSlash("github.com/keep/pkg/sub"),
}
discardLogger := log.New(ioutil.Discard, "", 0)
got, err := calculatePrune(h.Path(vendorDir), toKeep, discardLogger)
if err != nil {
t.Fatal(err)
}
sort.Sort(byLen(got))
want := []string{
h.Path(filepath.Join(vendorDir, "github.com/prune/pkg/sub")),
h.Path(filepath.Join(vendorDir, "github.com/prune/pkg")),
h.Path(filepath.Join(vendorDir, "github.com/prune")),
}
if !reflect.DeepEqual(want, got) {
t.Fatalf("calculated prune paths are not as expected.\n(WNT) %s\n(GOT) %s", want, got)
}
}

4
cmd/dep/testdata/harness_tests/ensure/add/all-new-double-spec/final/Gopkg.toml поставляемый
Просмотреть файл

@ -3,6 +3,10 @@
branch = "master"
name = "github.com/sdboyer/deptesttres"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "0.8.1"

4
cmd/dep/testdata/harness_tests/ensure/add/all-new-double/final/Gopkg.toml поставляемый
Просмотреть файл

@ -3,6 +3,10 @@
name = "github.com/sdboyer/deptest"
version = "1.0.0"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
branch = "master"
name = "github.com/sdboyer/deptesttres"

4
cmd/dep/testdata/harness_tests/ensure/add/all-new-spec/final/Gopkg.toml поставляемый
Просмотреть файл

@ -3,6 +3,10 @@
branch = "master"
name = "github.com/sdboyer/deptesttres"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "0.8.1"

4
cmd/dep/testdata/harness_tests/ensure/add/all-new/final/Gopkg.toml поставляемый
Просмотреть файл

@ -3,6 +3,10 @@
branch = "master"
name = "github.com/sdboyer/deptesttres"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "1.0.0"

4
cmd/dep/testdata/harness_tests/ensure/add/errs/double-diff-spec/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
branch = "master"
name = "github.com/sdboyer/deptesttres"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/ensure/add/errs/self-add/case2/final/Gopkg.toml поставляемый
Просмотреть файл

@ -0,0 +1,4 @@
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/ensure/empty/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "1.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/ensure/pkg-errors/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -0,0 +1,4 @@
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "0.8.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/case2/final/Gopkg.toml поставляемый
Просмотреть файл

@ -6,3 +6,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/case3/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
branch = "master"
name = "github.com/sdboyer/deptest"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/case4/final/Gopkg.toml поставляемый
Просмотреть файл

@ -6,3 +6,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/glide/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -6,3 +6,7 @@ ignored = [
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/glide/case2/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/glide/case3/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/carolynvs/deptestglide"
version = "0.1.1"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/glock/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/godep/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/govend/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -6,3 +6,7 @@ ignored = [
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/gvt/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -10,3 +10,7 @@
[[constraint]]
branch = "v2"
name = "gopkg.in/yaml.v2"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/relative_path/final/project_dir/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
branch = "master"
name = "github.com/sdboyer/deptest"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/skip-hidden/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "1.0.0"
[prune]
go-tests = true
unused-packages = true

4
cmd/dep/testdata/harness_tests/init/vndr/case1/final/Gopkg.toml поставляемый
Просмотреть файл

@ -2,3 +2,7 @@
[[constraint]]
name = "github.com/sdboyer/deptestdos"
version = "2.0.0"
[prune]
go-tests = true
unused-packages = true

3
cmd/dep/testdata/harness_tests/prune/without_lock/final/Gopkg.toml поставляемый
Просмотреть файл

@ -1,3 +0,0 @@
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "^0.8.0"

3
cmd/dep/testdata/harness_tests/prune/without_lock/initial/Gopkg.toml поставляемый
Просмотреть файл

@ -1,3 +0,0 @@
[[constraint]]
name = "github.com/sdboyer/deptest"
version = "^0.8.0"

6
cmd/dep/testdata/harness_tests/prune/without_lock/testcase.json поставляемый
Просмотреть файл

@ -1,6 +0,0 @@
{
"commands": [
["prune"]
],
"error-expected": "Gopkg.lock must exist"
}

2
docs/FAQ.md
Просмотреть файл

@ -348,7 +348,7 @@ It's up to you:
**Cons**
- Your repo will be bigger, potentially a lot bigger,
though `dep prune` can help minimize this problem.
though [`prune`](Gopkg.toml.md#prune) can help minimize this problem.
- PR diffs will include changes for files under `vendor/` when Gopkg.lock is modified,
however files in `vendor/` are [hidden by default](https://github.com/github/linguist/blob/v5.2.0/lib/linguist/generated.rb#L328) on Github.

36
docs/Gopkg.toml.md
Просмотреть файл

@ -58,6 +58,35 @@ system1-data = "value that is used by a system"
system2-data = "value that is used by another system"
```
## `prune`
`prune` defines the global and per-project prune options for dependencies. The options control which files are not kept when writing the `vendor/` tree.
The following is the current available options:
* `unused-packages` prunes files in unused packages.
* `non-go` prunes files that are not used by Go.
* `go-tests` prunes Go test files.
Some files are preversed by default (check the [isPreservedFile](../gps/prune.go#L254) function for the details).
Prune options are off by default and can be turned on by setting them to `true` at the root level.
```toml
[prune]
non-go = true
```
The same prune options can be defined per-project. An addtional `name` field is required and should represent a project and not a package.
```toml
[prune]
non-go = true
[[prune.project]]
name = "github.com/project/name"
go-tests = true
non-go = false
```
## `constraint`
A `constraint` provides rules for how a [direct dependency](FAQ.md#what-is-a-direct-or-transitive-dependency) may be incorporated into the
dependency graph.
@ -171,4 +200,11 @@ codename = "foo"
[override.metadata]
propertyX = "valueX"
[prune]
unused-packages = true
[[prune.project]]
name = "github.com/user/project2"
unused-packages = false
```

136
gps/filesystem.go Normal file
Просмотреть файл

@ -0,0 +1,136 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gps
import (
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
)
// fsLink represents a symbolic link.
type fsLink struct {
path string
to string
// circular denotes if evaluating the symlink fails with "too many links" error.
// This errors means that it's very likely that the symlink has circual refernce.
circular bool
// broken denotes that attempting to resolve the link fails, most likely because
// the destaination doesn't exist.
broken bool
}
// filesystemState represents the state of a file system.
type filesystemState struct {
root string
dirs []string
files []string
links []fsLink
}
func (s filesystemState) setup() error {
for _, dir := range s.dirs {
p := filepath.Join(s.root, dir)
if err := os.MkdirAll(p, 0777); err != nil {
return errors.Errorf("os.MkdirAll(%q, 0777) err=%q", p, err)
}
}
for _, file := range s.files {
p := filepath.Join(s.root, file)
f, err := os.Create(p)
if err != nil {
return errors.Errorf("os.Create(%q) err=%q", p, err)
}
if err := f.Close(); err != nil {
return errors.Errorf("file %q Close() err=%q", p, err)
}
}
for _, link := range s.links {
p := filepath.Join(s.root, link.path)
// On Windows, relative symlinks confuse filepath.Walk. So, we'll just sigh
// and do absolute links, assuming they are relative to the directory of
// link.path.
//
// Reference: https://github.com/golang/go/issues/17540
//
// TODO(ibrasho): This was fixed in Go 1.9. Remove this when support for
// 1.8 is dropped.
dir := filepath.Dir(p)
to := ""
if link.to != "" {
to = filepath.Join(dir, link.to)
}
if err := os.Symlink(to, p); err != nil {
return errors.Errorf("os.Symlink(%q, %q) err=%q", to, p, err)
}
}
return nil
}
// deriveFilesystemState returns a filesystemState based on the state of
// the filesystem on root.
func deriveFilesystemState(root string) (filesystemState, error) {
fs := filesystemState{root: root}
err := filepath.Walk(fs.root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == fs.root {
return nil
}
relPath, err := filepath.Rel(fs.root, path)
if err != nil {
return err
}
if (info.Mode() & os.ModeSymlink) != 0 {
l := fsLink{path: relPath}
l.to, err = filepath.EvalSymlinks(path)
if err != nil && strings.HasSuffix(err.Error(), "too many links") {
l.circular = true
} else if err != nil && os.IsNotExist(err) {
l.broken = true
} else if err != nil {
return err
}
fs.links = append(fs.links, l)
return nil
}
if info.IsDir() {
fs.dirs = append(fs.dirs, relPath)
return nil
}
fs.files = append(fs.files, relPath)
return nil
})
if err != nil {
return filesystemState{}, err
}
return fs, nil
}

208
gps/filesystem_test.go
Просмотреть файл

@ -5,61 +5,45 @@
package gps
import (
"fmt"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/golang/dep/internal/test"
)
// This file contains utilities for running tests around file system state.
// fspath represents a file system path in an OS-agnostic way.
type fsPath []string
func (f fsPath) String() string { return filepath.Join(f...) }
func (f fsPath) prepend(prefix string) fsPath {
p := fsPath{filepath.FromSlash(prefix)}
return append(p, f...)
}
type fsTestCase struct {
before, after filesystemState
}
// filesystemState represents the state of a file system. It has a setup method
// which inflates its state to the actual host file system, and an assert
// method which checks that the actual file system matches the described state.
type filesystemState struct {
root string
dirs []fsPath
files []fsPath
links []fsLink
}
// assert makes sure that the fs state matches the state of the actual host
// file system
func (fs filesystemState) assert(t *testing.T) {
// assert makes sure that the tc.after state matches the state of the actual host
// file system at tc.after.root.
func (tc fsTestCase) assert(t *testing.T) {
dirMap := make(map[string]bool)
fileMap := make(map[string]bool)
linkMap := make(map[string]bool)
for _, d := range fs.dirs {
dirMap[d.prepend(fs.root).String()] = true
for _, d := range tc.after.dirs {
dirMap[filepath.Join(tc.after.root, d)] = true
}
for _, f := range fs.files {
fileMap[f.prepend(fs.root).String()] = true
for _, f := range tc.after.files {
fileMap[filepath.Join(tc.after.root, f)] = true
}
for _, l := range fs.links {
linkMap[l.path.prepend(fs.root).String()] = true
for _, l := range tc.after.links {
linkMap[filepath.Join(tc.after.root, l.path)] = true
}
err := filepath.Walk(fs.root, func(path string, info os.FileInfo, err error) error {
err := filepath.Walk(tc.after.root, func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Errorf("filepath.Walk path=%q err=%q", path, err)
return err
}
if path == fs.root {
if path == tc.after.root {
return nil
}
@ -106,53 +90,131 @@ func (fs filesystemState) assert(t *testing.T) {
}
}
// fsLink represents a symbolic link.
type fsLink struct {
path fsPath
to string
}
// setup inflates fs onto the actual host file system
func (fs filesystemState) setup(t *testing.T) {
fs.setupDirs(t)
fs.setupFiles(t)
fs.setupLinks(t)
}
func (fs filesystemState) setupDirs(t *testing.T) {
for _, dir := range fs.dirs {
p := dir.prepend(fs.root)
if err := os.MkdirAll(p.String(), 0777); err != nil {
t.Fatalf("os.MkdirAll(%q, 0777) err=%q", p, err)
}
// setup inflates fs onto the actual host file system at tc.before.root.
// It doesn't delete existing files and should be used on empty roots only.
func (tc fsTestCase) setup(t *testing.T) {
if err := tc.before.setup(); err != nil {
t.Fatal(err)
}
}
func (fs filesystemState) setupFiles(t *testing.T) {
for _, file := range fs.files {
p := file.prepend(fs.root)
f, err := os.Create(p.String())
func TestDeriveFilesystemState(t *testing.T) {
testcases := []struct {
name string
fs fsTestCase
}{
{
name: "simple-case",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"simple-dir",
},
files: []string{
"simple-file",
},
},
after: filesystemState{
dirs: []string{
"simple-dir",
},
files: []string{
"simple-file",
},
},
},
},
{
name: "simple-symlink-case",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"simple-dir",
},
files: []string{
"simple-file",
},
links: []fsLink{
fsLink{
path: "link",
to: "nonexisting",
broken: true,
},
},
},
after: filesystemState{
dirs: []string{
"simple-dir",
},
files: []string{
"simple-file",
},
links: []fsLink{
fsLink{
path: "link",
to: "",
broken: true,
},
},
},
},
},
{
name: "complex-symlink-case",
fs: fsTestCase{
before: filesystemState{
links: []fsLink{
fsLink{
path: "link1",
to: "link2",
circular: true,
},
fsLink{
path: "link2",
to: "link1",
circular: true,
},
},
},
after: filesystemState{
links: []fsLink{
fsLink{
path: "link1",
to: "",
circular: true,
},
fsLink{
path: "link2",
to: "",
circular: true,
},
},
},
},
},
}
for _, tc := range testcases {
h := test.NewHelper(t)
h.TempDir(tc.name)
tc.fs.before.root = h.Path(tc.name)
tc.fs.after.root = h.Path(tc.name)
tc.fs.setup(t)
state, err := deriveFilesystemState(h.Path(tc.name))
if err != nil {
t.Fatalf("os.Create(%q) err=%q", p, err)
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatalf("file %q Close() err=%q", p, err)
}
}
}
func (fs filesystemState) setupLinks(t *testing.T) {
for _, link := range fs.links {
p := link.path.prepend(fs.root)
// On Windows, relative symlinks confuse filepath.Walk. This is golang/go
// issue 17540. So, we'll just sigh and do absolute links, assuming they are
// relative to the directory of link.path.
dir := filepath.Dir(p.String())
to := filepath.Join(dir, link.to)
if err := os.Symlink(to, p.String()); err != nil {
t.Fatalf("os.Symlink(%q, %q) err=%q", to, p, err)
if !reflect.DeepEqual(tc.fs.after, state) {
fmt.Println(tc.fs.after)
fmt.Println(state)
t.Fatal("filesystem state mismatch")
}
h.Cleanup()
}
}

349
gps/prune.go
Просмотреть файл

@ -8,8 +8,10 @@ import (
"log"
"os"
"path/filepath"
"sort"
"strings"
"github.com/golang/dep/internal/fs"
"github.com/pkg/errors"
)
@ -19,6 +21,13 @@ type PruneOptions uint8
// PruneProjectOptions is map of prune options per project name.
type PruneProjectOptions map[ProjectRoot]PruneOptions
// RootPruneOptions represents the root prune options for the project.
// It contains the global options and a map of options per project.
type RootPruneOptions struct {
PruneOptions PruneOptions
ProjectOptions PruneProjectOptions
}
const (
// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
PruneNestedVendorDirs PruneOptions = 1 << iota
@ -32,6 +41,26 @@ const (
PruneGoTestFiles
)
// DefaultRootPruneOptions instantiates a copy of the default root prune options.
func DefaultRootPruneOptions() RootPruneOptions {
return RootPruneOptions{
PruneOptions: PruneNestedVendorDirs,
ProjectOptions: PruneProjectOptions{},
}
}
// PruneOptionsFor returns the prune options for the passed project root.
//
// It will return the root prune options if the project does not have specific
// options or if it does not exist in the manifest.
func (o *RootPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
if po, ok := o.ProjectOptions[pr]; ok {
return po
}
return o.PruneOptions
}
var (
// licenseFilePrefixes is a list of name prefixes for license files.
licenseFilePrefixes = []string{
@ -56,206 +85,183 @@ var (
}
)
// Prune removes excess files from the dep tree whose root is baseDir based
// on the PruneOptions passed.
//
// A Lock must be passed if PruneUnusedPackages is toggled on.
func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) error {
// TODO(ibrasho) allow passing specific options per project
for _, lp := range l.Projects() {
projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
err := PruneProject(projectDir, lp, options, logger)
if err != nil {
return err
}
}
return nil
}
// PruneProject remove excess files according to the options passed, from
// the lp directory in baseDir.
func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error {
projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot))
fsState, err := deriveFilesystemState(baseDir)
if err != nil {
return errors.Wrap(err, "could not derive filesystem state")
}
if (options & PruneNestedVendorDirs) != 0 {
if err := pruneNestedVendorDirs(projectDir); err != nil {
if err := pruneVendorDirs(fsState); err != nil {
return errors.Wrapf(err, "failed to prune nested vendor directories")
}
}
if (options & PruneUnusedPackages) != 0 {
if err := pruneUnusedPackages(lp, projectDir, logger); err != nil {
if _, err := pruneUnusedPackages(lp, fsState); err != nil {
return errors.Wrap(err, "failed to prune unused packages")
}
}
if (options & PruneNonGoFiles) != 0 {
if err := pruneNonGoFiles(projectDir, logger); err != nil {
if err := pruneNonGoFiles(fsState); err != nil {
return errors.Wrap(err, "failed to prune non-Go files")
}
}
if (options & PruneGoTestFiles) != 0 {
if err := pruneGoTestFiles(projectDir, logger); err != nil {
if err := pruneGoTestFiles(fsState); err != nil {
return errors.Wrap(err, "failed to prune Go test files")
}
}
if err := deleteEmptyDirs(fsState); err != nil {
return errors.Wrap(err, "could not delete empty dirs")
}
return nil
}
// pruneNestedVendorDirs deletes all nested vendor directories within baseDir.
func pruneNestedVendorDirs(baseDir string) error {
return filepath.Walk(baseDir, stripVendor)
// pruneVendorDirs deletes all nested vendor directories within baseDir.
func pruneVendorDirs(fsState filesystemState) error {
for _, dir := range fsState.dirs {
if filepath.Base(dir) == "vendor" {
err := os.RemoveAll(filepath.Join(fsState.root, dir))
if err != nil && !os.IsNotExist(err) {
return err
}
}
}
for _, link := range fsState.links {
if filepath.Base(link.path) == "vendor" {
err := os.Remove(filepath.Join(fsState.root, link.path))
if err != nil && !os.IsNotExist(err) {
return err
}
}
}
return nil
}
// pruneUnusedPackages deletes unimported packages found within baseDir.
// pruneUnusedPackages deletes unimported packages found in fsState.
// Determining whether packages are imported or not is based on the passed LockedProject.
func pruneUnusedPackages(lp LockedProject, projectDir string, logger *log.Logger) error {
pr := string(lp.Ident().ProjectRoot)
logger.Printf("Calculating unused packages in %s to prune.\n", pr)
func pruneUnusedPackages(lp LockedProject, fsState filesystemState) (map[string]interface{}, error) {
unusedPackages := calculateUnusedPackages(lp, fsState)
toDelete := collectUnusedPackagesFiles(fsState, unusedPackages)
unusedPackages, err := calculateUnusedPackages(lp, projectDir)
if err != nil {
return errors.Wrapf(err, "could not calculate unused packages in %s", pr)
for _, path := range toDelete {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return nil, err
}
}
logger.Printf("Found the following unused packages in %s:\n", pr)
for pkg := range unusedPackages {
logger.Printf(" * %s\n", filepath.Join(pr, pkg))
}
unusedPackagesFiles, err := collectUnusedPackagesFiles(projectDir, unusedPackages)
if err != nil {
return errors.Wrapf(err, "could not collect unused packages' files in %s", pr)
}
if err := deleteFiles(unusedPackagesFiles); err != nil {
return errors.Wrapf(err, "")
}
return nil
return unusedPackages, nil
}
// calculateUnusedPackages generates a list of unused packages in lp.
func calculateUnusedPackages(lp LockedProject, projectDir string) (map[string]struct{}, error) {
unused := make(map[string]struct{})
imported := make(map[string]struct{})
func calculateUnusedPackages(lp LockedProject, fsState filesystemState) map[string]interface{} {
unused := make(map[string]interface{})
imported := make(map[string]interface{})
for _, pkg := range lp.Packages() {
imported[pkg] = struct{}{}
imported[pkg] = nil
}
err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Add the root package if it's not imported.
if _, ok := imported["."]; !ok {
unused["."] = nil
}
// Ignore anything that's not a directory.
if !info.IsDir() {
return nil
}
for _, dirPath := range fsState.dirs {
pkg := filepath.ToSlash(dirPath)
pkg, err := filepath.Rel(projectDir, path)
if err != nil {
return errors.Wrap(err, "unexpected error while calculating unused packages")
}
pkg = filepath.ToSlash(pkg)
if _, ok := imported[pkg]; !ok {
unused[pkg] = struct{}{}
unused[pkg] = nil
}
}
return nil
})
return unused, err
return unused
}
// collectUnusedPackagesFiles returns a slice of all files in the unused packages in projectDir.
func collectUnusedPackagesFiles(projectDir string, unusedPackages map[string]struct{}) ([]string, error) {
// collectUnusedPackagesFiles returns a slice of all files in the unused
// packages based on fsState.
func collectUnusedPackagesFiles(fsState filesystemState, unusedPackages map[string]interface{}) []string {
// TODO(ibrasho): is this useful?
files := make([]string, 0, len(unusedPackages))
err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
for _, path := range fsState.files {
// Keep perserved files.
if isPreservedFile(filepath.Base(path)) {
continue
}
// Ignore directories.
if info.IsDir() {
return nil
}
pkg := filepath.ToSlash(filepath.Dir(path))
// Ignore preserved files.
if isPreservedFile(info.Name()) {
return nil
}
pkg, err := filepath.Rel(projectDir, filepath.Dir(path))
if err != nil {
return errors.Wrap(err, "unexpected error while calculating unused packages")
}
pkg = filepath.ToSlash(pkg)
if _, ok := unusedPackages[pkg]; ok {
files = append(files, path)
files = append(files, filepath.Join(fsState.root, path))
}
return nil
})
return files, err
}
// pruneNonGoFiles delete all non-Go files existing within baseDir.
// Files with names that are prefixed by any entry in preservedNonGoFiles
// are not deleted.
func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
files, err := collectNonGoFiles(baseDir, logger)
if err != nil {
return errors.Wrap(err, "could not collect non-Go files")
}
if err := deleteFiles(files); err != nil {
return errors.Wrap(err, "could not prune Go test files")
return files
}
// pruneNonGoFiles delete all non-Go files existing in fsState.
//
// Files matching licenseFilePrefixes and legalFileSubstrings are not pruned.
func pruneNonGoFiles(fsState filesystemState) error {
toDelete := make([]string, 0, len(fsState.files)/4)
for _, path := range fsState.files {
ext := fileExt(path)
// Refer to: https://github.com/golang/go/blob/release-branch.go1.9/src/go/build/build.go#L750
switch ext {
case ".go":
continue
case ".c":
continue
case ".cc", ".cpp", ".cxx":
continue
case ".m":
continue
case ".h", ".hh", ".hpp", ".hxx":
continue
case ".f", ".F", ".for", ".f90":
continue
case ".s":
continue
case ".S":
continue
case ".swig":
continue
case ".swigcxx":
continue
case ".syso":
continue
}
// Ignore perserved files.
if isPreservedFile(filepath.Base(path)) {
continue
}
toDelete = append(toDelete, filepath.Join(fsState.root, path))
}
for _, path := range toDelete {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
// collectNonGoFiles returns a slice containing all non-Go files in baseDir.
// Files meeting the checks in isPreservedFile are not returned.
func collectNonGoFiles(baseDir string, logger *log.Logger) ([]string, error) {
files := make([]string, 0)
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Ignore directories.
if info.IsDir() {
return nil
}
// Ignore all Go files.
if strings.HasSuffix(info.Name(), ".go") {
return nil
}
// Ignore preserved files.
if isPreservedFile(info.Name()) {
return nil
}
files = append(files, path)
return nil
})
return files, err
}
// isPreservedFile checks if the file name indicates that the file should be
// preserved based on licenseFilePrefixes or legalFileSubstrings.
func isPreservedFile(name string) bool {
@ -276,51 +282,50 @@ func isPreservedFile(name string) bool {
return false
}
// pruneGoTestFiles deletes all Go test files (*_test.go) within baseDir.
func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
files, err := collectGoTestFiles(baseDir)
if err != nil {
return errors.Wrap(err, "could not collect Go test files")
// pruneGoTestFiles deletes all Go test files (*_test.go) in fsState.
func pruneGoTestFiles(fsState filesystemState) error {
toDelete := make([]string, 0, len(fsState.files)/2)
for _, path := range fsState.files {
if strings.HasSuffix(path, "_test.go") {
toDelete = append(toDelete, filepath.Join(fsState.root, path))
}
}
if err := deleteFiles(files); err != nil {
return errors.Wrap(err, "could not prune Go test files")
for _, path := range toDelete {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
// collectGoTestFiles returns a slice contains all Go test files (any files
// prefixed with _test.go) in baseDir.
func collectGoTestFiles(baseDir string) ([]string, error) {
files := make([]string, 0)
func deleteEmptyDirs(fsState filesystemState) error {
sort.Sort(sort.Reverse(sort.StringSlice(fsState.dirs)))
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
for _, dir := range fsState.dirs {
path := filepath.Join(fsState.root, dir)
notEmpty, err := fs.IsNonEmptyDir(path)
if err != nil {
return err
}
// Ignore directories.
if info.IsDir() {
return nil
}
// Ignore any files that is not a Go test file.
if strings.HasSuffix(info.Name(), "_test.go") {
files = append(files, path)
}
return nil
})
return files, err
}
func deleteFiles(paths []string) error {
for _, path := range paths {
if err := os.Remove(path); err != nil {
return err
if !notEmpty {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}
}
return nil
}
func fileExt(name string) string {
i := strings.LastIndex(name, ".")
if i < 0 {
return ""
}
return name[i:]
}

606
gps/prune_test.go
Просмотреть файл

@ -7,18 +7,58 @@ package gps
import (
"io/ioutil"
"log"
"os"
"testing"
"github.com/golang/dep/internal/test"
)
func TestRootPruneOptions_PruneOptionsFor(t *testing.T) {
pr := ProjectRoot("github.com/golang/dep")
o := RootPruneOptions{
PruneOptions: PruneNestedVendorDirs,
ProjectOptions: PruneProjectOptions{
pr: PruneGoTestFiles,
},
}
if (o.PruneOptionsFor(pr) & PruneGoTestFiles) != PruneGoTestFiles {
t.Fatalf("invalid prune options.\n\t(GOT): %d\n\t(WNT): %d", o.PruneOptionsFor(pr), PruneGoTestFiles)
}
}
func TestPruneProject(t *testing.T) {
h := test.NewHelper(t)
defer h.Cleanup()
pr := "github.com/project/repository"
h.TempDir(pr)
baseDir := h.Path(".")
lp := LockedProject{
pi: ProjectIdentifier{
ProjectRoot: ProjectRoot(pr),
},
pkgs: []string{},
}
options := PruneNestedVendorDirs | PruneNonGoFiles | PruneGoTestFiles | PruneUnusedPackages
logger := log.New(ioutil.Discard, "", 0)
err := PruneProject(baseDir, lp, options, logger)
if err != nil {
t.Fatal(err)
}
}
func TestPruneUnusedPackages(t *testing.T) {
h := test.NewHelper(t)
defer h.Cleanup()
h.TempDir(".")
pr := "github.com/test/project"
pr := "github.com/sample/repository"
pi := ProjectIdentifier{ProjectRoot: ProjectRoot(pr)}
testcases := []struct {
@ -30,18 +70,20 @@ func TestPruneUnusedPackages(t *testing.T) {
{
"one-package",
LockedProject{
pi: pi,
pkgs: []string{"."},
pi: pi,
pkgs: []string{
".",
},
},
fsTestCase{
before: filesystemState{
files: []fsPath{
{"main.go"},
files: []string{
"main.go",
},
},
after: filesystemState{
files: []fsPath{
{"main.go"},
files: []string{
"main.go",
},
},
},
@ -50,25 +92,27 @@ func TestPruneUnusedPackages(t *testing.T) {
{
"nested-package",
LockedProject{
pi: pi,
pkgs: []string{"pkg"},
pi: pi,
pkgs: []string{
"pkg",
},
},
fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"pkg"},
dirs: []string{
"pkg",
},
files: []fsPath{
{"main.go"},
{"pkg", "main.go"},
files: []string{
"main.go",
"pkg/main.go",
},
},
after: filesystemState{
dirs: []fsPath{
{"pkg"},
dirs: []string{
"pkg",
},
files: []fsPath{
{"pkg", "main.go"},
files: []string{
"pkg/main.go",
},
},
},
@ -77,36 +121,39 @@ func TestPruneUnusedPackages(t *testing.T) {
{
"complex-project",
LockedProject{
pi: pi,
pkgs: []string{"pkg", "pkg/nestedpkg/otherpkg"},
pi: pi,
pkgs: []string{
"pkg",
"pkg/nestedpkg/otherpkg",
},
},
fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"pkg"},
{"pkg", "nestedpkg"},
{"pkg", "nestedpkg", "otherpkg"},
dirs: []string{
"pkg",
"pkg/nestedpkg",
"pkg/nestedpkg/otherpkg",
},
files: []fsPath{
{"main.go"},
{"COPYING"},
{"pkg", "main.go"},
{"pkg", "nestedpkg", "main.go"},
{"pkg", "nestedpkg", "PATENT.md"},
{"pkg", "nestedpkg", "otherpkg", "main.go"},
files: []string{
"main.go",
"COPYING",
"pkg/main.go",
"pkg/nestedpkg/main.go",
"pkg/nestedpkg/PATENT.md",
"pkg/nestedpkg/otherpkg/main.go",
},
},
after: filesystemState{
dirs: []fsPath{
{"pkg"},
{"pkg", "nestedpkg"},
{"pkg", "nestedpkg", "otherpkg"},
dirs: []string{
"pkg",
"pkg/nestedpkg",
"pkg/nestedpkg/otherpkg",
},
files: []fsPath{
{"COPYING"},
{"pkg", "main.go"},
{"pkg", "nestedpkg", "PATENT.md"},
{"pkg", "nestedpkg", "otherpkg", "main.go"},
files: []string{
"COPYING",
"pkg/main.go",
"pkg/nestedpkg/PATENT.md",
"pkg/nestedpkg/otherpkg/main.go",
},
},
},
@ -114,25 +161,27 @@ func TestPruneUnusedPackages(t *testing.T) {
},
}
logger := log.New(ioutil.Discard, "", 0)
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
h.TempDir(pr)
projectDir := h.Path(pr)
tc.fs.before.root = projectDir
tc.fs.after.root = projectDir
baseDir := h.Path(pr)
tc.fs.before.root = baseDir
tc.fs.after.root = baseDir
tc.fs.setup(t)
tc.fs.before.setup(t)
err := pruneUnusedPackages(tc.lp, projectDir, logger)
if tc.err && err == nil {
t.Errorf("expected an error, got nil")
} else if !tc.err && err != nil {
t.Errorf("unexpected error: %s", err)
fs, err := deriveFilesystemState(baseDir)
if err != nil {
t.Fatal(err)
}
tc.fs.after.assert(t)
_, err = pruneUnusedPackages(tc.lp, fs)
if tc.err && err == nil {
t.Fatalf("expected an error, got nil")
} else if !tc.err && err != nil {
t.Fatalf("unexpected error: %s", err)
}
tc.fs.assert(t)
})
}
}
@ -152,8 +201,8 @@ func TestPruneNonGoFiles(t *testing.T) {
"one-file",
fsTestCase{
before: filesystemState{
files: []fsPath{
{"README.md"},
files: []string{
"README.md",
},
},
after: filesystemState{},
@ -164,16 +213,16 @@ func TestPruneNonGoFiles(t *testing.T) {
"multiple-files",
fsTestCase{
before: filesystemState{
files: []fsPath{
{"main.go"},
{"main_test.go"},
{"README"},
files: []string{
"main.go",
"main_test.go",
"README",
},
},
after: filesystemState{
files: []fsPath{
{"main.go"},
{"main_test.go"},
files: []string{
"main.go",
"main_test.go",
},
},
},
@ -183,22 +232,22 @@ func TestPruneNonGoFiles(t *testing.T) {
"mixed-files",
fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
files: []fsPath{
{"dir", "main.go"},
{"dir", "main_test.go"},
{"dir", "db.sqlite"},
files: []string{
"dir/main.go",
"dir/main_test.go",
"dir/db.sqlite",
},
},
after: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
files: []fsPath{
{"dir", "main.go"},
{"dir", "main_test.go"},
files: []string{
"dir/main.go",
"dir/main_test.go",
},
},
},
@ -206,8 +255,6 @@ func TestPruneNonGoFiles(t *testing.T) {
},
}
logger := log.New(ioutil.Discard, "", 0)
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
h.TempDir(tc.name)
@ -215,16 +262,21 @@ func TestPruneNonGoFiles(t *testing.T) {
tc.fs.before.root = baseDir
tc.fs.after.root = baseDir
tc.fs.before.setup(t)
tc.fs.setup(t)
err := pruneNonGoFiles(baseDir, logger)
fs, err := deriveFilesystemState(baseDir)
if err != nil {
t.Fatal(err)
}
err = pruneNonGoFiles(fs)
if tc.err && err == nil {
t.Errorf("expected an error, got nil")
} else if !tc.err && err != nil {
t.Errorf("unexpected error: %s", err)
}
tc.fs.after.assert(t)
tc.fs.assert(t)
})
}
}
@ -244,8 +296,8 @@ func TestPruneGoTestFiles(t *testing.T) {
"one-test-file",
fsTestCase{
before: filesystemState{
files: []fsPath{
{"main_test.go"},
files: []string{
"main_test.go",
},
},
after: filesystemState{},
@ -256,17 +308,17 @@ func TestPruneGoTestFiles(t *testing.T) {
"multiple-files",
fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
files: []fsPath{
{"dir", "main_test.go"},
{"dir", "main2_test.go"},
files: []string{
"dir/main_test.go",
"dir/main2_test.go",
},
},
after: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
},
},
@ -276,23 +328,23 @@ func TestPruneGoTestFiles(t *testing.T) {
"mixed-files",
fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
files: []fsPath{
{"dir", "main.go"},
{"dir", "main2.go"},
{"dir", "main_test.go"},
{"dir", "main2_test.go"},
files: []string{
"dir/main.go",
"dir/main2.go",
"dir/main_test.go",
"dir/main2_test.go",
},
},
after: filesystemState{
dirs: []fsPath{
{"dir"},
dirs: []string{
"dir",
},
files: []fsPath{
{"dir", "main.go"},
{"dir", "main2.go"},
files: []string{
"dir/main.go",
"dir/main2.go",
},
},
},
@ -300,8 +352,6 @@ func TestPruneGoTestFiles(t *testing.T) {
},
}
logger := log.New(ioutil.Discard, "", 0)
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
h.TempDir(tc.name)
@ -309,16 +359,356 @@ func TestPruneGoTestFiles(t *testing.T) {
tc.fs.before.root = baseDir
tc.fs.after.root = baseDir
tc.fs.before.setup(t)
tc.fs.setup(t)
err := pruneGoTestFiles(baseDir, logger)
if tc.err && err == nil {
t.Errorf("expected an error, got nil")
} else if !tc.err && err != nil {
t.Errorf("unexpected error: %s", err)
fs, err := deriveFilesystemState(baseDir)
if err != nil {
t.Fatal(err)
}
tc.fs.after.assert(t)
err = pruneGoTestFiles(fs)
if tc.err && err == nil {
t.Fatalf("expected an error, got nil")
} else if !tc.err && err != nil {
t.Fatalf("unexpected error: %s", err)
}
tc.fs.assert(t)
})
}
}
func TestPruneVendorDirs(t *testing.T) {
tests := []struct {
name string
test fsTestCase
}{
{
name: "vendor directory",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
"package/vendor",
},
},
after: filesystemState{
dirs: []string{
"package",
},
},
},
},
{
name: "vendor file",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
},
files: []string{
"package/vendor",
},
},
after: filesystemState{
dirs: []string{
"package",
},
files: []string{
"package/vendor",
},
},
},
},
{
name: "vendor symlink",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
"package/_vendor",
},
links: []fsLink{
{
path: "package/vendor",
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []string{
"package",
"package/_vendor",
},
},
},
},
{
name: "nonvendor symlink",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
"package/_vendor",
},
links: []fsLink{
{
path: "package/link",
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []string{
"package",
"package/_vendor",
},
links: []fsLink{
{
path: "package/link",
to: "_vendor",
},
},
},
},
},
{
name: "vendor symlink to file",
test: fsTestCase{
before: filesystemState{
files: []string{
"file",
},
links: []fsLink{
{
path: "vendor",
to: "file",
},
},
},
after: filesystemState{
files: []string{
"file",
},
},
},
},
{
name: "broken vendor symlink",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
},
links: []fsLink{
{
path: "package/vendor",
to: "nonexistence",
},
},
},
after: filesystemState{
dirs: []string{
"package",
},
links: []fsLink{},
},
},
},
{
name: "chained symlinks",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"_vendor",
},
links: []fsLink{
{
path: "vendor",
to: "vendor2",
},
{
path: "vendor2",
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []string{
"_vendor",
},
links: []fsLink{
{
path: "vendor2",
to: "_vendor",
},
},
},
},
},
{
name: "circular symlinks",
test: fsTestCase{
before: filesystemState{
dirs: []string{
"package",
},
links: []fsLink{
{
path: "package/link1",
to: "link2",
},
{
path: "package/link2",
to: "link1",
},
},
},
after: filesystemState{
dirs: []string{
"package",
},
links: []fsLink{
{
path: "package/link1",
to: "link2",
},
{
path: "package/link2",
to: "link1",
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, pruneVendorDirsTestCase(test.test))
}
}
func pruneVendorDirsTestCase(tc fsTestCase) func(*testing.T) {
return func(t *testing.T) {
tempDir, err := ioutil.TempDir("", "pruneVendorDirsTestCase")
if err != nil {
t.Fatalf("ioutil.TempDir err=%q", err)
}
defer func() {
if err := os.RemoveAll(tempDir); err != nil {
t.Errorf("os.RemoveAll(%q) err=%q", tempDir, err)
}
}()
tc.before.root = tempDir
tc.after.root = tempDir
tc.setup(t)
fs, err := deriveFilesystemState(tempDir)
if err != nil {
t.Fatalf("deriveFilesystemState failed: %s", err)
}
if err := pruneVendorDirs(fs); err != nil {
t.Errorf("pruneVendorDirs err=%q", err)
}
tc.assert(t)
}
}
func TestDeleteEmptyDirs(t *testing.T) {
testcases := []struct {
name string
fs fsTestCase
}{
{
name: "empty-dir",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"pkg1",
},
},
after: filesystemState{},
},
},
{
name: "nested-empty-dirs",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"pkg1",
"pkg1/pkg2",
},
},
after: filesystemState{},
},
},
{
name: "non-empty-dir",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"pkg1",
},
files: []string{
"pkg1/file1",
},
},
after: filesystemState{
dirs: []string{
"pkg1",
},
files: []string{
"pkg1/file1",
},
},
},
},
{
name: "mixed-dirs",
fs: fsTestCase{
before: filesystemState{
dirs: []string{
"pkg1",
"pkg1/pkg2",
},
files: []string{
"pkg1/file1",
},
},
after: filesystemState{
dirs: []string{
"pkg1",
},
files: []string{
"pkg1/file1",
},
},
},
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
h := test.NewHelper(t)
h.Cleanup()
h.TempDir(".")
tc.fs.before.root = h.Path(".")
tc.fs.after.root = h.Path(".")
if err := tc.fs.before.setup(); err != nil {
t.Fatal("unexpected error in fs setup: ", err)
}
if err := deleteEmptyDirs(tc.fs.before); err != nil {
t.Fatal("unexpected error in deleteEmptyDirs: ", err)
}
tc.fs.assert(t)
})
}
}

24
gps/solution.go
Просмотреть файл

@ -50,16 +50,15 @@ type solution struct {
const concurrentWriters = 16
// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
// WriteDepTree takes a basedir, a Lock and a RootPruneOptions and exports all
// the projects listed in the lock to the appropriate target location within basedir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
// It requires a SourceManager to do the work. Prune options are read from the
// passed manifest.
func WriteDepTree(basedir string, l Lock, sm SourceManager, rpo RootPruneOptions, logger *log.Logger) error {
if l == nil {
return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
}
@ -96,14 +95,13 @@ func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log
return errors.Wrapf(err, "failed to export %s", projectRoot)
}
if sv {
if err := ctx.Err(); err != nil {
return err
}
err := PruneProject(to, p, rpo.PruneOptionsFor(ident.ProjectRoot), logger)
if err != nil {
return errors.Wrapf(err, "failed to prune %s", projectRoot)
}
if err := filepath.Walk(to, stripVendor); err != nil {
return errors.Wrapf(err, "failed to strip vendor from %s", projectRoot)
}
if err := ctx.Err(); err != nil {
return err
}
return nil

6
gps/solution_test.go
Просмотреть файл

@ -109,12 +109,12 @@ func testWriteDepTree(t *testing.T) {
}
// nil lock/result should err immediately
err = WriteDepTree(tmp, nil, sm, true, discardLogger())
err = WriteDepTree(tmp, nil, sm, DefaultRootPruneOptions(), discardLogger())
if err == nil {
t.Errorf("Should error if nil lock is passed to WriteDepTree")
}
err = WriteDepTree(tmp, r, sm, true, discardLogger())
err = WriteDepTree(tmp, r, sm, DefaultRootPruneOptions(), discardLogger())
if err != nil {
t.Errorf("Unexpected error while creating vendor tree: %s", err)
}
@ -166,7 +166,7 @@ func BenchmarkCreateVendorTree(b *testing.B) {
// ease manual inspection
os.RemoveAll(exp)
b.StartTimer()
err = WriteDepTree(exp, r, sm, true, logger)
err = WriteDepTree(exp, r, sm, DefaultRootPruneOptions(), logger)
b.StopTimer()
if err != nil {
b.Errorf("unexpected error after %v iterations: %s", i, err)

39
gps/strip_vendor.go
Просмотреть файл

@ -1,39 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//+build !windows
package gps
import (
"os"
"path/filepath"
)
func stripVendor(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip anything not named vendor
if info.Name() != "vendor" {
return nil
}
// If the file is a symlink to a directory, delete the symlink.
if (info.Mode() & os.ModeSymlink) != 0 {
if realInfo, err := os.Stat(path); err == nil && realInfo.IsDir() {
return os.Remove(path)
}
}
if info.IsDir() {
if err := os.RemoveAll(path); err != nil {
return err
}
return filepath.SkipDir
}
return nil
}

171
gps/strip_vendor_nonwindows_test.go
Просмотреть файл

@ -1,171 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !windows
package gps
import "testing"
func TestStripVendorSymlinks(t *testing.T) {
t.Run("vendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
},
}))
t.Run("nonvendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "link"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "link"},
to: "_vendor",
},
},
},
}))
t.Run("vendor symlink to file", stripVendorTestCase(fsTestCase{
before: filesystemState{
files: []fsPath{
{"file"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "file",
},
},
},
after: filesystemState{
files: []fsPath{
{"file"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "file",
},
},
},
}))
t.Run("broken vendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "nonexistence",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "nonexistence",
},
},
},
}))
t.Run("chained symlinks", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"_vendor"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "vendor2",
},
{
path: fsPath{"vendor2"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"_vendor"},
},
links: []fsLink{
{
path: fsPath{"vendor2"},
to: "_vendor",
},
},
},
}))
t.Run("circular symlinks", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "link1"},
to: "link2",
},
{
path: fsPath{"package", "link2"},
to: "link1",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "link1"},
to: "link2",
},
{
path: fsPath{"package", "link2"},
to: "link1",
},
},
},
}))
}

71
gps/strip_vendor_test.go
Просмотреть файл

@ -1,71 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gps
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
)
func stripVendorTestCase(tc fsTestCase) func(*testing.T) {
return func(t *testing.T) {
tempDir, err := ioutil.TempDir("", "TestStripVendor")
if err != nil {
t.Fatalf("ioutil.TempDir err=%q", err)
}
defer func() {
if err := os.RemoveAll(tempDir); err != nil {
t.Errorf("os.RemoveAll(%q) err=%q", tempDir, err)
}
}()
tc.before.root = tempDir
tc.after.root = tempDir
tc.before.setup(t)
if err := filepath.Walk(tempDir, stripVendor); err != nil {
t.Errorf("filepath.Walk err=%q", err)
}
tc.after.assert(t)
}
}
func TestStripVendorDirectory(t *testing.T) {
t.Run("vendor directory", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "vendor"},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
},
}))
t.Run("vendor file", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
},
files: []fsPath{
{"package", "vendor"},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
files: []fsPath{
{"package", "vendor"},
},
},
}))
}

52
gps/strip_vendor_windows.go
Просмотреть файл

@ -1,52 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gps
import (
"os"
"path/filepath"
)
func stripVendor(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() != "vendor" {
return nil
}
if _, err := os.Lstat(path); err != nil {
return nil
}
symlink := (info.Mode() & os.ModeSymlink) != 0
dir := info.IsDir()
switch {
case symlink && dir:
// This could be a windows junction directory. Support for these in the
// standard library is spotty, and we could easily delete an important
// folder if we called os.Remove or os.RemoveAll. Just skip these.
//
// TODO: If we could distinguish between junctions and Windows symlinks,
// we might be able to safely delete symlinks, even though junctions are
// dangerous.
return filepath.SkipDir
case symlink:
if realInfo, err := os.Stat(path); err == nil && realInfo.IsDir() {
return os.Remove(path)
}
case dir:
if err := os.RemoveAll(path); err != nil {
return err
}
return filepath.SkipDir
}
return nil
}

183
gps/strip_vendor_windows_test.go
Просмотреть файл

@ -1,183 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build windows
package gps
import "testing"
func TestStripVendorSymlinks(t *testing.T) {
// On windows, we skip symlinks, even if they're named 'vendor', because
// they're too hard to distinguish from junctions.
t.Run("vendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "_vendor",
},
},
},
}))
t.Run("nonvendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "link"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
{"package", "_vendor"},
},
links: []fsLink{
{
path: fsPath{"package", "link"},
to: "_vendor",
},
},
},
}))
t.Run("vendor symlink to file", stripVendorTestCase(fsTestCase{
before: filesystemState{
files: []fsPath{
{"file"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "file",
},
},
},
after: filesystemState{
files: []fsPath{
{"file"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "file",
},
},
},
}))
t.Run("broken vendor symlink", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "nonexistence",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "vendor"},
to: "nonexistence",
},
},
},
}))
t.Run("chained symlinks", stripVendorTestCase(fsTestCase{
// Curiously, if a symlink on windows points to *another* symlink which
// eventually points at a directory, we'll correctly remove that first
// symlink, because the first symlink doesn't appear to Go to be a
// directory.
before: filesystemState{
dirs: []fsPath{
{"_vendor"},
},
links: []fsLink{
{
path: fsPath{"vendor"},
to: "vendor2",
},
{
path: fsPath{"vendor2"},
to: "_vendor",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"_vendor"},
},
links: []fsLink{
{
path: fsPath{"vendor2"},
to: "_vendor",
},
},
},
}))
t.Run("circular symlinks", stripVendorTestCase(fsTestCase{
before: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "link1"},
to: "link2",
},
{
path: fsPath{"package", "link2"},
to: "link1",
},
},
},
after: filesystemState{
dirs: []fsPath{
{"package"},
},
links: []fsLink{
{
path: fsPath{"package", "link1"},
to: "link2",
},
{
path: fsPath{"package", "link2"},
to: "link1",
},
},
},
}))
}

82
manifest.go
Просмотреть файл

@ -15,7 +15,7 @@ import (
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/pkgtree"
"github.com/pelletier/go-toml"
toml "github.com/pelletier/go-toml"
"github.com/pkg/errors"
)
@ -51,8 +51,7 @@ type Manifest struct {
Ignored []string
Required []string
PruneOptions gps.PruneOptions
PruneProjectOptions gps.PruneProjectOptions
PruneOptions gps.RootPruneOptions
}
type rawManifest struct {
@ -97,7 +96,7 @@ func NewManifest() *Manifest {
return &Manifest{
Constraints: make(gps.ProjectConstraints),
Ovr: make(gps.ProjectConstraints),
PruneOptions: gps.PruneNestedVendorDirs,
PruneOptions: gps.DefaultRootPruneOptions(),
}
}
@ -303,7 +302,7 @@ func ValidateProjectRoots(c *Ctx, m *Manifest, sm gps.SourceManager) error {
wg.Add(1)
go validate(pr)
}
for pr := range m.PruneProjectOptions {
for pr := range m.PruneOptions.ProjectOptions {
wg.Add(1)
go validate(pr)
}
@ -379,41 +378,67 @@ func fromRawManifest(raw rawManifest) (*Manifest, error) {
m.Ovr[name] = prj
}
m.PruneOptions, m.PruneProjectOptions = fromRawPruneOptions(raw.PruneOptions)
m.PruneOptions = fromRawPruneOptions(raw.PruneOptions)
return m, nil
}
func fromRawPruneOptions(raw rawPruneOptions) (gps.PruneOptions, gps.PruneProjectOptions) {
rootOptions := gps.PruneNestedVendorDirs
pruneProjects := make(gps.PruneProjectOptions)
func fromRawPruneOptions(raw rawPruneOptions) gps.RootPruneOptions {
opts := gps.RootPruneOptions{
PruneOptions: gps.PruneNestedVendorDirs,
ProjectOptions: make(gps.PruneProjectOptions),
}
if raw.UnusedPackages {
rootOptions |= gps.PruneUnusedPackages
opts.PruneOptions |= gps.PruneUnusedPackages
}
if raw.GoTests {
rootOptions |= gps.PruneGoTestFiles
opts.PruneOptions |= gps.PruneGoTestFiles
}
if raw.NonGoFiles {
rootOptions |= gps.PruneNonGoFiles
opts.PruneOptions |= gps.PruneNonGoFiles
}
for _, p := range raw.Projects {
pr := gps.ProjectRoot(p.Name)
pruneProjects[pr] = gps.PruneNestedVendorDirs
opts.ProjectOptions[pr] = gps.PruneNestedVendorDirs
if raw.UnusedPackages {
pruneProjects[pr] |= gps.PruneUnusedPackages
opts.ProjectOptions[pr] |= gps.PruneUnusedPackages
}
if raw.GoTests {
pruneProjects[pr] |= gps.PruneGoTestFiles
opts.ProjectOptions[pr] |= gps.PruneGoTestFiles
}
if raw.NonGoFiles {
pruneProjects[pr] |= gps.PruneNonGoFiles
opts.ProjectOptions[pr] |= gps.PruneNonGoFiles
}
}
return rootOptions, pruneProjects
return opts
}
// toRawPruneOptions converts a gps.RootPruneOption's PruneOptions to rawPruneOptions
//
// Will panic if gps.RootPruneOption includes ProjectPruneOptions
// See https://github.com/golang/dep/pull/1460#discussion_r158128740 for more information
func toRawPruneOptions(root gps.RootPruneOptions) rawPruneOptions {
if len(root.ProjectOptions) != 0 {
panic("toRawPruneOptions cannot convert ProjectOptions to rawPruneOptions")
}
raw := rawPruneOptions{}
if (root.PruneOptions & gps.PruneUnusedPackages) != 0 {
raw.UnusedPackages = true
}
if (root.PruneOptions & gps.PruneNonGoFiles) != 0 {
raw.NonGoFiles = true
}
if (root.PruneOptions & gps.PruneGoTestFiles) != 0 {
raw.GoTests = true
}
return raw
}
// toProject interprets the string representations of project information held in
@ -463,11 +488,10 @@ func (m *Manifest) MarshalTOML() ([]byte, error) {
// toRaw converts the manifest into a representation suitable to write to the manifest file
func (m *Manifest) toRaw() rawManifest {
raw := rawManifest{
Constraints: make([]rawProject, 0, len(m.Constraints)),
Overrides: make([]rawProject, 0, len(m.Ovr)),
Ignored: m.Ignored,
Required: m.Required,
PruneOptions: rawPruneOptions{},
Constraints: make([]rawProject, 0, len(m.Constraints)),
Overrides: make([]rawProject, 0, len(m.Ovr)),
Ignored: m.Ignored,
Required: m.Required,
}
for n, prj := range m.Constraints {
@ -480,7 +504,7 @@ func (m *Manifest) toRaw() rawManifest {
}
sort.Sort(sortedRawProjects(raw.Overrides))
// TODO(ibrasho): write out prune options.
raw.PruneOptions = toRawPruneOptions(m.PruneOptions)
return raw
}
@ -574,15 +598,3 @@ func (m *Manifest) RequiredPackages() map[string]bool {
return mp
}
// PruneOptionsFor returns the prune options for the passed project root.
//
// It will return the root prune options if the project does not have specific
// options or if it does not exists in the manifest.
func (m *Manifest) PruneOptionsFor(pr gps.ProjectRoot) gps.PruneOptions {
if po, ok := m.PruneProjectOptions[pr]; ok {
return po
}
return m.PruneOptions
}

60
manifest_test.go
Просмотреть файл

@ -45,11 +45,13 @@ func TestReadManifest(t *testing.T) {
Constraint: gps.NewBranch("master"),
},
},
Ignored: []string{"github.com/foo/bar"},
PruneOptions: gps.PruneNestedVendorDirs | gps.PruneNonGoFiles,
PruneProjectOptions: gps.PruneProjectOptions{
gps.ProjectRoot("github.com/golang/dep"): gps.PruneNestedVendorDirs,
gps.ProjectRoot("github.com/babble/brook"): gps.PruneNestedVendorDirs | gps.PruneGoTestFiles,
Ignored: []string{"github.com/foo/bar"},
PruneOptions: gps.RootPruneOptions{
PruneOptions: gps.PruneNestedVendorDirs | gps.PruneNonGoFiles,
ProjectOptions: gps.PruneProjectOptions{
gps.ProjectRoot("github.com/golang/dep"): gps.PruneNestedVendorDirs,
gps.ProjectRoot("github.com/babble/brook"): gps.PruneNestedVendorDirs | gps.PruneGoTestFiles,
},
},
}
@ -606,8 +608,54 @@ func TestValidateProjectRoots(t *testing.T) {
}
}
func TestPruneOptionsFor(t *testing.T) {
func TestToRawPruneOptions(t *testing.T) {
cases := []struct {
name string
pruneOptions gps.RootPruneOptions
wantOptions rawPruneOptions
}{
{
name: "all options",
pruneOptions: gps.RootPruneOptions{PruneOptions: 15},
wantOptions: rawPruneOptions{
UnusedPackages: true,
NonGoFiles: true,
GoTests: true,
},
},
{
name: "no options",
pruneOptions: gps.RootPruneOptions{PruneOptions: 1},
wantOptions: rawPruneOptions{
UnusedPackages: false,
NonGoFiles: false,
GoTests: false,
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
raw := toRawPruneOptions(c.pruneOptions)
if !reflect.DeepEqual(raw, c.wantOptions) {
t.Fatalf("rawPruneOptions are not as expected:\n\t(GOT) %v\n\t(WNT) %v", raw, c.wantOptions)
}
})
}
}
func TestToRawPruneOptions_Panic(t *testing.T) {
pruneOptions := gps.RootPruneOptions{
PruneOptions: 1,
ProjectOptions: gps.PruneProjectOptions{"github.com/carolynvs/deptest": 1},
}
defer func() {
if err := recover(); err == nil {
t.Error("toRawPruneOptions did not panic with non-empty ProjectOptions")
}
}()
_ = toRawPruneOptions(pruneOptions)
}
func containsErr(s []error, e error) bool {

9
testdata/txn_writer/expected_manifest.toml поставляемый
Просмотреть файл

@ -16,8 +16,13 @@
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]

29
txn_writer.go
Просмотреть файл

@ -39,8 +39,13 @@ var exampleTOML = []byte(`# Gopkg.toml example
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
`)
@ -56,11 +61,12 @@ var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes
// It is not impervious to errors (writing to disk is hard), but it should
// guard against non-arcane failure conditions.
type SafeWriter struct {
Manifest *Manifest
lock *Lock
lockDiff *gps.LockDiff
writeVendor bool
writeLock bool
Manifest *Manifest
lock *Lock
lockDiff *gps.LockDiff
writeVendor bool
writeLock bool
pruneOptions gps.RootPruneOptions
}
// NewSafeWriter sets up a SafeWriter to write a set of manifest, lock, and
@ -78,10 +84,11 @@ type SafeWriter struct {
// - If oldLock is provided without newLock, error.
//
// - If vendor is VendorAlways without a newLock, error.
func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBehavior) (*SafeWriter, error) {
func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBehavior, prune gps.RootPruneOptions) (*SafeWriter, error) {
sw := &SafeWriter{
Manifest: manifest,
lock: newLock,
Manifest: manifest,
lock: newLock,
pruneOptions: prune,
}
if oldLock != nil {
@ -312,7 +319,7 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
}
if sw.writeVendor {
err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true, logger)
err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, sw.pruneOptions, logger)
if err != nil {
return errors.Wrap(err, "error while writing out vendor tree")
}

35
txn_writer_test.go
Просмотреть файл

@ -11,6 +11,7 @@ import (
"strings"
"testing"
"github.com/golang/dep/gps"
"github.com/golang/dep/internal/test"
"github.com/pkg/errors"
)
@ -25,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) {
pc := NewTestProjectContext(h, safeWriterProject)
defer pc.Release()
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged, gps.DefaultRootPruneOptions())
err := sw.Write("", pc.SourceManager, true, discardLogger())
if err == nil {
@ -43,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways, gps.DefaultRootPruneOptions())
err := sw.Write(pc.Project.AbsRoot, nil, true, discardLogger())
if err == nil {
@ -59,7 +60,7 @@ func TestSafeWriter_BadInput_ForceVendorMissingLock(t *testing.T) {
pc := NewTestProjectContext(h, safeWriterProject)
defer pc.Release()
_, err := NewSafeWriter(nil, nil, nil, VendorAlways)
_, err := NewSafeWriter(nil, nil, nil, VendorAlways, gps.DefaultRootPruneOptions())
if err == nil {
t.Fatal("should have errored without a lock when forceVendor is true, but did not")
} else if !strings.Contains(err.Error(), "newLock") {
@ -75,7 +76,7 @@ func TestSafeWriter_BadInput_OldLockOnly(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
_, err := NewSafeWriter(nil, pc.Project.Lock, nil, VendorAlways)
_, err := NewSafeWriter(nil, pc.Project.Lock, nil, VendorAlways, gps.DefaultRootPruneOptions())
if err == nil {
t.Fatal("should have errored with only an old lock, but did not")
} else if !strings.Contains(err.Error(), "oldLock") {
@ -89,7 +90,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) {
pc := NewTestProjectContext(h, safeWriterProject)
defer pc.Release()
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged, gps.DefaultRootPruneOptions())
missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent")
err := sw.Write(missingroot, pc.SourceManager, true, discardLogger())
@ -107,7 +108,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) {
pc := NewTestProjectContext(h, safeWriterProject)
defer pc.Release()
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged, gps.DefaultRootPruneOptions())
fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot")
err := sw.Write(fileroot, pc.SourceManager, true, discardLogger())
@ -131,7 +132,7 @@ func TestSafeWriter_Manifest(t *testing.T) {
pc.CopyFile(ManifestName, safeWriterGoldenManifest)
pc.Load()
sw, _ := NewSafeWriter(pc.Project.Manifest, nil, nil, VendorOnChanged)
sw, _ := NewSafeWriter(pc.Project.Manifest, nil, nil, VendorOnChanged, gps.DefaultRootPruneOptions())
// Verify prepared actions
if !sw.HasManifest() {
@ -173,7 +174,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorOnChanged)
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorOnChanged, gps.DefaultRootPruneOptions())
// Verify prepared actions
if !sw.HasManifest() {
@ -218,7 +219,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorAlways)
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorAlways, gps.DefaultRootPruneOptions())
// Verify prepared actions
if !sw.HasManifest() {
@ -268,7 +269,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) {
originalLock := new(Lock)
*originalLock = *pc.Project.Lock
originalLock.SolveMeta.InputsDigest = []byte{} // zero out the input hash to ensure non-equivalency
sw, _ := NewSafeWriter(nil, originalLock, pc.Project.Lock, VendorOnChanged)
sw, _ := NewSafeWriter(nil, originalLock, pc.Project.Lock, VendorOnChanged, gps.DefaultRootPruneOptions())
// Verify prepared actions
if sw.HasManifest() {
@ -318,7 +319,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) {
originalLock := new(Lock)
*originalLock = *pc.Project.Lock
originalLock.SolveMeta.InputsDigest = []byte{} // zero out the input hash to ensure non-equivalency
sw, _ := NewSafeWriter(nil, originalLock, pc.Project.Lock, VendorNever)
sw, _ := NewSafeWriter(nil, originalLock, pc.Project.Lock, VendorNever, gps.DefaultRootPruneOptions())
// Verify prepared actions
if sw.HasManifest() {
@ -362,12 +363,12 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways)
sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways, gps.DefaultRootPruneOptions())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))
// Verify prepared actions
sw, _ = NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
sw, _ = NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways, gps.DefaultRootPruneOptions())
if sw.HasManifest() {
t.Fatal("Did not expect the payload to contain the manifest")
}
@ -414,7 +415,7 @@ func TestSafeWriter_NewLock(t *testing.T) {
defer lf.Close()
newLock, err := readLock(lf)
h.Must(err)
sw, _ := NewSafeWriter(nil, nil, newLock, VendorOnChanged)
sw, _ := NewSafeWriter(nil, nil, newLock, VendorOnChanged, gps.DefaultRootPruneOptions())
// Verify prepared actions
if sw.HasManifest() {
@ -461,7 +462,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) {
defer lf.Close()
newLock, err := readLock(lf)
h.Must(err)
sw, _ := NewSafeWriter(nil, nil, newLock, VendorNever)
sw, _ := NewSafeWriter(nil, nil, newLock, VendorNever, gps.DefaultRootPruneOptions())
// Verify prepared actions
if sw.HasManifest() {
@ -510,7 +511,7 @@ func TestSafeWriter_DiffLocks(t *testing.T) {
updatedLock, err := readLock(ulf)
h.Must(err)
sw, _ := NewSafeWriter(nil, pc.Project.Lock, updatedLock, VendorOnChanged)
sw, _ := NewSafeWriter(nil, pc.Project.Lock, updatedLock, VendorOnChanged, gps.DefaultRootPruneOptions())
// Verify lock diff
diff := sw.lockDiff
@ -555,7 +556,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) {
pc.CopyFile(LockName, safeWriterGoldenLock)
pc.Load()
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorAlways)
sw, _ := NewSafeWriter(pc.Project.Manifest, pc.Project.Lock, pc.Project.Lock, VendorAlways, gps.DefaultRootPruneOptions())
// Verify prepared actions
if !sw.HasManifest() {