go
/
dep
зеркало из https://github.com/golang/dep.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

dep: Get DeltaWriter into a working state 

This encompasses the first pass at the new, more abstracted diffing
system, and the DeltaWriter implementation on top of it. Tests are
needed, but cursory testing indicates that we successfully capture all
types of diffs and regenerate only the subset of projects that actually
need to be touched.
This commit is contained in:
sam boyer 2018-06-28 01:32:17 -04:00
Родитель 0b2482d688
Коммит df2c26b777
23 изменённых файлов: 573 добавлений и 232 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

43
Gopkg.lock сгенерированный
Просмотреть файл

@ -3,91 +3,132 @@
[[projects]]
branch = "2.x"
digest = "1:ee2887fecb4d923fa90f8dd9cf33e876bf9260fed62f2ca5a5c3f41b4eb07683"
name = "github.com/Masterminds/semver"
packages = ["."]
pruneopts = "NUT"
revision = "24642bd0573145a5ee04f9be773641695289be46"
[[projects]]
digest = "1:442020d26d1f891d5014cae4353b6ff589562c2b303504627de3660adf3fb217"
name = "github.com/Masterminds/vcs"
packages = ["."]
pruneopts = "NUT"
revision = "3084677c2c188840777bff30054f2b553729d329"
version = "v1.11.1"
[[projects]]
branch = "master"
digest = "1:60861e762bdbe39c4c7bf292c291329b731c9925388fd41125888f5c1c595feb"
name = "github.com/armon/go-radix"
packages = ["."]
pruneopts = "NUT"
revision = "4239b77079c7b5d1243b7b4736304ce8ddb6f0f2"
[[projects]]
digest = "1:a12d94258c5298ead75e142e8001224bf029f302fed9e96cd39c0eaf90f3954d"
name = "github.com/boltdb/bolt"
packages = ["."]
pruneopts = "NUT"
revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
version = "v1.3.1"
[[projects]]
digest = "1:9f35c1344b56e5868d511d231f215edd0650aa572664f856444affdd256e43e4"
name = "github.com/golang/protobuf"
packages = ["proto"]
pruneopts = "NUT"
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
[[projects]]
digest = "1:f5169729244becc423886eae4d72547e28ac3f13f861bed8a9d749bc7238a1c3"
name = "github.com/jmank88/nuts"
packages = ["."]
pruneopts = "NUT"
revision = "8b28145dffc87104e66d074f62ea8080edfad7c8"
version = "v0.3.0"
[[projects]]
branch = "master"
digest = "1:01af3a6abe28784782680e1f75ef8767cfc5d4b230dc156ff7eb8db395cbbfd2"
name = "github.com/nightlyone/lockfile"
packages = ["."]
pruneopts = "NUT"
revision = "e83dc5e7bba095e8d32fb2124714bf41f2a30cb5"
[[projects]]
digest = "1:13b8f1a2ce177961dc9231606a52f709fab896c565f3988f60a7f6b4e543a902"
name = "github.com/pelletier/go-toml"
packages = ["."]
pruneopts = "NUT"
revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
version = "v1.1.0"
[[projects]]
digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = "NUT"
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
branch = "master"
digest = "1:abb4b60c28323cde32c193ce6083bb600fac462d1780cf83461b4c23ed5ce904"
name = "github.com/sdboyer/constext"
packages = ["."]
pruneopts = "NUT"
revision = "836a144573533ea4da4e6929c235fd348aed1c80"
[[projects]]
branch = "master"
digest = "1:6ad2104db8f34b8656382ef0a7297b9a5cc42e7bdce95d968e02b92fc97470d1"
name = "golang.org/x/net"
packages = ["context"]
pruneopts = "NUT"
revision = "66aacef3dd8a676686c7ae3716979581e8b03c47"
[[projects]]
branch = "master"
digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239"
name = "golang.org/x/sync"
packages = ["errgroup"]
pruneopts = "NUT"
revision = "f52d1811a62927559de87708c8913c1650ce4f26"
[[projects]]
branch = "master"
digest = "1:51912e607c5e28a89fdc7e41d3377b92086ab7f76ded236765dbf98d0a704c5d"
name = "golang.org/x/sys"
packages = ["unix"]
pruneopts = "NUT"
revision = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366"
[[projects]]
branch = "v2"
digest = "1:13e704c08924325be00f96e47e7efe0bfddf0913cdfc237423c83f9b183ff590"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = "NUT"
revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "460ad7112866da4b9a0a626aa3e2fe80699c17bf871afb73b93f836418fb9298"
input-imports = [
"github.com/Masterminds/semver",
"github.com/Masterminds/vcs",
"github.com/armon/go-radix",
"github.com/boltdb/bolt",
"github.com/golang/protobuf/proto",
"github.com/jmank88/nuts",
"github.com/nightlyone/lockfile",
"github.com/pelletier/go-toml",
"github.com/pkg/errors",
"github.com/sdboyer/constext",
"golang.org/x/sync/errgroup",
"gopkg.in/yaml.v2"
]
solver-name = "gps-cdcl"
solver-version = 1

123
cmd/dep/ensure.go
Просмотреть файл

@ -5,7 +5,6 @@
package main
import (
"bytes"
"context"
"flag"
"fmt"
@ -21,6 +20,7 @@ import (
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/paths"
"github.com/golang/dep/gps/pkgtree"
"github.com/golang/dep/gps/verify"
"github.com/pkg/errors"
)
@ -184,6 +184,33 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error {
return cmd.runVendorOnly(ctx, args, p, sm, params)
}
statchan := make(chan map[string]verify.VendorStatus)
var lps []gps.LockedProject
if p.Lock != nil {
lps = p.Lock.Projects()
}
go func(vendorDir string, p []gps.LockedProject) {
// Make sure vendor dir exists
err := os.MkdirAll(vendorDir, os.FileMode(0777))
if err != nil {
ctx.Err.Printf("Error creating vendor directory: %q", err.Error())
// TODO(sdboyer) handle these better
os.Exit(1)
}
sums := make(map[string]verify.VersionedDigest)
for _, lp := range p {
sums[string(lp.Ident().ProjectRoot)] = lp.(verify.VerifiableProject).Digest
}
status, err := verify.VerifyDepTree(vendorDir, sums)
if err != nil {
ctx.Err.Printf("Error while verifying vendor directory: %q", err.Error())
os.Exit(1)
}
statchan <- status
}(filepath.Join(p.AbsRoot, "vendor"), lps)
params.RootPackageTree, err = p.ParseRootPackageTree()
if err != nil {
return err
@ -212,11 +239,11 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error {
}
if cmd.add {
return cmd.runAdd(ctx, args, p, sm, params)
return cmd.runAdd(ctx, args, p, sm, params, statchan)
} else if cmd.update {
return cmd.runUpdate(ctx, args, p, sm, params)
return cmd.runUpdate(ctx, args, p, sm, params, statchan)
}
return cmd.runDefault(ctx, args, p, sm, params)
return cmd.runDefault(ctx, args, p, sm, params, statchan)
}
func (cmd *ensureCommand) validateFlags() error {
@ -246,7 +273,7 @@ func (cmd *ensureCommand) vendorBehavior() dep.VendorBehavior {
return dep.VendorOnChanged
}
func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error {
// Bare ensure doesn't take any args.
if len(args) != 0 {
return errors.New("dep ensure only takes spec arguments with -add or -update")
@ -256,52 +283,37 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
return err
}
if lsat, err := p.LockSatisfiesInputs(sm); err != nil {
return err
} else if !lsat.Passes() {
if ctx.Verbose {
ctx.Out.Printf("%s was already in sync with imports and %s\n", dep.LockName, dep.ManifestName)
}
lock := p.Lock
if lock != nil {
lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree)
if !lsat.Passed() {
// TODO(sdboyer) print out what bits are unsatisfied here
solver, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "prepare solver")
}
if cmd.noVendor {
if cmd.noVendor && cmd.dryRun {
return errors.New("Gopkg.lock was not up to date")
}
solution, err := solver.Solve(context.TODO())
if err != nil {
return handleAllTheFailuresOfTheWorld(err)
}
lock = dep.LockFromSolution(solution, p.Manifest.PruneOptions)
} else if cmd.noVendor {
// The user said not to touch vendor/, so definitely nothing to do.
return nil
}
sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorOnChanged, p.Manifest.PruneOptions)
if err != nil {
return err
}
if cmd.dryRun {
return sw.PrintPreparedActions(ctx.Out, ctx.Verbose)
}
var logger *log.Logger
if ctx.Verbose {
logger = ctx.Err
}
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
}
solver, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "prepare solver")
}
if cmd.noVendor && cmd.dryRun {
return errors.New("Gopkg.lock was not up to date")
}
solution, err := solver.Solve(context.TODO())
if err != nil {
return handleAllTheFailuresOfTheWorld(err)
}
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions)
sw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor"))
if err != nil {
return err
}
if cmd.dryRun {
return sw.PrintPreparedActions(ctx.Out, ctx.Verbose)
}
@ -310,7 +322,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
if ctx.Verbose {
logger = ctx.Err
}
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
}
func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
@ -339,7 +351,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
}
func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error {
if p.Lock == nil {
return errors.Errorf("-update works by updating the versions recorded in %s, but %s does not exist", dep.LockName, dep.LockName)
}
@ -348,14 +360,6 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
return err
}
// We'll need to discard this prepared solver as later work changes params,
// but solver preparation is cheap and worth doing up front in order to
// perform the fastpath check of hash comparison.
solver, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "fastpath solver prepare")
}
// When -update is specified without args, allow every dependency to change
// versions, regardless of the lock file.
if len(args) == 0 {
@ -367,7 +371,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
}
// Re-prepare a solver now that our params are complete.
solver, err = gps.Prepare(params, sm)
solver, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "fastpath solver prepare")
}
@ -379,7 +383,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
return handleAllTheFailuresOfTheWorld(err)
}
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions)
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior(), p.Manifest.PruneOptions)
if err != nil {
return err
}
@ -394,7 +398,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
}
func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error {
if len(args) == 0 {
return errors.New("must specify at least one project or package to -add")
}
@ -411,15 +415,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
return errors.Wrap(err, "fastpath solver prepare")
}
// Compare the hashes. If they're not equal, bail out and ask the user to
// run a straight `dep ensure` before updating. This is handholding the
// user a bit, but the extra effort required is minimal, and it ensures the
// user is isolating variables in the event of solve problems (was it the
// "pending" changes, or the -add that caused the problem?).
if p.Lock != nil && !bytes.Equal(p.Lock.InputsDigest(), solver.HashInputs()) {
ctx.Out.Printf("Warning: %s is out of sync with %s or the project's imports.", dep.LockName, dep.ManifestName)
}
rm, _ := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages())
// TODO(sdboyer) re-enable this once we ToReachMap() intelligently filters out normally-excluded (_*, .*), dirs from errmap
@ -678,7 +673,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
}
sort.Strings(reqlist)
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), dep.VendorOnChanged, p.Manifest.PruneOptions)
sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), dep.VendorOnChanged, p.Manifest.PruneOptions)
if err != nil {
return err
}

4
cmd/dep/ensure_test.go
Просмотреть файл

@ -50,11 +50,11 @@ func TestInvalidEnsureFlagCombinations(t *testing.T) {
// anything other than the error being non-nil. For now, it works well
// because a panic will quickly result if the initial arg length validation
// checks are incorrectly handled.
if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}); err == nil {
if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}, nil); err == nil {
t.Errorf("no args to plain ensure with -vendor-only")
}
ec.vendorOnly = false
if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}); err == nil {
if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}, nil); err == nil {
t.Errorf("no args to plain ensure")
}
}

4
cmd/dep/init.go
Просмотреть файл

@ -157,7 +157,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
err = handleAllTheFailuresOfTheWorld(err)
return errors.Wrap(err, "init failed: unable to solve the dependency graph")
}
p.Lock = dep.LockFromSolution(soln)
p.Lock = dep.LockFromSolution(soln, p.Manifest.PruneOptions)
rootAnalyzer.FinalizeRootManifestAndLock(p.Manifest, p.Lock, copyLock)
@ -168,7 +168,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
return errors.Wrap(err, "init failed: unable to recalculate the lock digest")
}
p.Lock.SolveMeta.InputsDigest = s.HashInputs()
//p.Lock.SolveMeta.InputsDigest = s.HashInputs()
// Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name.
vendorbak, err := dep.BackupVendor(filepath.Join(root, "vendor"), time.Now().Format("20060102150405"))

28
cmd/dep/integration_test.go
Просмотреть файл

@ -74,34 +74,6 @@ func TestDepCachedir(t *testing.T) {
initPath := filepath.Join("testdata", "cachedir")
t.Run("env-cachedir", func(t *testing.T) {
t.Parallel()
testProj := integration.NewTestProject(t, initPath, wd, runMain)
defer testProj.Cleanup()
testProj.TempDir("cachedir")
cachedir := testProj.Path("cachedir")
testProj.Setenv("DEPCACHEDIR", cachedir)
// Running `dep ensure` will pull in the dependency into cachedir.
err = testProj.DoRun([]string{"ensure"})
if err != nil {
// Log the error output from running `dep ensure`, could be useful.
t.Logf("`dep ensure` error output: \n%s", testProj.GetStderr())
t.Errorf("got an unexpected error: %s", err)
}
// Check that the cache was created in the cachedir. Our fixture has the dependency
// `github.com/sdboyer/deptest`
_, err = os.Stat(testProj.Path("cachedir", "sources", "https---github.com-sdboyer-deptest"))
if err != nil {
if os.IsNotExist(err) {
t.Error("expected cachedir to have been populated but none was found")
} else {
t.Errorf("got an unexpected error: %s", err)
}
}
})
t.Run("env-invalid-cachedir", func(t *testing.T) {
t.Parallel()
testProj := integration.NewTestProject(t, initPath, wd, runMain)

1
cmd/dep/main.go
Просмотреть файл

@ -91,7 +91,6 @@ func (c *Config) Run() int {
&statusCommand{},
&ensureCommand{},
&pruneCommand{},
&hashinCommand{},
&versionCommand{},
}

10
cmd/dep/prune.go
Просмотреть файл

@ -5,7 +5,6 @@
package main
import (
"bytes"
"flag"
"io/ioutil"
"log"
@ -75,19 +74,10 @@ func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
params.TraceLogger = ctx.Err
}
s, err := gps.Prepare(params, sm)
if err != nil {
return errors.Wrap(err, "could not set up solver for input hashing")
}
if p.Lock == nil {
return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
}
if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
}
pruneLogger := ctx.Err
if !ctx.Verbose {
pruneLogger = log.New(ioutil.Discard, "", 0)

3
cmd/dep/root_analyzer.go
Просмотреть файл

@ -11,6 +11,7 @@ import (
"github.com/golang/dep"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/verify"
fb "github.com/golang/dep/internal/feedback"
"github.com/golang/dep/internal/importers"
"golang.org/x/sync/errgroup"
@ -167,7 +168,7 @@ func (a *rootAnalyzer) DeriveManifestAndLock(dir string, pr gps.ProjectRoot) (gp
func (a *rootAnalyzer) FinalizeRootManifestAndLock(m *dep.Manifest, l *dep.Lock, ol dep.Lock) {
// Iterate through the new projects in solved lock and add them to manifest
// if they are direct deps and log feedback for all the new projects.
diff := gps.DiffLocks(&ol, l)
diff := verify.DiffLocks(&ol, l)
bi := fb.NewBrokenImportFeedback(diff)
bi.LogFeedback(a.ctx.Err)
for _, y := range l.Projects() {

9
cmd/dep/status.go
Просмотреть файл

@ -24,6 +24,7 @@ import (
"github.com/golang/dep"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/paths"
"github.com/golang/dep/gps/verify"
"github.com/pkg/errors"
)
@ -912,11 +913,6 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje
return false, 0, err
}
s, err := gps.Prepare(params, sm)
if err != nil {
return false, 0, errors.Wrapf(err, "could not set up solver for input hashing")
}
// Errors while collecting constraints should not fail the whole status run.
// It should count the error and tell the user about incomplete results.
cm, ccerrs := collectConstraints(ctx, p, sm)
@ -932,7 +928,8 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje
return slp[i].Ident().Less(slp[j].Ident())
})
if bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree)
if lsat.Passed() {
// If these are equal, we're guaranteed that the lock is a transitively
// complete picture of all deps. That eliminates the need for at least
// some checks.

13
gps/solver.go
Просмотреть файл

@ -450,11 +450,16 @@ func (s *solver) Solve(ctx context.Context) (Solution, error) {
soln.i = s.rd.externalImportList(s.stdLibFn)
// Convert ProjectAtoms into LockedProjects
soln.p = make([]LockedProject, len(all))
k := 0
soln.p = make([]LockedProject, 0, len(all))
for pa, pl := range all {
soln.p[k] = pa2lp(pa, pl)
k++
lp := pa2lp(pa, pl)
// Pass back the original inputlp directly if it Eqs what was
// selected.
if inputlp, has := s.rd.rlm[lp.Ident().ProjectRoot]; has && lp.Eq(inputlp) {
lp = inputlp
}
soln.p = append(soln.p, lp)
}
}

17
gps/source_cache.go
Просмотреть файл

@ -271,3 +271,20 @@ func (c *singleSourceCacheMemory) toUnpaired(v Version) (UnpairedVersion, bool)
panic(fmt.Sprintf("unknown version type %T", v))
}
}
// TODO(sdboyer) remove once source caching can be moved into separate package
func locksAreEq(l1, l2 Lock) bool {
p1, p2 := l1.Projects(), l2.Projects()
if len(p1) != len(p2) {
return false
}
p1, p2 = sortLockedProjects(p1), sortLockedProjects(p2)
for k, lp := range p1 {
if !lp.Eq(p2[k]) {
return false
}
}
return true
}

5
gps/source_cache_bolt_encode.go
Просмотреть файл

@ -342,8 +342,9 @@ func cachePutLock(b *bolt.Bucket, l Lock) error {
// cacheGetLock returns a new *safeLock with the fields retrieved from the bolt.Bucket.
func cacheGetLock(b *bolt.Bucket) (*safeLock, error) {
l := &safeLock{
i: strings.Split(string(b.Get(cacheKeyInputImports)), "#"),
l := &safeLock{}
if ii := b.Get(cacheKeyInputImports); len(ii) > 0 {
l.i = strings.Split(string(ii), "#")
}
if locked := b.Bucket(cacheKeyLock); locked != nil {

22
gps/source_cache_bolt_test.go
Просмотреть файл

@ -53,7 +53,6 @@ func TestBoltCacheTimeout(t *testing.T) {
}
lock := &safeLock{
//h: []byte("test_hash"),
p: []LockedProject{
NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0"), nil),
@ -120,8 +119,9 @@ func TestBoltCacheTimeout(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, manifest, gotM)
if dl := DiffLocks(lock, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(lock, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL)
}
got, ok := c.getPackageTree(rev, root)
@ -162,8 +162,9 @@ func TestBoltCacheTimeout(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, manifest, gotM)
if dl := DiffLocks(lock, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(lock, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL)
}
gotPtree, ok := c.getPackageTree(rev, root)
@ -195,8 +196,9 @@ func TestBoltCacheTimeout(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, manifest, gotM)
if dl := DiffLocks(lock, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(lock, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL)
}
got, ok := c.getPackageTree(rev, root)
@ -233,7 +235,6 @@ func TestBoltCacheTimeout(t *testing.T) {
}
newLock := &safeLock{
//h: []byte("new_test_hash"),
p: []LockedProject{
NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v1"), []string{"gps"}),
},
@ -283,8 +284,9 @@ func TestBoltCacheTimeout(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, newManifest, gotM)
if dl := DiffLocks(newLock, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(lock, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL)
}
got, ok := c.getPackageTree(rev, root)

24
gps/source_cache_test.go
Просмотреть файл

@ -116,13 +116,12 @@ func (test singleSourceCacheTest) run(t *testing.T) {
ig: pkgtree.NewIgnoredRuleset([]string{"a", "b"}),
}
var l Lock = &safeLock{
//h: []byte("test_hash"),
p: []LockedProject{
NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0"), nil),
NewLockedProject(mkPI("github.com/sdboyer/gps3"), NewVersion("v0.10.0"), []string{"gps", "flugle"}),
NewLockedProject(mkPI("foo"), NewVersion("nada"), []string{"foo"}),
NewLockedProject(mkPI("github.com/sdboyer/gps4"), NewVersion("v0.10.0"), []string{"flugle", "gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0").Pair("anything"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0").Pair("whatever"), nil),
NewLockedProject(mkPI("github.com/sdboyer/gps3"), NewVersion("v0.10.0").Pair("again"), []string{"gps", "flugle"}),
NewLockedProject(mkPI("foo"), NewVersion("nada").Pair("itsaliving"), []string{"foo"}),
NewLockedProject(mkPI("github.com/sdboyer/gps4"), NewVersion("v0.10.0").Pair("meow"), []string{"flugle", "gps"}),
},
}
c.setManifestAndLock(rev, testAnalyzerInfo, m, l)
@ -140,8 +139,9 @@ func (test singleSourceCacheTest) run(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, m, gotM)
if dl := DiffLocks(l, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(l, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", l, gotL)
}
m = &simpleRootManifest{
@ -163,10 +163,9 @@ func (test singleSourceCacheTest) run(t *testing.T) {
ig: pkgtree.NewIgnoredRuleset([]string{"c", "d"}),
}
l = &safeLock{
//h: []byte("different_test_hash"),
p: []LockedProject{
NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0").Pair("278a227dfc3d595a33a77ff3f841fd8ca1bc8cd0"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.11.0"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.11.0").Pair("anything"), []string{"gps"}),
NewLockedProject(mkPI("github.com/sdboyer/gps3"), Revision("278a227dfc3d595a33a77ff3f841fd8ca1bc8cd0"), []string{"gps"}),
},
}
@ -185,8 +184,9 @@ func (test singleSourceCacheTest) run(t *testing.T) {
t.Error("no manifest and lock found for revision")
}
compareManifests(t, m, gotM)
if dl := DiffLocks(l, gotL); dl != nil {
t.Errorf("lock differences:\n\t %#v", dl)
// TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles
if !locksAreEq(l, gotL) {
t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", l, gotL)
}
})

13
gps/verify/digest.go
Просмотреть файл

@ -347,6 +347,11 @@ func (vd VersionedDigest) String() string {
return fmt.Sprintf("%s:%s", strconv.Itoa(vd.HashVersion), hex.EncodeToString(vd.Digest))
}
// IsEmpty indicates if the VersionedDigest is the zero value.
func (vd VersionedDigest) IsEmpty() bool {
return vd.HashVersion == 0 && len(vd.Digest) == 0
}
// ParseVersionedDigest decodes the string representation of versioned digest
// information - a colon-separated string with a version number in the first
// part and the hex-encdoed hash digest in the second - as a VersionedDigest.
@ -378,7 +383,7 @@ func ParseVersionedDigest(input string) (VersionedDigest, error) {
// platform where the file system path separator is a character other than
// solidus, one particular dependency would be represented as
// "github.com/alice/alice1".
func VerifyDepTree(osDirname string, wantDigests map[string][]byte) (map[string]VendorStatus, error) {
func VerifyDepTree(osDirname string, wantDigests map[string]VersionedDigest) (map[string]VendorStatus, error) {
osDirname = filepath.Clean(osDirname)
// Ensure top level pathname is a directory
@ -455,12 +460,14 @@ func VerifyDepTree(osDirname string, wantDigests map[string][]byte) (map[string]
if expectedSum, ok := wantDigests[slashPathname]; ok {
ls := EmptyDigestInLock
if len(expectedSum) > 0 {
if expectedSum.HashVersion != HashVersion {
ls = HashVersionMismatch
} else if len(expectedSum.Digest) > 0 {
projectSum, err := DigestFromDirectory(osPathname)
if err != nil {
return nil, errors.Wrap(err, "cannot compute dependency hash")
}
if bytes.Equal(projectSum.Digest, expectedSum) {
if bytes.Equal(projectSum.Digest, expectedSum.Digest) {
ls = NoMismatch
} else {
ls = DigestMismatchInLock

245
gps/verify/lockdiff.go
Просмотреть файл

@ -5,6 +5,7 @@
package verify
import (
"bytes"
"fmt"
"sort"
"strings"
@ -49,8 +50,10 @@ func sortLockedProjects(lps []gps.LockedProject) []gps.LockedProject {
}) {
return lps
}
cp := make([]gps.LockedProject, len(lps))
copy(cp, lps)
sort.Slice(cp, func(i, j int) bool {
return cp[i].Ident().Less(cp[j].Ident())
})
@ -65,6 +68,12 @@ type LockDiff struct {
Modify []LockedProjectDiff
}
type LockDiff2 struct {
AddedImportInputs []string
RemovedImportInputs []string
ProjectDiffs map[gps.ProjectRoot]LockedProjectDiff2
}
// LockedProjectDiff contains the before and after snapshot of a project reference.
// Fields are only populated when there is a difference, otherwise they are empty.
type LockedProjectDiff struct {
@ -76,15 +85,247 @@ type LockedProjectDiff struct {
Packages []StringDiff
}
type LockedProjectDiff2 struct {
Name gps.ProjectRoot
ProjectRemoved, ProjectAdded bool
LockedProjectPartsDiff
}
type LockedProjectPartsDiff struct {
PackagesAdded, PackagesRemoved []string
VersionBefore, VersionAfter gps.UnpairedVersion
RevisionBefore, RevisionAfter gps.Revision
SourceBefore, SourceAfter string
PruneOptsBefore, PruneOptsAfter gps.PruneOptions
HashChanged, HashVersionChanged bool
}
// DiffLocks compares two locks and identifies the differences between them.
// Returns nil if there are no differences.
func DiffLocks2(l1, l2 gps.Lock) LockDiff2 {
// Default nil locks to empty locks, so that we can still generate a diff
if l1 == nil {
if l2 == nil {
return LockDiff2{}
}
l1 = gps.SimpleLock{}
}
if l2 == nil {
l2 = gps.SimpleLock{}
}
p1, p2 := l1.Projects(), l2.Projects()
p1 = sortLockedProjects(p1)
p2 = sortLockedProjects(p2)
diff := LockDiff2{
ProjectDiffs: make(map[gps.ProjectRoot]LockedProjectDiff2),
}
var i2next int
for i1 := 0; i1 < len(p1); i1++ {
lp1 := p1[i1]
pr1 := lp1.Ident().ProjectRoot
lpd := LockedProjectDiff2{
Name: pr1,
}
for i2 := i2next; i2 < len(p2); i2++ {
lp2 := p2[i2]
pr2 := lp2.Ident().ProjectRoot
switch strings.Compare(string(pr1), string(pr2)) {
case 0: // Found a matching project
lpd.LockedProjectPartsDiff = DiffProjects2(lp1, lp2)
i2next = i2 + 1 // Don't visit this project again
case +1: // Found a new project
diff.ProjectDiffs[pr2] = LockedProjectDiff2{
Name: pr2,
ProjectAdded: true,
}
i2next = i2 + 1 // Don't visit this project again
continue // Keep looking for a matching project
case -1: // Project has been removed, handled below
lpd.ProjectRemoved = true
}
break // Done evaluating this project, move onto the next
}
diff.ProjectDiffs[pr1] = lpd
}
// Anything that still hasn't been evaluated are adds
for i2 := i2next; i2 < len(p2); i2++ {
lp2 := p2[i2]
pr2 := lp2.Ident().ProjectRoot
diff.ProjectDiffs[pr2] = LockedProjectDiff2{
Name: pr2,
ProjectAdded: true,
}
}
// Only do the import inputs if both of the locks fulfill the interface, AND
// both have non-empty inputs.
il1, ok1 := l1.(gps.LockWithImports)
il2, ok2 := l2.(gps.LockWithImports)
if ok1 && ok2 && len(il1.InputImports()) > 0 && len(il2.InputImports()) > 0 {
diff.AddedImportInputs, diff.RemovedImportInputs = findAddedAndRemoved(il1.InputImports(), il2.InputImports())
}
return diff
}
func findAddedAndRemoved(l1, l2 []string) (add, remove []string) {
// Computing package add/removes could probably be optimized to O(n), but
// it's not critical path for any known case, so not worth the effort right now.
p1, p2 := make(map[string]bool, len(l1)), make(map[string]bool, len(l2))
for _, pkg := range l1 {
p1[pkg] = true
}
for _, pkg := range l2 {
p2[pkg] = true
}
for pkg := range p1 {
if !p2[pkg] {
remove = append(remove, pkg)
}
}
for pkg := range p2 {
if !p1[pkg] {
add = append(add, pkg)
}
}
return add, remove
}
func DiffProjects2(lp1, lp2 gps.LockedProject) LockedProjectPartsDiff {
ld := LockedProjectPartsDiff{
SourceBefore: lp1.Ident().Source,
SourceAfter: lp2.Ident().Source,
}
ld.PackagesRemoved, ld.PackagesAdded = findAddedAndRemoved(lp1.Packages(), lp2.Packages())
switch v := lp1.Version().(type) {
case gps.PairedVersion:
ld.VersionBefore, ld.RevisionBefore = v.Unpair(), v.Revision()
case gps.Revision:
ld.RevisionBefore = v
case gps.UnpairedVersion:
// This should ideally never happen
ld.VersionBefore = v
}
switch v := lp2.Version().(type) {
case gps.PairedVersion:
ld.VersionAfter, ld.RevisionAfter = v.Unpair(), v.Revision()
case gps.Revision:
ld.RevisionAfter = v
case gps.UnpairedVersion:
// This should ideally never happen
ld.VersionAfter = v
}
vp1, ok1 := lp1.(VerifiableProject)
vp2, ok2 := lp2.(VerifiableProject)
if ok1 && ok2 {
ld.PruneOptsBefore, ld.PruneOptsAfter = vp1.PruneOpts, vp2.PruneOpts
// Only consider hashes for diffing if neither were the zero value.
if !vp1.Digest.IsEmpty() && !vp2.Digest.IsEmpty() {
if vp1.Digest.HashVersion != vp2.Digest.HashVersion {
ld.HashVersionChanged = true
}
if !bytes.Equal(vp1.Digest.Digest, vp2.Digest.Digest) {
ld.HashChanged = true
}
}
}
return ld
}
func (ld LockDiff2) Changed() bool {
if len(ld.AddedImportInputs) > 0 || len(ld.RemovedImportInputs) > 0 {
return true
}
for _, ld := range ld.ProjectDiffs {
if ld.Changed() {
return true
}
}
return false
}
func (ld LockedProjectDiff2) Changed() bool {
return ld.WasRemoved() || ld.WasAdded() || ld.RevisionChanged() || ld.VersionChanged() || ld.SourceChanged() || ld.PackagesChanged() || ld.HashChanged || ld.HashVersionChanged
}
func (ld LockedProjectDiff2) WasRemoved() bool {
return ld.ProjectRemoved
}
func (ld LockedProjectDiff2) WasAdded() bool {
return ld.ProjectAdded
}
func (ld LockedProjectPartsDiff) SourceChanged() bool {
return ld.SourceBefore != ld.SourceAfter
}
func (ld LockedProjectPartsDiff) VersionChanged() bool {
if ld.VersionBefore == nil && ld.VersionAfter == nil {
return false
} else if (ld.VersionBefore == nil || ld.VersionAfter == nil) || (ld.VersionBefore.Type() != ld.VersionAfter.Type()) {
return true
} else if !ld.VersionBefore.Matches(ld.VersionAfter) {
return true
}
return false
}
func (ld LockedProjectPartsDiff) VersionTypeChanged() bool {
if ld.VersionBefore == nil && ld.VersionAfter == nil {
return false
} else if (ld.VersionBefore == nil || ld.VersionAfter == nil) || (ld.VersionBefore.Type() != ld.VersionAfter.Type()) {
return true
}
return false
}
func (ld LockedProjectPartsDiff) RevisionChanged() bool {
return ld.RevisionBefore != ld.RevisionAfter
}
func (ld LockedProjectPartsDiff) PackagesChanged() bool {
return len(ld.PackagesAdded) > 0 || len(ld.PackagesRemoved) > 0
}
func (ld LockedProjectPartsDiff) PruneOptsChanged() bool {
return ld.PruneOptsBefore != ld.PruneOptsAfter
}
// DiffLocks compares two locks and identifies the differences between them.
// Returns nil if there are no differences.
func DiffLocks(l1, l2 gps.Lock) *LockDiff {
// Default nil locks to empty locks, so that we can still generate a diff
if l1 == nil {
l1 = &gps.SimpleLock{}
l1 = gps.SimpleLock{}
}
if l2 == nil {
l2 = &gps.SimpleLock{}
l2 = gps.SimpleLock{}
}
p1, p2 := l1.Projects(), l2.Projects()

7
internal/feedback/feedback.go
Просмотреть файл

@ -10,6 +10,7 @@ import (
"log"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/verify"
)
const (
@ -87,7 +88,7 @@ type brokenImport interface {
}
type modifiedImport struct {
source, branch, revision, version *gps.StringDiff
source, branch, revision, version *verify.StringDiff
projectPath string
}
@ -123,7 +124,7 @@ func (mi modifiedImport) String() string {
}
type removedImport struct {
source, branch, revision, version *gps.StringDiff
source, branch, revision, version *verify.StringDiff
projectPath string
}
@ -157,7 +158,7 @@ type BrokenImportFeedback struct {
// NewBrokenImportFeedback builds a feedback entry that compares an initially
// imported, unsolved lock to the same lock after it has been solved.
func NewBrokenImportFeedback(ld *gps.LockDiff) *BrokenImportFeedback {
func NewBrokenImportFeedback(ld *verify.LockDiff) *BrokenImportFeedback {
bi := &BrokenImportFeedback{}
for _, lpd := range ld.Modify {
// Ignore diffs where it's just a modified package set

3
internal/feedback/feedback_test.go
Просмотреть файл

@ -12,6 +12,7 @@ import (
"github.com/golang/dep"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/verify"
_ "github.com/golang/dep/internal/test" // DO NOT REMOVE, allows go test ./... -update to work
)
@ -150,7 +151,7 @@ func TestFeedback_BrokenImport(t *testing.T) {
P: []gps.LockedProject{gps.NewLockedProject(c.altPID, c.currentVersion, nil)},
}
log := log2.New(buf, "", 0)
feedback := NewBrokenImportFeedback(gps.DiffLocks(&ol, &l))
feedback := NewBrokenImportFeedback(verify.DiffLocks(&ol, &l))
feedback.LogFeedback(log)
got := strings.TrimSpace(buf.String())
if c.want != got {

42
lock.go
Просмотреть файл

@ -10,7 +10,6 @@ import (
"sort"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/pkgtree"
"github.com/golang/dep/gps/verify"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"
@ -77,7 +76,7 @@ func readLock(r io.Reader) (*Lock, error) {
func fromRawLock(raw rawLock) (*Lock, error) {
l := &Lock{
P: make([]gps.LockedProject, len(raw.Projects)),
P: make([]gps.LockedProject, 0, len(raw.Projects)),
}
l.SolveMeta.AnalyzerName = raw.SolveMeta.AnalyzerName
@ -86,7 +85,7 @@ func fromRawLock(raw rawLock) (*Lock, error) {
l.SolveMeta.SolverVersion = raw.SolveMeta.SolverVersion
l.SolveMeta.InputImports = raw.SolveMeta.InputImports
for i, ld := range raw.Projects {
for _, ld := range raw.Projects {
r := gps.Revision(ld.Revision)
var v gps.Version = r
@ -111,7 +110,7 @@ func fromRawLock(raw rawLock) (*Lock, error) {
LockedProject: gps.NewLockedProject(id, v, ld.Packages),
}
if ld.Digest != "" {
vp.Digest, err = pkgtree.ParseVersionedDigest(ld.Digest)
vp.Digest, err = verify.ParseVersionedDigest(ld.Digest)
if err != nil {
return nil, err
}
@ -124,24 +123,23 @@ func fromRawLock(raw rawLock) (*Lock, error) {
// Add the vendor pruning bit so that gps doesn't get confused
vp.PruneOpts = po | gps.PruneNestedVendorDirs
l.P[i] = vp
l.P = append(l.P, vp)
}
return l, nil
}
// InputsDigest returns the hash of inputs which produced this lock data.
//
// TODO(sdboyer) remove, this is now deprecated
func (l *Lock) InputsDigest() []byte {
return nil
}
// Projects returns the list of LockedProjects contained in the lock data.
func (l *Lock) Projects() []gps.LockedProject {
return l.P
}
// InputImports reports the list of input imports that were used in generating
// this Lock.
func (l *Lock) InputImports() []string {
return l.SolveMeta.InputImports
}
// HasProjectWithRoot checks if the lock contains a project with the provided
// ProjectRoot.
//
@ -162,6 +160,7 @@ func (l *Lock) toRaw() rawLock {
SolveMeta: solveMeta{
AnalyzerName: l.SolveMeta.AnalyzerName,
AnalyzerVersion: l.SolveMeta.AnalyzerVersion,
InputImports: l.SolveMeta.InputImports,
SolverName: l.SolveMeta.SolverName,
SolverVersion: l.SolveMeta.SolverVersion,
},
@ -207,22 +206,35 @@ func (l *Lock) MarshalTOML() ([]byte, error) {
}
// LockFromSolution converts a gps.Solution to dep's representation of a lock.
// It makes sure that that the provided prune options are set correctly, as the
// solver does not use VerifiableProjects for new selections it makes.
//
// Data is defensively copied wherever necessary to ensure the resulting *Lock
// shares no memory with the input solution.
func LockFromSolution(in gps.Solution) *Lock {
func LockFromSolution(in gps.Solution, prune gps.CascadingPruneOptions) *Lock {
p := in.Projects()
l := &Lock{
SolveMeta: SolveMeta{
AnalyzerName: in.AnalyzerName(),
AnalyzerVersion: in.AnalyzerVersion(),
InputImports: in.InputImports(),
SolverName: in.SolverName(),
SolverVersion: in.SolverVersion(),
},
P: make([]gps.LockedProject, len(p)),
P: make([]gps.LockedProject, 0, len(p)),
}
for _, lp := range p {
if vp, ok := lp.(verify.VerifiableProject); ok {
l.P = append(l.P, vp)
} else {
l.P = append(l.P, verify.VerifiableProject{
LockedProject: lp,
PruneOpts: prune.PruneOptionsFor(lp.Ident().ProjectRoot),
})
}
}
copy(l.P, p)
return l
}

17
lock_test.go
Просмотреть файл

@ -10,7 +10,6 @@ import (
"testing"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/pkgtree"
"github.com/golang/dep/gps/verify"
"github.com/golang/dep/internal/test"
)
@ -37,8 +36,8 @@ func TestReadLock(t *testing.T) {
[]string{"."},
),
PruneOpts: gps.PruneOptions(1),
Digest: pkgtree.VersionedDigest{
HashVersion: pkgtree.HashVersion,
Digest: verify.VersionedDigest{
HashVersion: verify.HashVersion,
Digest: []byte("foo"),
},
},
@ -67,8 +66,8 @@ func TestReadLock(t *testing.T) {
[]string{"."},
),
PruneOpts: gps.PruneOptions(15),
Digest: pkgtree.VersionedDigest{
HashVersion: pkgtree.HashVersion,
Digest: verify.VersionedDigest{
HashVersion: verify.HashVersion,
Digest: []byte("foo"),
},
},
@ -95,8 +94,8 @@ func TestWriteLock(t *testing.T) {
[]string{"."},
),
PruneOpts: gps.PruneOptions(1),
Digest: pkgtree.VersionedDigest{
HashVersion: pkgtree.HashVersion,
Digest: verify.VersionedDigest{
HashVersion: verify.HashVersion,
Digest: []byte("foo"),
},
},
@ -129,8 +128,8 @@ func TestWriteLock(t *testing.T) {
[]string{"."},
),
PruneOpts: gps.PruneOptions(15),
Digest: pkgtree.VersionedDigest{
HashVersion: pkgtree.HashVersion,
Digest: verify.VersionedDigest{
HashVersion: verify.HashVersion,
Digest: []byte("foo"),
},
},

3
project.go
Просмотреть файл

@ -105,6 +105,9 @@ type Project struct {
Manifest *Manifest
Lock *Lock // Optional
RootPackageTree pkgtree.PackageTree
// If populated, contains the results of comparing the Lock against the
// current vendor tree, per verify.VerifyDepTree().
//VendorStatus map[string]verify.VendorStatus
}
// SetRoot sets the project AbsRoot and ResolvedAbsRoot. If root is not a symlink, ResolvedAbsRoot will be set to root.

2
testdata/lock/golden1.toml поставляемый
Просмотреть файл

@ -3,7 +3,7 @@
digest = "1:666f6f"
name = "github.com/golang/dep"
packages = ["."]
pruneopts = "TUN"
pruneopts = "NUT"
revision = "d05d5aca9f895d19e9265839bffeadd74a2d2ecb"
version = "0.12.2"

167
txn_writer.go
Просмотреть файл

@ -7,13 +7,13 @@ package dep
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/pkgtree"
"github.com/golang/dep/gps/verify"
"github.com/golang/dep/internal/fs"
"github.com/pelletier/go-toml"
@ -65,7 +65,7 @@ var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes
type SafeWriter struct {
Manifest *Manifest
lock *Lock
lockDiff *gps.LockDiff
lockDiff *verify.LockDiff
writeVendor bool
writeLock bool
pruneOptions gps.CascadingPruneOptions
@ -98,7 +98,7 @@ func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBeha
return nil, errors.New("must provide newLock when oldLock is specified")
}
sw.lockDiff = gps.DiffLocks(oldLock, newLock)
sw.lockDiff = verify.DiffLocks(oldLock, newLock)
if sw.lockDiff != nil {
sw.writeLock = true
}
@ -131,7 +131,7 @@ func (sw *SafeWriter) HasManifest() bool {
}
type rawStringDiff struct {
*gps.StringDiff
*verify.StringDiff
}
// MarshalTOML serializes the diff as a string.
@ -148,7 +148,7 @@ type rawLockedProjectDiff struct {
Packages []rawStringDiff `toml:"packages,omitempty"`
}
func toRawLockedProjectDiff(diff gps.LockedProjectDiff) rawLockedProjectDiff {
func toRawLockedProjectDiff(diff verify.LockedProjectDiff) rawLockedProjectDiff {
// this is a shallow copy since we aren't modifying the raw diff
raw := rawLockedProjectDiff{Name: diff.Name}
if diff.Source != nil {
@ -174,7 +174,7 @@ type rawLockedProjectDiffs struct {
Projects []rawLockedProjectDiff `toml:"projects"`
}
func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiffs {
func toRawLockedProjectDiffs(diffs []verify.LockedProjectDiff) rawLockedProjectDiffs {
raw := rawLockedProjectDiffs{
Projects: make([]rawLockedProjectDiff, len(diffs)),
}
@ -186,10 +186,10 @@ func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiff
return raw
}
func formatLockDiff(diff gps.LockDiff) (string, error) {
func formatLockDiff(diff verify.LockDiff) (string, error) {
var buf bytes.Buffer
writeDiffs := func(diffs []gps.LockedProjectDiff) error {
writeDiffs := func(diffs []verify.LockedProjectDiff) error {
raw := toRawLockedProjectDiffs(diffs)
chunk, err := toml.Marshal(raw)
if err != nil {
@ -309,17 +309,6 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
}
}
if sw.writeLock {
l, err := sw.lock.MarshalTOML()
if err != nil {
return errors.Wrap(err, "failed to marshal lock to TOML")
}
if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil {
return errors.Wrap(err, "failed to write lock file to temp dir")
}
}
if sw.writeVendor {
var onWrite func(gps.WriteProgress)
if logger != nil {
@ -331,6 +320,26 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
if err != nil {
return errors.Wrap(err, "error while writing out vendor tree")
}
for k, lp := range sw.lock.Projects() {
vp := lp.(verify.VerifiableProject)
vp.Digest, err = verify.DigestFromDirectory(filepath.Join(td, "vendor", string(lp.Ident().ProjectRoot)))
if err != nil {
return errors.Wrapf(err, "error while hashing tree of %s in vendor", lp.Ident().ProjectRoot)
}
sw.lock.P[k] = vp
}
}
if sw.writeLock {
l, err := sw.lock.MarshalTOML()
if err != nil {
return errors.Wrap(err, "failed to marshal lock to TOML")
}
if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil {
return errors.Wrap(err, "failed to write lock file to temp dir")
}
}
// Ensure vendor/.git is preserved if present
@ -490,11 +499,11 @@ func hasDotGit(path string) bool {
type DeltaWriter struct {
lock *Lock
lockDiff *gps.LockDiff
lockDiff verify.LockDiff2
pruneOptions gps.CascadingPruneOptions
vendorDir string
changed map[gps.ProjectRoot]changeType
status map[string]pkgtree.VendorStatus
status map[string]verify.VendorStatus
}
type changeType uint8
@ -502,16 +511,18 @@ type changeType uint8
const (
noChange changeType = iota
solveChanged
pruneChanged
hashChanged
// FIXME need added/removed up here
hashMismatch
hashVersionMismatch
missingFromTree
projectAdded
projectRemoved
)
// NewDeltaWriter prepares a vendor writer that will construct a vendor
// directory by writing out only those projects that actually need to be written
// out - they have changed in some way, or they lack the necessary hash
// information to be verified.
func NewDeltaWriter(oldLock, newLock *Lock, prune gps.CascadingPruneOptions, vendorDir string) (TransactionWriter, error) {
func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatus, prune gps.CascadingPruneOptions, vendorDir string) (TransactionWriter, error) {
sw := &DeltaWriter{
lock: newLock,
pruneOptions: prune,
@ -530,40 +541,39 @@ func NewDeltaWriter(oldLock, newLock *Lock, prune gps.CascadingPruneOptions, ven
return NewSafeWriter(nil, oldLock, newLock, VendorOnChanged, prune)
}
sw.lockDiff = gps.DiffLocks(oldLock, newLock)
sw.lockDiff = verify.DiffLocks2(oldLock, newLock)
// 1. find all the ones that truly changed in solve
// 2. find the ones that only changed pruneopts
// 3. find the ones that (already) had a mismatch with what's in vendor
sums := make(map[string][]byte)
for pr, lpd := range sw.lockDiff.ProjectDiffs {
// Turn off all the hash diffing markers in the lock, unless we already
// know there's a mismatch. We don't want to rely on them for our case,
// as we're not sure they'll have been correctly populated in the new
// lock at this point.
lpd.HashVersionChanged, lpd.HashChanged = false, false
sw.lockDiff.ProjectDiffs[pr] = lpd
for _, lp := range newLock.Projects() {
pr := lp.Ident().ProjectRoot
// TODO(sdboyer) Not the best heuristic to assume that a PPS indicates
if vp, ok := lp.(verify.VerifiableProject); !ok {
sw.changed[pr] = solveChanged
sums[string(pr)] = []byte{}
} else {
sums[string(pr)] = vp.Digest.Digest
sw.changed[pr] = pruneChanged
//if _, has := sw.changed[pr]; !has && vp.PruneOpts != prune.PruneOptionsFor(pr) {
//}
if lpd.Changed() {
if lpd.WasAdded() {
sw.changed[pr] = projectAdded
} else if lpd.WasRemoved() {
sw.changed[pr] = projectRemoved
} else {
sw.changed[pr] = solveChanged
}
}
}
status, err := pkgtree.VerifyDepTree(vendorDir, sums)
if err != nil {
return nil, err
}
for spr, stat := range status {
pr := gps.ProjectRoot(spr)
switch stat {
case pkgtree.NotInLock, pkgtree.NotInTree:
// FIXME
case pkgtree.EmptyDigestInLock, pkgtree.DigestMismatchInLock:
if _, has := sw.changed[pr]; !has {
sw.changed[gps.ProjectRoot(pr)] = hashChanged
// These cases only matter if there was no change already recorded via
// the differ.
if _, has := sw.changed[pr]; !has {
switch stat {
case verify.NotInTree:
sw.changed[pr] = missingFromTree
case verify.EmptyDigestInLock, verify.DigestMismatchInLock:
sw.changed[pr] = hashMismatch
case verify.HashVersionMismatch:
sw.changed[pr] = hashVersionMismatch
}
}
}
@ -604,17 +614,57 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l
projs[lp.Ident().ProjectRoot] = lp
}
dropped := []gps.ProjectRoot{}
// TODO(sdboyer) add a txn/rollback layer, like the safewriter?
//for pr, reason := range dw.changed {
for pr, _ := range dw.changed {
for pr, reason := range dw.changed {
to := filepath.FromSlash(filepath.Join(vnewpath, string(pr)))
po := dw.pruneOptions.PruneOptionsFor(pr)
lpd := dw.lockDiff.ProjectDiffs[pr]
switch reason {
case noChange:
panic(fmt.Sprintf("wtf, no change for %s", pr))
case solveChanged:
if lpd.SourceChanged() {
logger.Printf("Writing %s: source changed (%s -> %s)", pr, lpd.SourceBefore, lpd.SourceAfter)
} else if lpd.VersionChanged() {
logger.Printf("Writing %s: version changed (%s -> %s)", pr, lpd.VersionBefore, lpd.VersionAfter)
} else if lpd.RevisionChanged() {
logger.Printf("Writing %s: revision changed (%s -> %s)", pr, lpd.RevisionBefore, lpd.RevisionAfter)
} else if lpd.PackagesChanged() {
la, lr := len(lpd.PackagesAdded), len(lpd.PackagesRemoved)
if la > 0 && lr > 0 {
logger.Printf("Writing %s: packages changed (%v added, %v removed)", pr, la, lr)
} else if la > 0 {
logger.Printf("Writing %s: packages changed (%v added)", pr, la)
} else {
logger.Printf("Writing %s: packages changed (%v removed)", pr, lr)
}
} else if lpd.PruneOptsChanged() {
// Override what's on the lockdiff with the extra info we have;
// this lets us excise PruneNestedVendorDirs and get the real
// value from the input param in place.
old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs
new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs
logger.Printf("Writing %s: prune options changed (%s -> %s)", pr, old, new)
}
case hashMismatch:
logger.Printf("Writing %s: hash mismatch between Gopkg.lock and vendor contents", pr)
case hashVersionMismatch:
logger.Printf("Writing %s: hashing algorithm mismatch", pr)
case projectAdded:
logger.Printf("Writing new project %s", pr)
case projectRemoved:
dropped = append(dropped, pr)
continue
case missingFromTree:
logger.Printf("Writing %s: missing from vendor", pr)
}
if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil {
return errors.Wrapf(err, "failed to export %s", pr)
}
digest, err := pkgtree.DigestFromDirectory(to)
digest, err := verify.DigestFromDirectory(to)
if err != nil {
return errors.Wrapf(err, "failed to hash %s", pr)
}
@ -622,6 +672,8 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l
// Update the new Lock with verification information.
for k, lp := range dw.lock.P {
if lp.Ident().ProjectRoot == pr {
vp := lp.(verify.VerifiableProject)
vp.Digest = digest
dw.lock.P[k] = verify.VerifiableProject{
LockedProject: lp,
PruneOpts: po,
@ -649,6 +701,11 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l
}
}
for _, pr := range dropped {
// Kind of a lie to print this here. ¯\_(ツ)_/¯
logger.Printf("Discarding unused project %s", pr)
}
err = os.RemoveAll(vpath)
if err != nil {
return errors.Wrap(err, "failed to remove original vendor directory")