From df2c26b7774797c869fa842d05069b49f5d36c79 Mon Sep 17 00:00:00 2001 From: sam boyer Date: Thu, 28 Jun 2018 01:32:17 -0400 Subject: [PATCH] dep: Get DeltaWriter into a working state This encompasses the first pass at the new, more abstracted diffing system, and the DeltaWriter implementation on top of it. Tests are needed, but cursory testing indicates that we successfully capture all types of diffs and regenerate only the subset of projects that actually need to be touched. --- Gopkg.lock | 43 ++++- cmd/dep/ensure.go | 123 +++++++-------- cmd/dep/ensure_test.go | 4 +- cmd/dep/init.go | 4 +- cmd/dep/integration_test.go | 28 ---- cmd/dep/main.go | 1 - cmd/dep/prune.go | 10 -- cmd/dep/root_analyzer.go | 3 +- cmd/dep/status.go | 9 +- gps/solver.go | 13 +- gps/source_cache.go | 17 ++ gps/source_cache_bolt_encode.go | 5 +- gps/source_cache_bolt_test.go | 22 +-- gps/source_cache_test.go | 24 +-- gps/verify/digest.go | 13 +- gps/verify/lockdiff.go | 245 ++++++++++++++++++++++++++++- internal/feedback/feedback.go | 7 +- internal/feedback/feedback_test.go | 3 +- lock.go | 42 +++-- lock_test.go | 17 +- project.go | 3 + testdata/lock/golden1.toml | 2 +- txn_writer.go | 167 +++++++++++++------- 23 files changed, 573 insertions(+), 232 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a65c3106..a1aa86af 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,91 +3,132 @@ [[projects]] branch = "2.x" + digest = "1:ee2887fecb4d923fa90f8dd9cf33e876bf9260fed62f2ca5a5c3f41b4eb07683" name = "github.com/Masterminds/semver" packages = ["."] + pruneopts = "NUT" revision = "24642bd0573145a5ee04f9be773641695289be46" [[projects]] + digest = "1:442020d26d1f891d5014cae4353b6ff589562c2b303504627de3660adf3fb217" name = "github.com/Masterminds/vcs" packages = ["."] + pruneopts = "NUT" revision = "3084677c2c188840777bff30054f2b553729d329" version = "v1.11.1" [[projects]] branch = "master" + digest = "1:60861e762bdbe39c4c7bf292c291329b731c9925388fd41125888f5c1c595feb" name = "github.com/armon/go-radix" packages = ["."] + pruneopts = "NUT" revision = "4239b77079c7b5d1243b7b4736304ce8ddb6f0f2" [[projects]] + digest = "1:a12d94258c5298ead75e142e8001224bf029f302fed9e96cd39c0eaf90f3954d" name = "github.com/boltdb/bolt" packages = ["."] + pruneopts = "NUT" revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" [[projects]] + digest = "1:9f35c1344b56e5868d511d231f215edd0650aa572664f856444affdd256e43e4" name = "github.com/golang/protobuf" packages = ["proto"] + pruneopts = "NUT" revision = "925541529c1fa6821df4e44ce2723319eb2be768" version = "v1.0.0" [[projects]] + digest = "1:f5169729244becc423886eae4d72547e28ac3f13f861bed8a9d749bc7238a1c3" name = "github.com/jmank88/nuts" packages = ["."] + pruneopts = "NUT" revision = "8b28145dffc87104e66d074f62ea8080edfad7c8" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:01af3a6abe28784782680e1f75ef8767cfc5d4b230dc156ff7eb8db395cbbfd2" name = "github.com/nightlyone/lockfile" packages = ["."] + pruneopts = "NUT" revision = "e83dc5e7bba095e8d32fb2124714bf41f2a30cb5" [[projects]] + digest = "1:13b8f1a2ce177961dc9231606a52f709fab896c565f3988f60a7f6b4e543a902" name = "github.com/pelletier/go-toml" packages = ["."] + pruneopts = "NUT" revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" version = "v1.1.0" [[projects]] + digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "NUT" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] branch = "master" + digest = "1:abb4b60c28323cde32c193ce6083bb600fac462d1780cf83461b4c23ed5ce904" name = "github.com/sdboyer/constext" packages = ["."] + pruneopts = "NUT" revision = "836a144573533ea4da4e6929c235fd348aed1c80" [[projects]] branch = "master" + digest = "1:6ad2104db8f34b8656382ef0a7297b9a5cc42e7bdce95d968e02b92fc97470d1" name = "golang.org/x/net" packages = ["context"] + pruneopts = "NUT" revision = "66aacef3dd8a676686c7ae3716979581e8b03c47" [[projects]] branch = "master" + digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239" name = "golang.org/x/sync" packages = ["errgroup"] + pruneopts = "NUT" revision = "f52d1811a62927559de87708c8913c1650ce4f26" [[projects]] branch = "master" + digest = "1:51912e607c5e28a89fdc7e41d3377b92086ab7f76ded236765dbf98d0a704c5d" name = "golang.org/x/sys" packages = ["unix"] + pruneopts = "NUT" revision = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366" [[projects]] branch = "v2" + digest = "1:13e704c08924325be00f96e47e7efe0bfddf0913cdfc237423c83f9b183ff590" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "NUT" revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "460ad7112866da4b9a0a626aa3e2fe80699c17bf871afb73b93f836418fb9298" + input-imports = [ + "github.com/Masterminds/semver", + "github.com/Masterminds/vcs", + "github.com/armon/go-radix", + "github.com/boltdb/bolt", + "github.com/golang/protobuf/proto", + "github.com/jmank88/nuts", + "github.com/nightlyone/lockfile", + "github.com/pelletier/go-toml", + "github.com/pkg/errors", + "github.com/sdboyer/constext", + "golang.org/x/sync/errgroup", + "gopkg.in/yaml.v2" + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/dep/ensure.go b/cmd/dep/ensure.go index 7bc066cc..0024482b 100644 --- a/cmd/dep/ensure.go +++ b/cmd/dep/ensure.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "context" "flag" "fmt" @@ -21,6 +20,7 @@ import ( "github.com/golang/dep/gps" "github.com/golang/dep/gps/paths" "github.com/golang/dep/gps/pkgtree" + "github.com/golang/dep/gps/verify" "github.com/pkg/errors" ) @@ -184,6 +184,33 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error { return cmd.runVendorOnly(ctx, args, p, sm, params) } + statchan := make(chan map[string]verify.VendorStatus) + var lps []gps.LockedProject + if p.Lock != nil { + lps = p.Lock.Projects() + } + go func(vendorDir string, p []gps.LockedProject) { + // Make sure vendor dir exists + err := os.MkdirAll(vendorDir, os.FileMode(0777)) + if err != nil { + ctx.Err.Printf("Error creating vendor directory: %q", err.Error()) + // TODO(sdboyer) handle these better + os.Exit(1) + } + + sums := make(map[string]verify.VersionedDigest) + for _, lp := range p { + sums[string(lp.Ident().ProjectRoot)] = lp.(verify.VerifiableProject).Digest + } + + status, err := verify.VerifyDepTree(vendorDir, sums) + if err != nil { + ctx.Err.Printf("Error while verifying vendor directory: %q", err.Error()) + os.Exit(1) + } + statchan <- status + }(filepath.Join(p.AbsRoot, "vendor"), lps) + params.RootPackageTree, err = p.ParseRootPackageTree() if err != nil { return err @@ -212,11 +239,11 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error { } if cmd.add { - return cmd.runAdd(ctx, args, p, sm, params) + return cmd.runAdd(ctx, args, p, sm, params, statchan) } else if cmd.update { - return cmd.runUpdate(ctx, args, p, sm, params) + return cmd.runUpdate(ctx, args, p, sm, params, statchan) } - return cmd.runDefault(ctx, args, p, sm, params) + return cmd.runDefault(ctx, args, p, sm, params, statchan) } func (cmd *ensureCommand) validateFlags() error { @@ -246,7 +273,7 @@ func (cmd *ensureCommand) vendorBehavior() dep.VendorBehavior { return dep.VendorOnChanged } -func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { +func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error { // Bare ensure doesn't take any args. if len(args) != 0 { return errors.New("dep ensure only takes spec arguments with -add or -update") @@ -256,52 +283,37 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project return err } - if lsat, err := p.LockSatisfiesInputs(sm); err != nil { - return err - } else if !lsat.Passes() { - if ctx.Verbose { - ctx.Out.Printf("%s was already in sync with imports and %s\n", dep.LockName, dep.ManifestName) - } + lock := p.Lock + if lock != nil { + lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + if !lsat.Passed() { + // TODO(sdboyer) print out what bits are unsatisfied here + solver, err := gps.Prepare(params, sm) + if err != nil { + return errors.Wrap(err, "prepare solver") + } - if cmd.noVendor { + if cmd.noVendor && cmd.dryRun { + return errors.New("Gopkg.lock was not up to date") + } + + solution, err := solver.Solve(context.TODO()) + if err != nil { + return handleAllTheFailuresOfTheWorld(err) + } + lock = dep.LockFromSolution(solution, p.Manifest.PruneOptions) + } else if cmd.noVendor { // The user said not to touch vendor/, so definitely nothing to do. return nil } - sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorOnChanged, p.Manifest.PruneOptions) - if err != nil { - return err - } - - if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) - } - - var logger *log.Logger - if ctx.Verbose { - logger = ctx.Err - } - return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") } - solver, err := gps.Prepare(params, sm) - if err != nil { - return errors.Wrap(err, "prepare solver") - } - - if cmd.noVendor && cmd.dryRun { - return errors.New("Gopkg.lock was not up to date") - } - - solution, err := solver.Solve(context.TODO()) - if err != nil { - return handleAllTheFailuresOfTheWorld(err) - } - - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions) + sw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } + if cmd.dryRun { return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) } @@ -310,7 +322,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project if ctx.Verbose { logger = ctx.Err } - return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") + return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -339,7 +351,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") } -func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { +func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error { if p.Lock == nil { return errors.Errorf("-update works by updating the versions recorded in %s, but %s does not exist", dep.LockName, dep.LockName) } @@ -348,14 +360,6 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return err } - // We'll need to discard this prepared solver as later work changes params, - // but solver preparation is cheap and worth doing up front in order to - // perform the fastpath check of hash comparison. - solver, err := gps.Prepare(params, sm) - if err != nil { - return errors.Wrap(err, "fastpath solver prepare") - } - // When -update is specified without args, allow every dependency to change // versions, regardless of the lock file. if len(args) == 0 { @@ -367,7 +371,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, } // Re-prepare a solver now that our params are complete. - solver, err = gps.Prepare(params, sm) + solver, err := gps.Prepare(params, sm) if err != nil { return errors.Wrap(err, "fastpath solver prepare") } @@ -379,7 +383,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return handleAllTheFailuresOfTheWorld(err) } - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), cmd.vendorBehavior(), p.Manifest.PruneOptions) + sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior(), p.Manifest.PruneOptions) if err != nil { return err } @@ -394,7 +398,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") } -func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { +func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error { if len(args) == 0 { return errors.New("must specify at least one project or package to -add") } @@ -411,15 +415,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm return errors.Wrap(err, "fastpath solver prepare") } - // Compare the hashes. If they're not equal, bail out and ask the user to - // run a straight `dep ensure` before updating. This is handholding the - // user a bit, but the extra effort required is minimal, and it ensures the - // user is isolating variables in the event of solve problems (was it the - // "pending" changes, or the -add that caused the problem?). - if p.Lock != nil && !bytes.Equal(p.Lock.InputsDigest(), solver.HashInputs()) { - ctx.Out.Printf("Warning: %s is out of sync with %s or the project's imports.", dep.LockName, dep.ManifestName) - } - rm, _ := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages()) // TODO(sdboyer) re-enable this once we ToReachMap() intelligently filters out normally-excluded (_*, .*), dirs from errmap @@ -678,7 +673,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm } sort.Strings(reqlist) - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution), dep.VendorOnChanged, p.Manifest.PruneOptions) + sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), dep.VendorOnChanged, p.Manifest.PruneOptions) if err != nil { return err } diff --git a/cmd/dep/ensure_test.go b/cmd/dep/ensure_test.go index 0b75d3bf..0b101c17 100644 --- a/cmd/dep/ensure_test.go +++ b/cmd/dep/ensure_test.go @@ -50,11 +50,11 @@ func TestInvalidEnsureFlagCombinations(t *testing.T) { // anything other than the error being non-nil. For now, it works well // because a panic will quickly result if the initial arg length validation // checks are incorrectly handled. - if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}); err == nil { + if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}, nil); err == nil { t.Errorf("no args to plain ensure with -vendor-only") } ec.vendorOnly = false - if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}); err == nil { + if err := ec.runDefault(nil, []string{"foo"}, nil, nil, gps.SolveParameters{}, nil); err == nil { t.Errorf("no args to plain ensure") } } diff --git a/cmd/dep/init.go b/cmd/dep/init.go index dc3f38d9..1570e2ac 100644 --- a/cmd/dep/init.go +++ b/cmd/dep/init.go @@ -157,7 +157,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { err = handleAllTheFailuresOfTheWorld(err) return errors.Wrap(err, "init failed: unable to solve the dependency graph") } - p.Lock = dep.LockFromSolution(soln) + p.Lock = dep.LockFromSolution(soln, p.Manifest.PruneOptions) rootAnalyzer.FinalizeRootManifestAndLock(p.Manifest, p.Lock, copyLock) @@ -168,7 +168,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { return errors.Wrap(err, "init failed: unable to recalculate the lock digest") } - p.Lock.SolveMeta.InputsDigest = s.HashInputs() + //p.Lock.SolveMeta.InputsDigest = s.HashInputs() // Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name. vendorbak, err := dep.BackupVendor(filepath.Join(root, "vendor"), time.Now().Format("20060102150405")) diff --git a/cmd/dep/integration_test.go b/cmd/dep/integration_test.go index 1468300f..d0008991 100644 --- a/cmd/dep/integration_test.go +++ b/cmd/dep/integration_test.go @@ -74,34 +74,6 @@ func TestDepCachedir(t *testing.T) { initPath := filepath.Join("testdata", "cachedir") - t.Run("env-cachedir", func(t *testing.T) { - t.Parallel() - testProj := integration.NewTestProject(t, initPath, wd, runMain) - defer testProj.Cleanup() - - testProj.TempDir("cachedir") - cachedir := testProj.Path("cachedir") - testProj.Setenv("DEPCACHEDIR", cachedir) - - // Running `dep ensure` will pull in the dependency into cachedir. - err = testProj.DoRun([]string{"ensure"}) - if err != nil { - // Log the error output from running `dep ensure`, could be useful. - t.Logf("`dep ensure` error output: \n%s", testProj.GetStderr()) - t.Errorf("got an unexpected error: %s", err) - } - - // Check that the cache was created in the cachedir. Our fixture has the dependency - // `github.com/sdboyer/deptest` - _, err = os.Stat(testProj.Path("cachedir", "sources", "https---github.com-sdboyer-deptest")) - if err != nil { - if os.IsNotExist(err) { - t.Error("expected cachedir to have been populated but none was found") - } else { - t.Errorf("got an unexpected error: %s", err) - } - } - }) t.Run("env-invalid-cachedir", func(t *testing.T) { t.Parallel() testProj := integration.NewTestProject(t, initPath, wd, runMain) diff --git a/cmd/dep/main.go b/cmd/dep/main.go index 57f0a196..a4a2efcb 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -91,7 +91,6 @@ func (c *Config) Run() int { &statusCommand{}, &ensureCommand{}, &pruneCommand{}, - &hashinCommand{}, &versionCommand{}, } diff --git a/cmd/dep/prune.go b/cmd/dep/prune.go index 30f3efb5..d02f7deb 100644 --- a/cmd/dep/prune.go +++ b/cmd/dep/prune.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "flag" "io/ioutil" "log" @@ -75,19 +74,10 @@ func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error { params.TraceLogger = ctx.Err } - s, err := gps.Prepare(params, sm) - if err != nil { - return errors.Wrap(err, "could not set up solver for input hashing") - } - if p.Lock == nil { return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.") } - if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) { - return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.") - } - pruneLogger := ctx.Err if !ctx.Verbose { pruneLogger = log.New(ioutil.Discard, "", 0) diff --git a/cmd/dep/root_analyzer.go b/cmd/dep/root_analyzer.go index 1cdee4db..d3d43268 100644 --- a/cmd/dep/root_analyzer.go +++ b/cmd/dep/root_analyzer.go @@ -11,6 +11,7 @@ import ( "github.com/golang/dep" "github.com/golang/dep/gps" + "github.com/golang/dep/gps/verify" fb "github.com/golang/dep/internal/feedback" "github.com/golang/dep/internal/importers" "golang.org/x/sync/errgroup" @@ -167,7 +168,7 @@ func (a *rootAnalyzer) DeriveManifestAndLock(dir string, pr gps.ProjectRoot) (gp func (a *rootAnalyzer) FinalizeRootManifestAndLock(m *dep.Manifest, l *dep.Lock, ol dep.Lock) { // Iterate through the new projects in solved lock and add them to manifest // if they are direct deps and log feedback for all the new projects. - diff := gps.DiffLocks(&ol, l) + diff := verify.DiffLocks(&ol, l) bi := fb.NewBrokenImportFeedback(diff) bi.LogFeedback(a.ctx.Err) for _, y := range l.Projects() { diff --git a/cmd/dep/status.go b/cmd/dep/status.go index ed1311dd..4e9e0f9c 100644 --- a/cmd/dep/status.go +++ b/cmd/dep/status.go @@ -24,6 +24,7 @@ import ( "github.com/golang/dep" "github.com/golang/dep/gps" "github.com/golang/dep/gps/paths" + "github.com/golang/dep/gps/verify" "github.com/pkg/errors" ) @@ -912,11 +913,6 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje return false, 0, err } - s, err := gps.Prepare(params, sm) - if err != nil { - return false, 0, errors.Wrapf(err, "could not set up solver for input hashing") - } - // Errors while collecting constraints should not fail the whole status run. // It should count the error and tell the user about incomplete results. cm, ccerrs := collectConstraints(ctx, p, sm) @@ -932,7 +928,8 @@ func (cmd *statusCommand) runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Proje return slp[i].Ident().Less(slp[j].Ident()) }) - if bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) { + lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + if lsat.Passed() { // If these are equal, we're guaranteed that the lock is a transitively // complete picture of all deps. That eliminates the need for at least // some checks. diff --git a/gps/solver.go b/gps/solver.go index dc5f4cd7..c9b541d9 100644 --- a/gps/solver.go +++ b/gps/solver.go @@ -450,11 +450,16 @@ func (s *solver) Solve(ctx context.Context) (Solution, error) { soln.i = s.rd.externalImportList(s.stdLibFn) // Convert ProjectAtoms into LockedProjects - soln.p = make([]LockedProject, len(all)) - k := 0 + soln.p = make([]LockedProject, 0, len(all)) for pa, pl := range all { - soln.p[k] = pa2lp(pa, pl) - k++ + lp := pa2lp(pa, pl) + // Pass back the original inputlp directly if it Eqs what was + // selected. + if inputlp, has := s.rd.rlm[lp.Ident().ProjectRoot]; has && lp.Eq(inputlp) { + lp = inputlp + } + + soln.p = append(soln.p, lp) } } diff --git a/gps/source_cache.go b/gps/source_cache.go index 1123f317..fe3c9b72 100644 --- a/gps/source_cache.go +++ b/gps/source_cache.go @@ -271,3 +271,20 @@ func (c *singleSourceCacheMemory) toUnpaired(v Version) (UnpairedVersion, bool) panic(fmt.Sprintf("unknown version type %T", v)) } } + +// TODO(sdboyer) remove once source caching can be moved into separate package +func locksAreEq(l1, l2 Lock) bool { + p1, p2 := l1.Projects(), l2.Projects() + if len(p1) != len(p2) { + return false + } + + p1, p2 = sortLockedProjects(p1), sortLockedProjects(p2) + + for k, lp := range p1 { + if !lp.Eq(p2[k]) { + return false + } + } + return true +} diff --git a/gps/source_cache_bolt_encode.go b/gps/source_cache_bolt_encode.go index d91d9018..5e2c2553 100644 --- a/gps/source_cache_bolt_encode.go +++ b/gps/source_cache_bolt_encode.go @@ -342,8 +342,9 @@ func cachePutLock(b *bolt.Bucket, l Lock) error { // cacheGetLock returns a new *safeLock with the fields retrieved from the bolt.Bucket. func cacheGetLock(b *bolt.Bucket) (*safeLock, error) { - l := &safeLock{ - i: strings.Split(string(b.Get(cacheKeyInputImports)), "#"), + l := &safeLock{} + if ii := b.Get(cacheKeyInputImports); len(ii) > 0 { + l.i = strings.Split(string(ii), "#") } if locked := b.Bucket(cacheKeyLock); locked != nil { diff --git a/gps/source_cache_bolt_test.go b/gps/source_cache_bolt_test.go index a661ab82..d0bf0fce 100644 --- a/gps/source_cache_bolt_test.go +++ b/gps/source_cache_bolt_test.go @@ -53,7 +53,6 @@ func TestBoltCacheTimeout(t *testing.T) { } lock := &safeLock{ - //h: []byte("test_hash"), p: []LockedProject{ NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}), NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0"), nil), @@ -120,8 +119,9 @@ func TestBoltCacheTimeout(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, manifest, gotM) - if dl := DiffLocks(lock, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(lock, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL) } got, ok := c.getPackageTree(rev, root) @@ -162,8 +162,9 @@ func TestBoltCacheTimeout(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, manifest, gotM) - if dl := DiffLocks(lock, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(lock, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL) } gotPtree, ok := c.getPackageTree(rev, root) @@ -195,8 +196,9 @@ func TestBoltCacheTimeout(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, manifest, gotM) - if dl := DiffLocks(lock, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(lock, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL) } got, ok := c.getPackageTree(rev, root) @@ -233,7 +235,6 @@ func TestBoltCacheTimeout(t *testing.T) { } newLock := &safeLock{ - //h: []byte("new_test_hash"), p: []LockedProject{ NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v1"), []string{"gps"}), }, @@ -283,8 +284,9 @@ func TestBoltCacheTimeout(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, newManifest, gotM) - if dl := DiffLocks(newLock, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(lock, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", lock, gotL) } got, ok := c.getPackageTree(rev, root) diff --git a/gps/source_cache_test.go b/gps/source_cache_test.go index 14d6d0d4..5e390633 100644 --- a/gps/source_cache_test.go +++ b/gps/source_cache_test.go @@ -116,13 +116,12 @@ func (test singleSourceCacheTest) run(t *testing.T) { ig: pkgtree.NewIgnoredRuleset([]string{"a", "b"}), } var l Lock = &safeLock{ - //h: []byte("test_hash"), p: []LockedProject{ - NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0"), []string{"gps"}), - NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0"), nil), - NewLockedProject(mkPI("github.com/sdboyer/gps3"), NewVersion("v0.10.0"), []string{"gps", "flugle"}), - NewLockedProject(mkPI("foo"), NewVersion("nada"), []string{"foo"}), - NewLockedProject(mkPI("github.com/sdboyer/gps4"), NewVersion("v0.10.0"), []string{"flugle", "gps"}), + NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0").Pair("anything"), []string{"gps"}), + NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.10.0").Pair("whatever"), nil), + NewLockedProject(mkPI("github.com/sdboyer/gps3"), NewVersion("v0.10.0").Pair("again"), []string{"gps", "flugle"}), + NewLockedProject(mkPI("foo"), NewVersion("nada").Pair("itsaliving"), []string{"foo"}), + NewLockedProject(mkPI("github.com/sdboyer/gps4"), NewVersion("v0.10.0").Pair("meow"), []string{"flugle", "gps"}), }, } c.setManifestAndLock(rev, testAnalyzerInfo, m, l) @@ -140,8 +139,9 @@ func (test singleSourceCacheTest) run(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, m, gotM) - if dl := DiffLocks(l, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(l, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", l, gotL) } m = &simpleRootManifest{ @@ -163,10 +163,9 @@ func (test singleSourceCacheTest) run(t *testing.T) { ig: pkgtree.NewIgnoredRuleset([]string{"c", "d"}), } l = &safeLock{ - //h: []byte("different_test_hash"), p: []LockedProject{ NewLockedProject(mkPI("github.com/sdboyer/gps"), NewVersion("v0.10.0").Pair("278a227dfc3d595a33a77ff3f841fd8ca1bc8cd0"), []string{"gps"}), - NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.11.0"), []string{"gps"}), + NewLockedProject(mkPI("github.com/sdboyer/gps2"), NewVersion("v0.11.0").Pair("anything"), []string{"gps"}), NewLockedProject(mkPI("github.com/sdboyer/gps3"), Revision("278a227dfc3d595a33a77ff3f841fd8ca1bc8cd0"), []string{"gps"}), }, } @@ -185,8 +184,9 @@ func (test singleSourceCacheTest) run(t *testing.T) { t.Error("no manifest and lock found for revision") } compareManifests(t, m, gotM) - if dl := DiffLocks(l, gotL); dl != nil { - t.Errorf("lock differences:\n\t %#v", dl) + // TODO(sdboyer) use DiffLocks after refactoring to avoid import cycles + if !locksAreEq(l, gotL) { + t.Errorf("locks are different:\n\t(GOT): %s\n\t(WNT): %s", l, gotL) } }) diff --git a/gps/verify/digest.go b/gps/verify/digest.go index 3ce0edfa..b6363521 100644 --- a/gps/verify/digest.go +++ b/gps/verify/digest.go @@ -347,6 +347,11 @@ func (vd VersionedDigest) String() string { return fmt.Sprintf("%s:%s", strconv.Itoa(vd.HashVersion), hex.EncodeToString(vd.Digest)) } +// IsEmpty indicates if the VersionedDigest is the zero value. +func (vd VersionedDigest) IsEmpty() bool { + return vd.HashVersion == 0 && len(vd.Digest) == 0 +} + // ParseVersionedDigest decodes the string representation of versioned digest // information - a colon-separated string with a version number in the first // part and the hex-encdoed hash digest in the second - as a VersionedDigest. @@ -378,7 +383,7 @@ func ParseVersionedDigest(input string) (VersionedDigest, error) { // platform where the file system path separator is a character other than // solidus, one particular dependency would be represented as // "github.com/alice/alice1". -func VerifyDepTree(osDirname string, wantDigests map[string][]byte) (map[string]VendorStatus, error) { +func VerifyDepTree(osDirname string, wantDigests map[string]VersionedDigest) (map[string]VendorStatus, error) { osDirname = filepath.Clean(osDirname) // Ensure top level pathname is a directory @@ -455,12 +460,14 @@ func VerifyDepTree(osDirname string, wantDigests map[string][]byte) (map[string] if expectedSum, ok := wantDigests[slashPathname]; ok { ls := EmptyDigestInLock - if len(expectedSum) > 0 { + if expectedSum.HashVersion != HashVersion { + ls = HashVersionMismatch + } else if len(expectedSum.Digest) > 0 { projectSum, err := DigestFromDirectory(osPathname) if err != nil { return nil, errors.Wrap(err, "cannot compute dependency hash") } - if bytes.Equal(projectSum.Digest, expectedSum) { + if bytes.Equal(projectSum.Digest, expectedSum.Digest) { ls = NoMismatch } else { ls = DigestMismatchInLock diff --git a/gps/verify/lockdiff.go b/gps/verify/lockdiff.go index b0f230b8..7d0b90b5 100644 --- a/gps/verify/lockdiff.go +++ b/gps/verify/lockdiff.go @@ -5,6 +5,7 @@ package verify import ( + "bytes" "fmt" "sort" "strings" @@ -49,8 +50,10 @@ func sortLockedProjects(lps []gps.LockedProject) []gps.LockedProject { }) { return lps } + cp := make([]gps.LockedProject, len(lps)) copy(cp, lps) + sort.Slice(cp, func(i, j int) bool { return cp[i].Ident().Less(cp[j].Ident()) }) @@ -65,6 +68,12 @@ type LockDiff struct { Modify []LockedProjectDiff } +type LockDiff2 struct { + AddedImportInputs []string + RemovedImportInputs []string + ProjectDiffs map[gps.ProjectRoot]LockedProjectDiff2 +} + // LockedProjectDiff contains the before and after snapshot of a project reference. // Fields are only populated when there is a difference, otherwise they are empty. type LockedProjectDiff struct { @@ -76,15 +85,247 @@ type LockedProjectDiff struct { Packages []StringDiff } +type LockedProjectDiff2 struct { + Name gps.ProjectRoot + ProjectRemoved, ProjectAdded bool + LockedProjectPartsDiff +} + +type LockedProjectPartsDiff struct { + PackagesAdded, PackagesRemoved []string + VersionBefore, VersionAfter gps.UnpairedVersion + RevisionBefore, RevisionAfter gps.Revision + SourceBefore, SourceAfter string + PruneOptsBefore, PruneOptsAfter gps.PruneOptions + HashChanged, HashVersionChanged bool +} + +// DiffLocks compares two locks and identifies the differences between them. +// Returns nil if there are no differences. +func DiffLocks2(l1, l2 gps.Lock) LockDiff2 { + // Default nil locks to empty locks, so that we can still generate a diff + if l1 == nil { + if l2 == nil { + return LockDiff2{} + } + l1 = gps.SimpleLock{} + } + if l2 == nil { + l2 = gps.SimpleLock{} + } + + p1, p2 := l1.Projects(), l2.Projects() + + p1 = sortLockedProjects(p1) + p2 = sortLockedProjects(p2) + + diff := LockDiff2{ + ProjectDiffs: make(map[gps.ProjectRoot]LockedProjectDiff2), + } + + var i2next int + for i1 := 0; i1 < len(p1); i1++ { + lp1 := p1[i1] + pr1 := lp1.Ident().ProjectRoot + + lpd := LockedProjectDiff2{ + Name: pr1, + } + + for i2 := i2next; i2 < len(p2); i2++ { + lp2 := p2[i2] + pr2 := lp2.Ident().ProjectRoot + + switch strings.Compare(string(pr1), string(pr2)) { + case 0: // Found a matching project + lpd.LockedProjectPartsDiff = DiffProjects2(lp1, lp2) + i2next = i2 + 1 // Don't visit this project again + case +1: // Found a new project + diff.ProjectDiffs[pr2] = LockedProjectDiff2{ + Name: pr2, + ProjectAdded: true, + } + i2next = i2 + 1 // Don't visit this project again + continue // Keep looking for a matching project + case -1: // Project has been removed, handled below + lpd.ProjectRemoved = true + } + + break // Done evaluating this project, move onto the next + } + + diff.ProjectDiffs[pr1] = lpd + } + + // Anything that still hasn't been evaluated are adds + for i2 := i2next; i2 < len(p2); i2++ { + lp2 := p2[i2] + pr2 := lp2.Ident().ProjectRoot + diff.ProjectDiffs[pr2] = LockedProjectDiff2{ + Name: pr2, + ProjectAdded: true, + } + } + + // Only do the import inputs if both of the locks fulfill the interface, AND + // both have non-empty inputs. + il1, ok1 := l1.(gps.LockWithImports) + il2, ok2 := l2.(gps.LockWithImports) + + if ok1 && ok2 && len(il1.InputImports()) > 0 && len(il2.InputImports()) > 0 { + diff.AddedImportInputs, diff.RemovedImportInputs = findAddedAndRemoved(il1.InputImports(), il2.InputImports()) + } + + return diff +} + +func findAddedAndRemoved(l1, l2 []string) (add, remove []string) { + // Computing package add/removes could probably be optimized to O(n), but + // it's not critical path for any known case, so not worth the effort right now. + p1, p2 := make(map[string]bool, len(l1)), make(map[string]bool, len(l2)) + + for _, pkg := range l1 { + p1[pkg] = true + } + for _, pkg := range l2 { + p2[pkg] = true + } + + for pkg := range p1 { + if !p2[pkg] { + remove = append(remove, pkg) + } + } + for pkg := range p2 { + if !p1[pkg] { + add = append(add, pkg) + } + } + + return add, remove +} + +func DiffProjects2(lp1, lp2 gps.LockedProject) LockedProjectPartsDiff { + ld := LockedProjectPartsDiff{ + SourceBefore: lp1.Ident().Source, + SourceAfter: lp2.Ident().Source, + } + + ld.PackagesRemoved, ld.PackagesAdded = findAddedAndRemoved(lp1.Packages(), lp2.Packages()) + + switch v := lp1.Version().(type) { + case gps.PairedVersion: + ld.VersionBefore, ld.RevisionBefore = v.Unpair(), v.Revision() + case gps.Revision: + ld.RevisionBefore = v + case gps.UnpairedVersion: + // This should ideally never happen + ld.VersionBefore = v + } + + switch v := lp2.Version().(type) { + case gps.PairedVersion: + ld.VersionAfter, ld.RevisionAfter = v.Unpair(), v.Revision() + case gps.Revision: + ld.RevisionAfter = v + case gps.UnpairedVersion: + // This should ideally never happen + ld.VersionAfter = v + } + + vp1, ok1 := lp1.(VerifiableProject) + vp2, ok2 := lp2.(VerifiableProject) + + if ok1 && ok2 { + ld.PruneOptsBefore, ld.PruneOptsAfter = vp1.PruneOpts, vp2.PruneOpts + + // Only consider hashes for diffing if neither were the zero value. + if !vp1.Digest.IsEmpty() && !vp2.Digest.IsEmpty() { + if vp1.Digest.HashVersion != vp2.Digest.HashVersion { + ld.HashVersionChanged = true + } + if !bytes.Equal(vp1.Digest.Digest, vp2.Digest.Digest) { + ld.HashChanged = true + } + } + } + + return ld +} + +func (ld LockDiff2) Changed() bool { + if len(ld.AddedImportInputs) > 0 || len(ld.RemovedImportInputs) > 0 { + return true + } + + for _, ld := range ld.ProjectDiffs { + if ld.Changed() { + return true + } + } + + return false +} + +func (ld LockedProjectDiff2) Changed() bool { + return ld.WasRemoved() || ld.WasAdded() || ld.RevisionChanged() || ld.VersionChanged() || ld.SourceChanged() || ld.PackagesChanged() || ld.HashChanged || ld.HashVersionChanged +} + +func (ld LockedProjectDiff2) WasRemoved() bool { + return ld.ProjectRemoved +} + +func (ld LockedProjectDiff2) WasAdded() bool { + return ld.ProjectAdded +} + +func (ld LockedProjectPartsDiff) SourceChanged() bool { + return ld.SourceBefore != ld.SourceAfter +} + +func (ld LockedProjectPartsDiff) VersionChanged() bool { + if ld.VersionBefore == nil && ld.VersionAfter == nil { + return false + } else if (ld.VersionBefore == nil || ld.VersionAfter == nil) || (ld.VersionBefore.Type() != ld.VersionAfter.Type()) { + return true + } else if !ld.VersionBefore.Matches(ld.VersionAfter) { + return true + } + + return false +} + +func (ld LockedProjectPartsDiff) VersionTypeChanged() bool { + if ld.VersionBefore == nil && ld.VersionAfter == nil { + return false + } else if (ld.VersionBefore == nil || ld.VersionAfter == nil) || (ld.VersionBefore.Type() != ld.VersionAfter.Type()) { + return true + } + + return false +} + +func (ld LockedProjectPartsDiff) RevisionChanged() bool { + return ld.RevisionBefore != ld.RevisionAfter +} + +func (ld LockedProjectPartsDiff) PackagesChanged() bool { + return len(ld.PackagesAdded) > 0 || len(ld.PackagesRemoved) > 0 +} + +func (ld LockedProjectPartsDiff) PruneOptsChanged() bool { + return ld.PruneOptsBefore != ld.PruneOptsAfter +} + // DiffLocks compares two locks and identifies the differences between them. // Returns nil if there are no differences. func DiffLocks(l1, l2 gps.Lock) *LockDiff { // Default nil locks to empty locks, so that we can still generate a diff if l1 == nil { - l1 = &gps.SimpleLock{} + l1 = gps.SimpleLock{} } if l2 == nil { - l2 = &gps.SimpleLock{} + l2 = gps.SimpleLock{} } p1, p2 := l1.Projects(), l2.Projects() diff --git a/internal/feedback/feedback.go b/internal/feedback/feedback.go index 20c4dc4d..2f20cbcb 100644 --- a/internal/feedback/feedback.go +++ b/internal/feedback/feedback.go @@ -10,6 +10,7 @@ import ( "log" "github.com/golang/dep/gps" + "github.com/golang/dep/gps/verify" ) const ( @@ -87,7 +88,7 @@ type brokenImport interface { } type modifiedImport struct { - source, branch, revision, version *gps.StringDiff + source, branch, revision, version *verify.StringDiff projectPath string } @@ -123,7 +124,7 @@ func (mi modifiedImport) String() string { } type removedImport struct { - source, branch, revision, version *gps.StringDiff + source, branch, revision, version *verify.StringDiff projectPath string } @@ -157,7 +158,7 @@ type BrokenImportFeedback struct { // NewBrokenImportFeedback builds a feedback entry that compares an initially // imported, unsolved lock to the same lock after it has been solved. -func NewBrokenImportFeedback(ld *gps.LockDiff) *BrokenImportFeedback { +func NewBrokenImportFeedback(ld *verify.LockDiff) *BrokenImportFeedback { bi := &BrokenImportFeedback{} for _, lpd := range ld.Modify { // Ignore diffs where it's just a modified package set diff --git a/internal/feedback/feedback_test.go b/internal/feedback/feedback_test.go index cd3b4f1c..9baf0fb1 100644 --- a/internal/feedback/feedback_test.go +++ b/internal/feedback/feedback_test.go @@ -12,6 +12,7 @@ import ( "github.com/golang/dep" "github.com/golang/dep/gps" + "github.com/golang/dep/gps/verify" _ "github.com/golang/dep/internal/test" // DO NOT REMOVE, allows go test ./... -update to work ) @@ -150,7 +151,7 @@ func TestFeedback_BrokenImport(t *testing.T) { P: []gps.LockedProject{gps.NewLockedProject(c.altPID, c.currentVersion, nil)}, } log := log2.New(buf, "", 0) - feedback := NewBrokenImportFeedback(gps.DiffLocks(&ol, &l)) + feedback := NewBrokenImportFeedback(verify.DiffLocks(&ol, &l)) feedback.LogFeedback(log) got := strings.TrimSpace(buf.String()) if c.want != got { diff --git a/lock.go b/lock.go index ce6d5d4f..fce0643b 100644 --- a/lock.go +++ b/lock.go @@ -10,7 +10,6 @@ import ( "sort" "github.com/golang/dep/gps" - "github.com/golang/dep/gps/pkgtree" "github.com/golang/dep/gps/verify" "github.com/pelletier/go-toml" "github.com/pkg/errors" @@ -77,7 +76,7 @@ func readLock(r io.Reader) (*Lock, error) { func fromRawLock(raw rawLock) (*Lock, error) { l := &Lock{ - P: make([]gps.LockedProject, len(raw.Projects)), + P: make([]gps.LockedProject, 0, len(raw.Projects)), } l.SolveMeta.AnalyzerName = raw.SolveMeta.AnalyzerName @@ -86,7 +85,7 @@ func fromRawLock(raw rawLock) (*Lock, error) { l.SolveMeta.SolverVersion = raw.SolveMeta.SolverVersion l.SolveMeta.InputImports = raw.SolveMeta.InputImports - for i, ld := range raw.Projects { + for _, ld := range raw.Projects { r := gps.Revision(ld.Revision) var v gps.Version = r @@ -111,7 +110,7 @@ func fromRawLock(raw rawLock) (*Lock, error) { LockedProject: gps.NewLockedProject(id, v, ld.Packages), } if ld.Digest != "" { - vp.Digest, err = pkgtree.ParseVersionedDigest(ld.Digest) + vp.Digest, err = verify.ParseVersionedDigest(ld.Digest) if err != nil { return nil, err } @@ -124,24 +123,23 @@ func fromRawLock(raw rawLock) (*Lock, error) { // Add the vendor pruning bit so that gps doesn't get confused vp.PruneOpts = po | gps.PruneNestedVendorDirs - l.P[i] = vp + l.P = append(l.P, vp) } return l, nil } -// InputsDigest returns the hash of inputs which produced this lock data. -// -// TODO(sdboyer) remove, this is now deprecated -func (l *Lock) InputsDigest() []byte { - return nil -} - // Projects returns the list of LockedProjects contained in the lock data. func (l *Lock) Projects() []gps.LockedProject { return l.P } +// InputImports reports the list of input imports that were used in generating +// this Lock. +func (l *Lock) InputImports() []string { + return l.SolveMeta.InputImports +} + // HasProjectWithRoot checks if the lock contains a project with the provided // ProjectRoot. // @@ -162,6 +160,7 @@ func (l *Lock) toRaw() rawLock { SolveMeta: solveMeta{ AnalyzerName: l.SolveMeta.AnalyzerName, AnalyzerVersion: l.SolveMeta.AnalyzerVersion, + InputImports: l.SolveMeta.InputImports, SolverName: l.SolveMeta.SolverName, SolverVersion: l.SolveMeta.SolverVersion, }, @@ -207,22 +206,35 @@ func (l *Lock) MarshalTOML() ([]byte, error) { } // LockFromSolution converts a gps.Solution to dep's representation of a lock. +// It makes sure that that the provided prune options are set correctly, as the +// solver does not use VerifiableProjects for new selections it makes. // // Data is defensively copied wherever necessary to ensure the resulting *Lock // shares no memory with the input solution. -func LockFromSolution(in gps.Solution) *Lock { +func LockFromSolution(in gps.Solution, prune gps.CascadingPruneOptions) *Lock { p := in.Projects() l := &Lock{ SolveMeta: SolveMeta{ AnalyzerName: in.AnalyzerName(), AnalyzerVersion: in.AnalyzerVersion(), + InputImports: in.InputImports(), SolverName: in.SolverName(), SolverVersion: in.SolverVersion(), }, - P: make([]gps.LockedProject, len(p)), + P: make([]gps.LockedProject, 0, len(p)), + } + + for _, lp := range p { + if vp, ok := lp.(verify.VerifiableProject); ok { + l.P = append(l.P, vp) + } else { + l.P = append(l.P, verify.VerifiableProject{ + LockedProject: lp, + PruneOpts: prune.PruneOptionsFor(lp.Ident().ProjectRoot), + }) + } } - copy(l.P, p) return l } diff --git a/lock_test.go b/lock_test.go index c2b8a3d9..e6acc742 100644 --- a/lock_test.go +++ b/lock_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/golang/dep/gps" - "github.com/golang/dep/gps/pkgtree" "github.com/golang/dep/gps/verify" "github.com/golang/dep/internal/test" ) @@ -37,8 +36,8 @@ func TestReadLock(t *testing.T) { []string{"."}, ), PruneOpts: gps.PruneOptions(1), - Digest: pkgtree.VersionedDigest{ - HashVersion: pkgtree.HashVersion, + Digest: verify.VersionedDigest{ + HashVersion: verify.HashVersion, Digest: []byte("foo"), }, }, @@ -67,8 +66,8 @@ func TestReadLock(t *testing.T) { []string{"."}, ), PruneOpts: gps.PruneOptions(15), - Digest: pkgtree.VersionedDigest{ - HashVersion: pkgtree.HashVersion, + Digest: verify.VersionedDigest{ + HashVersion: verify.HashVersion, Digest: []byte("foo"), }, }, @@ -95,8 +94,8 @@ func TestWriteLock(t *testing.T) { []string{"."}, ), PruneOpts: gps.PruneOptions(1), - Digest: pkgtree.VersionedDigest{ - HashVersion: pkgtree.HashVersion, + Digest: verify.VersionedDigest{ + HashVersion: verify.HashVersion, Digest: []byte("foo"), }, }, @@ -129,8 +128,8 @@ func TestWriteLock(t *testing.T) { []string{"."}, ), PruneOpts: gps.PruneOptions(15), - Digest: pkgtree.VersionedDigest{ - HashVersion: pkgtree.HashVersion, + Digest: verify.VersionedDigest{ + HashVersion: verify.HashVersion, Digest: []byte("foo"), }, }, diff --git a/project.go b/project.go index d2677e88..0247ae9b 100644 --- a/project.go +++ b/project.go @@ -105,6 +105,9 @@ type Project struct { Manifest *Manifest Lock *Lock // Optional RootPackageTree pkgtree.PackageTree + // If populated, contains the results of comparing the Lock against the + // current vendor tree, per verify.VerifyDepTree(). + //VendorStatus map[string]verify.VendorStatus } // SetRoot sets the project AbsRoot and ResolvedAbsRoot. If root is not a symlink, ResolvedAbsRoot will be set to root. diff --git a/testdata/lock/golden1.toml b/testdata/lock/golden1.toml index 292fdb2d..4ffbce35 100644 --- a/testdata/lock/golden1.toml +++ b/testdata/lock/golden1.toml @@ -3,7 +3,7 @@ digest = "1:666f6f" name = "github.com/golang/dep" packages = ["."] - pruneopts = "TUN" + pruneopts = "NUT" revision = "d05d5aca9f895d19e9265839bffeadd74a2d2ecb" version = "0.12.2" diff --git a/txn_writer.go b/txn_writer.go index f78e9c76..791497f5 100644 --- a/txn_writer.go +++ b/txn_writer.go @@ -7,13 +7,13 @@ package dep import ( "bytes" "context" + "fmt" "io/ioutil" "log" "os" "path/filepath" "github.com/golang/dep/gps" - "github.com/golang/dep/gps/pkgtree" "github.com/golang/dep/gps/verify" "github.com/golang/dep/internal/fs" "github.com/pelletier/go-toml" @@ -65,7 +65,7 @@ var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes type SafeWriter struct { Manifest *Manifest lock *Lock - lockDiff *gps.LockDiff + lockDiff *verify.LockDiff writeVendor bool writeLock bool pruneOptions gps.CascadingPruneOptions @@ -98,7 +98,7 @@ func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBeha return nil, errors.New("must provide newLock when oldLock is specified") } - sw.lockDiff = gps.DiffLocks(oldLock, newLock) + sw.lockDiff = verify.DiffLocks(oldLock, newLock) if sw.lockDiff != nil { sw.writeLock = true } @@ -131,7 +131,7 @@ func (sw *SafeWriter) HasManifest() bool { } type rawStringDiff struct { - *gps.StringDiff + *verify.StringDiff } // MarshalTOML serializes the diff as a string. @@ -148,7 +148,7 @@ type rawLockedProjectDiff struct { Packages []rawStringDiff `toml:"packages,omitempty"` } -func toRawLockedProjectDiff(diff gps.LockedProjectDiff) rawLockedProjectDiff { +func toRawLockedProjectDiff(diff verify.LockedProjectDiff) rawLockedProjectDiff { // this is a shallow copy since we aren't modifying the raw diff raw := rawLockedProjectDiff{Name: diff.Name} if diff.Source != nil { @@ -174,7 +174,7 @@ type rawLockedProjectDiffs struct { Projects []rawLockedProjectDiff `toml:"projects"` } -func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiffs { +func toRawLockedProjectDiffs(diffs []verify.LockedProjectDiff) rawLockedProjectDiffs { raw := rawLockedProjectDiffs{ Projects: make([]rawLockedProjectDiff, len(diffs)), } @@ -186,10 +186,10 @@ func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiff return raw } -func formatLockDiff(diff gps.LockDiff) (string, error) { +func formatLockDiff(diff verify.LockDiff) (string, error) { var buf bytes.Buffer - writeDiffs := func(diffs []gps.LockedProjectDiff) error { + writeDiffs := func(diffs []verify.LockedProjectDiff) error { raw := toRawLockedProjectDiffs(diffs) chunk, err := toml.Marshal(raw) if err != nil { @@ -309,17 +309,6 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo } } - if sw.writeLock { - l, err := sw.lock.MarshalTOML() - if err != nil { - return errors.Wrap(err, "failed to marshal lock to TOML") - } - - if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil { - return errors.Wrap(err, "failed to write lock file to temp dir") - } - } - if sw.writeVendor { var onWrite func(gps.WriteProgress) if logger != nil { @@ -331,6 +320,26 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo if err != nil { return errors.Wrap(err, "error while writing out vendor tree") } + + for k, lp := range sw.lock.Projects() { + vp := lp.(verify.VerifiableProject) + vp.Digest, err = verify.DigestFromDirectory(filepath.Join(td, "vendor", string(lp.Ident().ProjectRoot))) + if err != nil { + return errors.Wrapf(err, "error while hashing tree of %s in vendor", lp.Ident().ProjectRoot) + } + sw.lock.P[k] = vp + } + } + + if sw.writeLock { + l, err := sw.lock.MarshalTOML() + if err != nil { + return errors.Wrap(err, "failed to marshal lock to TOML") + } + + if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil { + return errors.Wrap(err, "failed to write lock file to temp dir") + } } // Ensure vendor/.git is preserved if present @@ -490,11 +499,11 @@ func hasDotGit(path string) bool { type DeltaWriter struct { lock *Lock - lockDiff *gps.LockDiff + lockDiff verify.LockDiff2 pruneOptions gps.CascadingPruneOptions vendorDir string changed map[gps.ProjectRoot]changeType - status map[string]pkgtree.VendorStatus + status map[string]verify.VendorStatus } type changeType uint8 @@ -502,16 +511,18 @@ type changeType uint8 const ( noChange changeType = iota solveChanged - pruneChanged - hashChanged - // FIXME need added/removed up here + hashMismatch + hashVersionMismatch + missingFromTree + projectAdded + projectRemoved ) // NewDeltaWriter prepares a vendor writer that will construct a vendor // directory by writing out only those projects that actually need to be written // out - they have changed in some way, or they lack the necessary hash // information to be verified. -func NewDeltaWriter(oldLock, newLock *Lock, prune gps.CascadingPruneOptions, vendorDir string) (TransactionWriter, error) { +func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatus, prune gps.CascadingPruneOptions, vendorDir string) (TransactionWriter, error) { sw := &DeltaWriter{ lock: newLock, pruneOptions: prune, @@ -530,40 +541,39 @@ func NewDeltaWriter(oldLock, newLock *Lock, prune gps.CascadingPruneOptions, ven return NewSafeWriter(nil, oldLock, newLock, VendorOnChanged, prune) } - sw.lockDiff = gps.DiffLocks(oldLock, newLock) + sw.lockDiff = verify.DiffLocks2(oldLock, newLock) - // 1. find all the ones that truly changed in solve - // 2. find the ones that only changed pruneopts - // 3. find the ones that (already) had a mismatch with what's in vendor - sums := make(map[string][]byte) + for pr, lpd := range sw.lockDiff.ProjectDiffs { + // Turn off all the hash diffing markers in the lock, unless we already + // know there's a mismatch. We don't want to rely on them for our case, + // as we're not sure they'll have been correctly populated in the new + // lock at this point. + lpd.HashVersionChanged, lpd.HashChanged = false, false + sw.lockDiff.ProjectDiffs[pr] = lpd - for _, lp := range newLock.Projects() { - pr := lp.Ident().ProjectRoot - // TODO(sdboyer) Not the best heuristic to assume that a PPS indicates - if vp, ok := lp.(verify.VerifiableProject); !ok { - sw.changed[pr] = solveChanged - sums[string(pr)] = []byte{} - } else { - sums[string(pr)] = vp.Digest.Digest - sw.changed[pr] = pruneChanged - //if _, has := sw.changed[pr]; !has && vp.PruneOpts != prune.PruneOptionsFor(pr) { - //} + if lpd.Changed() { + if lpd.WasAdded() { + sw.changed[pr] = projectAdded + } else if lpd.WasRemoved() { + sw.changed[pr] = projectRemoved + } else { + sw.changed[pr] = solveChanged + } } } - status, err := pkgtree.VerifyDepTree(vendorDir, sums) - if err != nil { - return nil, err - } - for spr, stat := range status { pr := gps.ProjectRoot(spr) - switch stat { - case pkgtree.NotInLock, pkgtree.NotInTree: - // FIXME - case pkgtree.EmptyDigestInLock, pkgtree.DigestMismatchInLock: - if _, has := sw.changed[pr]; !has { - sw.changed[gps.ProjectRoot(pr)] = hashChanged + // These cases only matter if there was no change already recorded via + // the differ. + if _, has := sw.changed[pr]; !has { + switch stat { + case verify.NotInTree: + sw.changed[pr] = missingFromTree + case verify.EmptyDigestInLock, verify.DigestMismatchInLock: + sw.changed[pr] = hashMismatch + case verify.HashVersionMismatch: + sw.changed[pr] = hashVersionMismatch } } } @@ -604,17 +614,57 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l projs[lp.Ident().ProjectRoot] = lp } + dropped := []gps.ProjectRoot{} // TODO(sdboyer) add a txn/rollback layer, like the safewriter? - //for pr, reason := range dw.changed { - for pr, _ := range dw.changed { + for pr, reason := range dw.changed { to := filepath.FromSlash(filepath.Join(vnewpath, string(pr))) po := dw.pruneOptions.PruneOptionsFor(pr) + lpd := dw.lockDiff.ProjectDiffs[pr] + switch reason { + case noChange: + panic(fmt.Sprintf("wtf, no change for %s", pr)) + case solveChanged: + if lpd.SourceChanged() { + logger.Printf("Writing %s: source changed (%s -> %s)", pr, lpd.SourceBefore, lpd.SourceAfter) + } else if lpd.VersionChanged() { + logger.Printf("Writing %s: version changed (%s -> %s)", pr, lpd.VersionBefore, lpd.VersionAfter) + } else if lpd.RevisionChanged() { + logger.Printf("Writing %s: revision changed (%s -> %s)", pr, lpd.RevisionBefore, lpd.RevisionAfter) + } else if lpd.PackagesChanged() { + la, lr := len(lpd.PackagesAdded), len(lpd.PackagesRemoved) + if la > 0 && lr > 0 { + logger.Printf("Writing %s: packages changed (%v added, %v removed)", pr, la, lr) + } else if la > 0 { + logger.Printf("Writing %s: packages changed (%v added)", pr, la) + } else { + logger.Printf("Writing %s: packages changed (%v removed)", pr, lr) + } + } else if lpd.PruneOptsChanged() { + // Override what's on the lockdiff with the extra info we have; + // this lets us excise PruneNestedVendorDirs and get the real + // value from the input param in place. + old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs + new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs + logger.Printf("Writing %s: prune options changed (%s -> %s)", pr, old, new) + } + case hashMismatch: + logger.Printf("Writing %s: hash mismatch between Gopkg.lock and vendor contents", pr) + case hashVersionMismatch: + logger.Printf("Writing %s: hashing algorithm mismatch", pr) + case projectAdded: + logger.Printf("Writing new project %s", pr) + case projectRemoved: + dropped = append(dropped, pr) + continue + case missingFromTree: + logger.Printf("Writing %s: missing from vendor", pr) + } if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil { return errors.Wrapf(err, "failed to export %s", pr) } - digest, err := pkgtree.DigestFromDirectory(to) + digest, err := verify.DigestFromDirectory(to) if err != nil { return errors.Wrapf(err, "failed to hash %s", pr) } @@ -622,6 +672,8 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l // Update the new Lock with verification information. for k, lp := range dw.lock.P { if lp.Ident().ProjectRoot == pr { + vp := lp.(verify.VerifiableProject) + vp.Digest = digest dw.lock.P[k] = verify.VerifiableProject{ LockedProject: lp, PruneOpts: po, @@ -649,6 +701,11 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l } } + for _, pr := range dropped { + // Kind of a lie to print this here. ¯\_(ツ)_/¯ + logger.Printf("Discarding unused project %s", pr) + } + err = os.RemoveAll(vpath) if err != nil { return errors.Wrap(err, "failed to remove original vendor directory")