// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package dep import ( "bytes" "fmt" "io/ioutil" "log" "os" "path/filepath" "github.com/golang/dep/internal/fs" "github.com/golang/dep/internal/gps" "github.com/pelletier/go-toml" "github.com/pkg/errors" ) // Example string to be written to the manifest file // if no dependencies are found in the project // during `dep init` var exampleTOML = []byte(` # Gopkg.toml example # # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md # for detailed Gopkg.toml documentation. # # required = ["github.com/user/thing/cmd/thing"] # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] # # [[constraint]] # name = "github.com/user/project" # version = "1.0.0" # # [[constraint]] # name = "github.com/user/project2" # branch = "dev" # source = "github.com/myfork/project2" # # [[override]] # name = "github.com/x/y" # version = "2.4.0" `) // String added on top of lock file var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. `) // SafeWriter transactionalizes writes of manifest, lock, and vendor dir, both // individually and in any combination, into a pseudo-atomic action with // transactional rollback. // // It is not impervious to errors (writing to disk is hard), but it should // guard against non-arcane failure conditions. type SafeWriter struct { Manifest *Manifest lock *Lock lockDiff *gps.LockDiff writeVendor bool writeLock bool } // NewSafeWriter sets up a SafeWriter to write a set of manifest, lock, and // vendor tree. // // - If manifest is provided, it will be written to the standard manifest file // name beneath root. // // - If newLock is provided, it will be written to the standard lock file // name beneath root. // // - If vendor is VendorAlways, or is VendorOnChanged and the locks are different, // the vendor directory will be written beneath root based on newLock. // // - If oldLock is provided without newLock, error. // // - If vendor is VendorAlways without a newLock, error. func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBehavior) (*SafeWriter, error) { sw := &SafeWriter{ Manifest: manifest, lock: newLock, } if oldLock != nil { if newLock == nil { return nil, errors.New("must provide newLock when oldLock is specified") } sw.lockDiff = gps.DiffLocks(oldLock, newLock) if sw.lockDiff != nil { sw.writeLock = true } } else if newLock != nil { sw.writeLock = true } switch vendor { case VendorAlways: sw.writeVendor = true case VendorOnChanged: sw.writeVendor = sw.lockDiff != nil || (newLock != nil && oldLock == nil) } if sw.writeVendor && newLock == nil { return nil, errors.New("must provide newLock in order to write out vendor") } return sw, nil } // HasLock checks if a Lock is present in the SafeWriter func (sw *SafeWriter) HasLock() bool { return sw.lock != nil } // HasManifest checks if a Manifest is present in the SafeWriter func (sw *SafeWriter) HasManifest() bool { return sw.Manifest != nil } type rawStringDiff struct { *gps.StringDiff } // MarshalTOML serializes the diff as a string. func (diff rawStringDiff) MarshalTOML() ([]byte, error) { return []byte(diff.String()), nil } type rawLockedProjectDiff struct { Name gps.ProjectRoot `toml:"name"` Source *rawStringDiff `toml:"source,omitempty"` Version *rawStringDiff `toml:"version,omitempty"` Branch *rawStringDiff `toml:"branch,omitempty"` Revision *rawStringDiff `toml:"revision,omitempty"` Packages []rawStringDiff `toml:"packages,omitempty"` } func toRawLockedProjectDiff(diff gps.LockedProjectDiff) rawLockedProjectDiff { // this is a shallow copy since we aren't modifying the raw diff raw := rawLockedProjectDiff{Name: diff.Name} if diff.Source != nil { raw.Source = &rawStringDiff{diff.Source} } if diff.Version != nil { raw.Version = &rawStringDiff{diff.Version} } if diff.Branch != nil { raw.Branch = &rawStringDiff{diff.Branch} } if diff.Revision != nil { raw.Revision = &rawStringDiff{diff.Revision} } raw.Packages = make([]rawStringDiff, len(diff.Packages)) for i := 0; i < len(diff.Packages); i++ { raw.Packages[i] = rawStringDiff{&diff.Packages[i]} } return raw } type rawLockedProjectDiffs struct { Projects []rawLockedProjectDiff `toml:"projects"` } func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiffs { raw := rawLockedProjectDiffs{ Projects: make([]rawLockedProjectDiff, len(diffs)), } for i := 0; i < len(diffs); i++ { raw.Projects[i] = toRawLockedProjectDiff(diffs[i]) } return raw } func formatLockDiff(diff gps.LockDiff) (string, error) { var buf bytes.Buffer if diff.HashDiff != nil { buf.WriteString(fmt.Sprintf("Memo: %s\n\n", diff.HashDiff)) } writeDiffs := func(diffs []gps.LockedProjectDiff) error { raw := toRawLockedProjectDiffs(diffs) chunk, err := toml.Marshal(raw) if err != nil { return err } buf.Write(chunk) buf.WriteString("\n") return nil } if len(diff.Add) > 0 { buf.WriteString("Add:") err := writeDiffs(diff.Add) if err != nil { return "", errors.Wrap(err, "Unable to format LockDiff.Add") } } if len(diff.Remove) > 0 { buf.WriteString("Remove:") err := writeDiffs(diff.Remove) if err != nil { return "", errors.Wrap(err, "Unable to format LockDiff.Remove") } } if len(diff.Modify) > 0 { buf.WriteString("Modify:") err := writeDiffs(diff.Modify) if err != nil { return "", errors.Wrap(err, "Unable to format LockDiff.Modify") } } return buf.String(), nil } // VendorBehavior defines when the vendor directory should be written. type VendorBehavior int const ( // VendorOnChanged indicates that the vendor directory should be written when the lock is new or changed. VendorOnChanged VendorBehavior = iota // VendorAlways forces the vendor directory to always be written. VendorAlways // VendorNever indicates the vendor directory should never be written. VendorNever ) func (sw SafeWriter) validate(root string, sm gps.SourceManager) error { if root == "" { return errors.New("root path must be non-empty") } if is, err := fs.IsDir(root); !is { if err != nil && !os.IsNotExist(err) { return err } return errors.Errorf("root path %q does not exist", root) } if sw.writeVendor && sm == nil { return errors.New("must provide a SourceManager if writing out a vendor dir") } return nil } // Write saves some combination of config yaml, lock, and a vendor tree. // root is the absolute path of root dir in which to write. // sm is only required if vendor is being written. // // It first writes to a temp dir, then moves them in place if and only if all the write // operations succeeded. It also does its best to roll back if any moves fail. // This mostly guarantees that dep cannot exit with a partial write that would // leave an undefined state on disk. func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, logger *log.Logger) error { err := sw.validate(root, sm) if err != nil { return err } if !sw.HasManifest() && !sw.writeLock && !sw.writeVendor { // nothing to do return nil } mpath := filepath.Join(root, ManifestName) lpath := filepath.Join(root, LockName) vpath := filepath.Join(root, "vendor") td, err := ioutil.TempDir(os.TempDir(), "dep") if err != nil { return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor") } defer os.RemoveAll(td) if sw.HasManifest() { // Always write the example text to the bottom of the TOML file. tb, err := sw.Manifest.MarshalTOML() if err != nil { return errors.Wrap(err, "failed to marshal manifest to TOML") } var initOutput []byte // If examples are enabled, use the example text if examples { initOutput = exampleTOML } if err = ioutil.WriteFile(filepath.Join(td, ManifestName), append(initOutput, tb...), 0666); err != nil { return errors.Wrap(err, "failed to write manifest file to temp dir") } } if sw.writeLock { l, err := sw.lock.MarshalTOML() if err != nil { return errors.Wrap(err, "failed to marshal lock to TOML") } if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil { return errors.Wrap(err, "failed to write lock file to temp dir") } } if sw.writeVendor { err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true, logger) if err != nil { return errors.Wrap(err, "error while writing out vendor tree") } } // Ensure vendor/.git is preserved if present if hasDotGit(vpath) { err = fs.RenameWithFallback(filepath.Join(vpath, ".git"), filepath.Join(td, "vendor/.git")) if _, ok := err.(*os.LinkError); ok { return errors.Wrap(err, "failed to preserve vendor/.git") } } // Move the existing files and dirs to the temp dir while we put the new // ones in, to provide insurance against errors for as long as possible. type pathpair struct { from, to string } var restore []pathpair var failerr error var vendorbak string if sw.HasManifest() { if _, err := os.Stat(mpath); err == nil { // Move out the old one. tmploc := filepath.Join(td, ManifestName+".orig") failerr = fs.RenameWithFallback(mpath, tmploc) if failerr != nil { goto fail } restore = append(restore, pathpair{from: tmploc, to: mpath}) } // Move in the new one. failerr = fs.RenameWithFallback(filepath.Join(td, ManifestName), mpath) if failerr != nil { goto fail } } if sw.writeLock { if _, err := os.Stat(lpath); err == nil { // Move out the old one. tmploc := filepath.Join(td, LockName+".orig") failerr = fs.RenameWithFallback(lpath, tmploc) if failerr != nil { goto fail } restore = append(restore, pathpair{from: tmploc, to: lpath}) } // Move in the new one. failerr = fs.RenameWithFallback(filepath.Join(td, LockName), lpath) if failerr != nil { goto fail } } if sw.writeVendor { if _, err := os.Stat(vpath); err == nil { // Move out the old vendor dir. just do it into an adjacent dir, to // try to mitigate the possibility of a pointless cross-filesystem // move with a temp directory. vendorbak = vpath + ".orig" if _, err := os.Stat(vendorbak); err == nil { // If the adjacent dir already exists, bite the bullet and move // to a proper tempdir. vendorbak = filepath.Join(td, ".vendor.orig") } failerr = fs.RenameWithFallback(vpath, vendorbak) if failerr != nil { goto fail } restore = append(restore, pathpair{from: vendorbak, to: vpath}) } // Move in the new one. failerr = fs.RenameWithFallback(filepath.Join(td, "vendor"), vpath) if failerr != nil { goto fail } } // Renames all went smoothly. The deferred os.RemoveAll will get the temp // dir, but if we wrote vendor, we have to clean that up directly if sw.writeVendor { // Nothing we can really do about an error at this point, so ignore it os.RemoveAll(vendorbak) } return nil fail: // If we failed at any point, move all the things back into place, then bail. for _, pair := range restore { // Nothing we can do on err here, as we're already in recovery mode. fs.RenameWithFallback(pair.from, pair.to) } return failerr } // PrintPreparedActions logs the actions a call to Write would perform. func (sw *SafeWriter) PrintPreparedActions(output *log.Logger, verbose bool) error { if sw.HasManifest() { if verbose { m, err := sw.Manifest.MarshalTOML() if err != nil { return errors.Wrap(err, "ensure DryRun cannot serialize manifest") } output.Printf("Would have written the following %s:\n%s\n", ManifestName, string(m)) } else { output.Printf("Would have written %s.\n", ManifestName) } } if sw.writeLock { if sw.lockDiff == nil { if verbose { l, err := sw.lock.MarshalTOML() if err != nil { return errors.Wrap(err, "ensure DryRun cannot serialize lock") } output.Printf("Would have written the following %s:\n%s\n", LockName, string(l)) } else { output.Printf("Would have written %s.\n", LockName) } } else { output.Printf("Would have written the following changes to %s:\n", LockName) diff, err := formatLockDiff(*sw.lockDiff) if err != nil { return errors.Wrap(err, "ensure DryRun cannot serialize the lock diff") } output.Println(diff) } } if sw.writeVendor { if verbose { output.Printf("Would have written the following %d projects to the vendor directory:\n", len(sw.lock.Projects())) lps := sw.lock.Projects() for i, p := range lps { output.Printf("(%d/%d) %s@%s\n", i+1, len(lps), p.Ident(), p.Version()) } } else { output.Printf("Would have written %d projects to the vendor directory.\n", len(sw.lock.Projects())) } } return nil } // hasDotGit checks if a given path has .git file or directory in it. func hasDotGit(path string) bool { gitfilepath := filepath.Join(path, ".git") _, err := os.Stat(gitfilepath) return err == nil }