Windows: Revendor hcsshim v0.4.5

This update adds support for non-base layers that have utility VM updates
and hard links. This is necessary to support Microsoft-released layers
with servicing updates.

Signed-off-by: John Starks <jostarks@microsoft.com>
This commit is contained in:
John Starks 2016-09-28 19:13:10 -07:00
Родитель 2a765279fa
Коммит cab70c391f
4 изменённых файлов: 373 добавлений и 59 удалений

Просмотреть файл

@ -48,7 +48,7 @@ esac
# the following lines are in sorted order, FYI
clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
clone git github.com/Microsoft/hcsshim v0.4.3
clone git github.com/Microsoft/hcsshim v0.4.5
clone git github.com/Microsoft/go-winio v0.3.4
clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps
clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a

Просмотреть файл

@ -23,6 +23,26 @@ type dirInfo struct {
fileInfo winio.FileBasicInfo
}
// reapplyDirectoryTimes reapplies directory modification, creation, etc. times
// after processing of the directory tree has completed. The times are expected
// to be ordered such that parent directories come before child directories.
func reapplyDirectoryTimes(dis []dirInfo) error {
for i := range dis {
di := &dis[len(dis)-i-1] // reverse order: process child directories first
f, err := winio.OpenForBackup(di.path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, syscall.OPEN_EXISTING)
if err != nil {
return err
}
err = winio.SetFileBasicInfo(f, &di.fileInfo)
f.Close()
if err != nil {
return err
}
}
return nil
}
func (w *baseLayerWriter) closeCurrentFile() error {
if w.f != nil {
err := w.bw.Close()
@ -142,18 +162,9 @@ func (w *baseLayerWriter) Close() error {
if w.err == nil {
// Restore the file times of all the directories, since they may have
// been modified by creating child directories.
for i := range w.dirInfo {
di := &w.dirInfo[len(w.dirInfo)-i-1]
f, err := winio.OpenForBackup(di.path, uint32(syscall.GENERIC_READ|syscall.GENERIC_WRITE), syscall.FILE_SHARE_READ, syscall.OPEN_EXISTING)
if err != nil {
return makeError(err, "Failed to OpenForBackup", di.path)
}
err = winio.SetFileBasicInfo(f, &di.fileInfo)
f.Close()
if err != nil {
return makeError(err, "Failed to SetFileBasicInfo", di.path)
}
err = reapplyDirectoryTimes(w.dirInfo)
if err != nil {
return err
}
err = ProcessBaseLayer(w.root)

Просмотреть файл

@ -130,21 +130,42 @@ type legacyLayerWriterWrapper struct {
}
func (r *legacyLayerWriterWrapper) Close() error {
defer os.RemoveAll(r.root)
err := r.legacyLayerWriter.Close()
if err == nil {
var fullPath string
// Use the original path here because ImportLayer does not support long paths for the source in TP5.
// But do use a long path for the destination to work around another bug with directories
// with MAX_PATH - 12 < length < MAX_PATH.
info := r.info
fullPath, err = makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID))
if err == nil {
info.HomeDir = ""
err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths)
if err != nil {
return err
}
// Use the original path here because ImportLayer does not support long paths for the source in TP5.
// But do use a long path for the destination to work around another bug with directories
// with MAX_PATH - 12 < length < MAX_PATH.
info := r.info
fullPath, err := makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID))
if err != nil {
return err
}
info.HomeDir = ""
if err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths); err != nil {
return err
}
// Add any hard links that were collected.
for _, lnk := range r.PendingLinks {
if err = os.Remove(lnk.Path); err != nil && !os.IsNotExist(err) {
return err
}
if err = os.Link(lnk.Target, lnk.Path); err != nil {
return err
}
}
os.RemoveAll(r.root)
return err
// Prepare the utility VM for use if one is present in the layer.
if r.HasUtilityVM {
err = ProcessUtilityVMImage(filepath.Join(fullPath, "UtilityVM"))
if err != nil {
return err
}
}
return nil
}
// NewLayerWriter returns a new layer writer for creating a layer on disk.
@ -166,7 +187,7 @@ func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string)
return nil, err
}
return &legacyLayerWriterWrapper{
legacyLayerWriter: newLegacyLayerWriter(path),
legacyLayerWriter: newLegacyLayerWriter(path, parentLayerPaths, filepath.Join(info.HomeDir, layerID)),
info: info,
layerID: layerID,
path: path,

Просмотреть файл

@ -16,6 +16,13 @@ import (
var errorIterationCanceled = errors.New("")
var mutatedUtilityVMFiles = map[string]bool{
`EFI\Microsoft\Boot\BCD`: true,
`EFI\Microsoft\Boot\BCD.LOG`: true,
`EFI\Microsoft\Boot\BCD.LOG1`: true,
`EFI\Microsoft\Boot\BCD.LOG2`: true,
}
func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) {
return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition)
}
@ -49,17 +56,15 @@ type legacyLayerReader struct {
proceed chan bool
currentFile *os.File
backupReader *winio.BackupFileReader
isTP4Format bool
}
// newLegacyLayerReader returns a new LayerReader that can read the Windows
// TP4 transport format from disk.
// container layer transport format from disk.
func newLegacyLayerReader(root string) *legacyLayerReader {
r := &legacyLayerReader{
root: root,
result: make(chan *fileEntry),
proceed: make(chan bool),
isTP4Format: IsTP4(),
root: root,
result: make(chan *fileEntry),
proceed: make(chan bool),
}
go r.walk()
return r
@ -251,17 +256,14 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
fileInfo.LastAccessTime = fileInfo.LastWriteTime
} else {
beginning := int64(0)
if !r.isTP4Format {
// In TP5, the file attributes were added before the backup stream
var attr uint32
err = binary.Read(f, binary.LittleEndian, &attr)
if err != nil {
return
}
fileInfo.FileAttributes = uintptr(attr)
beginning = 4
// The file attributes are written before the backup stream.
var attr uint32
err = binary.Read(f, binary.LittleEndian, &attr)
if err != nil {
return
}
fileInfo.FileAttributes = uintptr(attr)
beginning := int64(4)
// Find the accurate file size.
if !fe.fi.IsDir() {
@ -301,21 +303,32 @@ func (r *legacyLayerReader) Close() error {
return nil
}
type pendingLink struct {
Path, Target string
}
type legacyLayerWriter struct {
root string
parentRoots []string
destRoot string
currentFile *os.File
backupWriter *winio.BackupFileWriter
tombstones []string
isTP4Format bool
pathFixed bool
HasUtilityVM bool
uvmDi []dirInfo
addedFiles map[string]bool
PendingLinks []pendingLink
}
// newLegacyLayerWriter returns a LayerWriter that can write the TP4 transport format
// to disk.
func newLegacyLayerWriter(root string) *legacyLayerWriter {
// newLegacyLayerWriter returns a LayerWriter that can write the contaler layer
// transport format to disk.
func newLegacyLayerWriter(root string, parentRoots []string, destRoot string) *legacyLayerWriter {
return &legacyLayerWriter{
root: root,
isTP4Format: IsTP4(),
parentRoots: parentRoots,
destRoot: destRoot,
addedFiles: make(map[string]bool),
}
}
@ -325,12 +338,42 @@ func (w *legacyLayerWriter) init() error {
if err != nil {
return err
}
for i, p := range w.parentRoots {
w.parentRoots[i], err = makeLongAbsPath(p)
if err != nil {
return err
}
}
destPath, err := makeLongAbsPath(w.destRoot)
if err != nil {
return err
}
w.root = path
w.destRoot = destPath
w.pathFixed = true
}
return nil
}
func (w *legacyLayerWriter) initUtilityVM() error {
if !w.HasUtilityVM {
err := os.Mkdir(filepath.Join(w.destRoot, `UtilityVM`), 0)
if err != nil {
return err
}
// Server 2016 does not support multiple layers for the utility VM, so
// clone the utility VM from the parent layer into this layer. Use hard
// links to avoid unnecessary copying, since most of the files are
// immutable.
err = cloneTree(filepath.Join(w.parentRoots[0], `UtilityVM\Files`), filepath.Join(w.destRoot, `UtilityVM\Files`), mutatedUtilityVMFiles)
if err != nil {
return fmt.Errorf("cloning the parent utility VM image failed: %s", err)
}
w.HasUtilityVM = true
}
return nil
}
func (w *legacyLayerWriter) reset() {
if w.backupWriter != nil {
w.backupWriter.Close()
@ -342,15 +385,180 @@ func (w *legacyLayerWriter) reset() {
}
}
// copyFileWithMetadata copies a file using the backup/restore APIs in order to preserve metadata
func copyFileWithMetadata(srcPath, destPath string, isDir bool) (fileInfo *winio.FileBasicInfo, err error) {
createDisposition := uint32(syscall.CREATE_NEW)
if isDir {
err = os.Mkdir(destPath, 0)
if err != nil {
return nil, err
}
createDisposition = syscall.OPEN_EXISTING
}
src, err := openFileOrDir(srcPath, syscall.GENERIC_READ|winio.ACCESS_SYSTEM_SECURITY, syscall.OPEN_EXISTING)
if err != nil {
return nil, err
}
defer src.Close()
srcr := winio.NewBackupFileReader(src, true)
defer srcr.Close()
fileInfo, err = winio.GetFileBasicInfo(src)
if err != nil {
return nil, err
}
dest, err := openFileOrDir(destPath, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition)
if err != nil {
return nil, err
}
defer dest.Close()
err = winio.SetFileBasicInfo(dest, fileInfo)
if err != nil {
return nil, err
}
destw := winio.NewBackupFileWriter(dest, true)
defer func() {
cerr := destw.Close()
if err == nil {
err = cerr
}
}()
_, err = io.Copy(destw, srcr)
if err != nil {
return nil, err
}
return fileInfo, nil
}
// cloneTree clones a directory tree using hard links. It skips hard links for
// the file names in the provided map and just copies those files.
func cloneTree(srcPath, destPath string, mutatedFiles map[string]bool) error {
var di []dirInfo
err := filepath.Walk(srcPath, func(srcFilePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(srcPath, srcFilePath)
if err != nil {
return err
}
destFilePath := filepath.Join(destPath, relPath)
// Directories, reparse points, and files that will be mutated during
// utility VM import must be copied. All other files can be hard linked.
isReparsePoint := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0
if info.IsDir() || isReparsePoint || mutatedFiles[relPath] {
fi, err := copyFileWithMetadata(srcFilePath, destFilePath, info.IsDir())
if err != nil {
return err
}
if info.IsDir() && !isReparsePoint {
di = append(di, dirInfo{path: destFilePath, fileInfo: *fi})
}
} else {
err = os.Link(srcFilePath, destFilePath)
if err != nil {
return err
}
}
// Don't recurse on reparse points.
if info.IsDir() && isReparsePoint {
return filepath.SkipDir
}
return nil
})
if err != nil {
return err
}
return reapplyDirectoryTimes(di)
}
func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error {
w.reset()
err := w.init()
if err != nil {
return err
}
path := filepath.Join(w.root, name)
createDisposition := uint32(syscall.CREATE_NEW)
if name == `UtilityVM` {
return w.initUtilityVM()
}
if strings.HasPrefix(name, `UtilityVM\`) {
if !w.HasUtilityVM {
return errors.New("missing UtilityVM directory")
}
if !strings.HasPrefix(name, `UtilityVM\Files\`) && name != `UtilityVM\Files` {
return errors.New("invalid UtilityVM layer")
}
path := filepath.Join(w.destRoot, name)
createDisposition := uint32(syscall.OPEN_EXISTING)
if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
st, err := os.Lstat(path)
if err != nil && !os.IsNotExist(err) {
return err
}
if st != nil {
// Delete the existing file/directory if it is not the same type as this directory.
existingAttr := st.Sys().(*syscall.Win32FileAttributeData).FileAttributes
if (uint32(fileInfo.FileAttributes)^existingAttr)&(syscall.FILE_ATTRIBUTE_DIRECTORY|syscall.FILE_ATTRIBUTE_REPARSE_POINT) != 0 {
if err = os.RemoveAll(path); err != nil {
return err
}
st = nil
}
}
if st == nil {
if err = os.Mkdir(path, 0); err != nil {
return err
}
}
if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
w.uvmDi = append(w.uvmDi, dirInfo{path: path, fileInfo: *fileInfo})
}
} else {
// Overwrite any existing hard link.
err = os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return err
}
createDisposition = syscall.CREATE_NEW
}
f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition)
if err != nil {
return err
}
defer func() {
if f != nil {
f.Close()
os.Remove(path)
}
}()
err = winio.SetFileBasicInfo(f, fileInfo)
if err != nil {
return err
}
w.backupWriter = winio.NewBackupFileWriter(f, true)
w.currentFile = f
w.addedFiles[name] = true
f = nil
return nil
}
path := filepath.Join(w.root, name)
if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
err := os.Mkdir(path, 0)
if err != nil {
@ -359,7 +567,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
path += ".$wcidirs$"
}
f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, createDisposition)
f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.CREATE_NEW)
if err != nil {
return err
}
@ -380,29 +588,97 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
if strings.HasPrefix(name, `Hives\`) {
w.backupWriter = winio.NewBackupFileWriter(f, false)
} else {
if !w.isTP4Format {
// In TP5, the file attributes were added to the header
err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes))
if err != nil {
return err
}
// The file attributes are written before the stream.
err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes))
if err != nil {
return err
}
}
w.currentFile = f
w.addedFiles[name] = true
f = nil
return nil
}
func (w *legacyLayerWriter) AddLink(name string, target string) error {
return errors.New("hard links not supported with legacy writer")
w.reset()
err := w.init()
if err != nil {
return err
}
var requiredPrefix string
var roots []string
if prefix := `Files\`; strings.HasPrefix(name, prefix) {
requiredPrefix = prefix
// Look for cross-layer hard link targets in the parent layers, since
// nothing is in the destination path yet.
roots = w.parentRoots
} else if prefix := `UtilityVM\Files\`; strings.HasPrefix(name, prefix) {
requiredPrefix = prefix
// Since the utility VM is fully cloned into the destination path
// already, look for cross-layer hard link targets directly in the
// destination path.
roots = []string{w.destRoot}
}
if requiredPrefix == "" || !strings.HasPrefix(target, requiredPrefix) {
return errors.New("invalid hard link in layer")
}
// Find to try the target of the link in a previously added file. If that
// fails, search in parent layers.
var selectedRoot string
if _, ok := w.addedFiles[target]; ok {
selectedRoot = w.destRoot
} else {
for _, r := range roots {
if _, err = os.Lstat(filepath.Join(r, target)); err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
selectedRoot = r
break
}
}
if selectedRoot == "" {
return fmt.Errorf("failed to find link target for '%s' -> '%s'", name, target)
}
}
// The link can't be written until after the ImportLayer call.
w.PendingLinks = append(w.PendingLinks, pendingLink{
Path: filepath.Join(w.destRoot, name),
Target: filepath.Join(selectedRoot, target),
})
w.addedFiles[name] = true
return nil
}
func (w *legacyLayerWriter) Remove(name string) error {
if !strings.HasPrefix(name, `Files\`) {
if strings.HasPrefix(name, `Files\`) {
w.tombstones = append(w.tombstones, name[len(`Files\`):])
} else if strings.HasPrefix(name, `UtilityVM\Files\`) {
err := w.initUtilityVM()
if err != nil {
return err
}
// Make sure the path exists; os.RemoveAll will not fail if the file is
// already gone, and this needs to be a fatal error for diagnostics
// purposes.
path := filepath.Join(w.destRoot, name)
if _, err := os.Lstat(path); err != nil {
return err
}
err = os.RemoveAll(path)
if err != nil {
return err
}
} else {
return fmt.Errorf("invalid tombstone %s", name)
}
w.tombstones = append(w.tombstones, name[len(`Files\`):])
return nil
}
@ -437,5 +713,11 @@ func (w *legacyLayerWriter) Close() error {
return err
}
}
if w.HasUtilityVM {
err = reapplyDirectoryTimes(w.uvmDi)
if err != nil {
return err
}
}
return nil
}