From 7bd75d547d6e3b27b4135869fffa4b18bcfed2d1 Mon Sep 17 00:00:00 2001 From: arvindkandhare Date: Tue, 16 Feb 2021 14:35:55 -0800 Subject: [PATCH] Overlay based diff image creation cherry pick (#611) * Overlay based diff image creation prototype Here is a link to the spec https://microsoft-my.sharepoint.com/:w:/g/personal/arvindka_microsoft_com1/ESrYHTpWUPBOgdi7LjDsE14Bf1mHSLG702551XctkFX1mA?e=CyCc2j. This is for early feedback on the approach. It introduces a new element, BaseImage for each partition. Instead of creating a complete new partition image, a new diff layer is created using overlay file system. Overlay file system is a simple implementation of union file system. The changes files are completely copied in the upper level overlay. The implementation then copies the higher level files in a tgz. This tgz can be transferred to the ADU agent which first rehydrates the base image and then uses SWUpdate to do the A/B switch. Co-authored-by: Arvind Kandhare --- toolkit/docs/formats/imageconfig.md | 36 ++- .../configuration/configuration_test.go | 18 +- .../configuration/partitionsetting.go | 10 +- .../testdata/test_configuration.json | 6 +- .../imagegen/installutils/installutils.go | 247 +++++++++++++++--- .../tools/imagegen/installutils/overlay.go | 84 ++++++ toolkit/tools/imager/imager.go | 57 ++-- toolkit/tools/roast/formats/diff.go | 36 +++ toolkit/tools/roast/formats/rdiff.go | 36 +++ toolkit/tools/roast/roast.go | 30 ++- 10 files changed, 486 insertions(+), 74 deletions(-) create mode 100644 toolkit/tools/imagegen/installutils/overlay.go create mode 100644 toolkit/tools/roast/formats/diff.go create mode 100644 toolkit/tools/roast/formats/rdiff.go diff --git a/toolkit/docs/formats/imageconfig.md b/toolkit/docs/formats/imageconfig.md index 338065bd3c..38f007eb34 100644 --- a/toolkit/docs/formats/imageconfig.md +++ b/toolkit/docs/formats/imageconfig.md @@ -102,6 +102,36 @@ A sample PartitionSettings entry, designating an EFI and a root partitions: ], ``` +It is possible to use `PartitionSettings` to configure diff disk image creation. Two types of diffs are possible. +`rdiff` and `overlay` diff. + +For small and deterministic images `rdiff` is a better algorithm. +For large images based on `ext4` `overlay` diff is a better algorithm. + +A sample `ParitionSettings` entry using `rdiff` algorithm: + +``` json +{ + "ID": "boot", + "MountPoint": "/boot/efi", + "MountOptions" : "umask=0077", + "RdiffBaseImage" : "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4" +}, + ``` + +A sample `ParitionSettings` entry using `overlay` algorithm: + +``` json +{ + "ID": "rootfs", + "MountPoint": "/", + "OverlayBaseImage" : "../out/images/core-efi/core-efi-rootfs-1.0.20200918.1751.ext4" +} + +``` +`RdiffBaseImage` represents the base image when `rdiff` algorithm is used. +`OverlayBaseImage` represents the base image when `overlay` algorithm is used. + ### PackageLists PackageLists key consists of an array of relative paths to the package lists (JSON files). @@ -301,11 +331,13 @@ A sample image configuration, producing a VHDX disk image: { "ID": "boot", "MountPoint": "/boot/efi", - "MountOptions" : "umask=0077" + "MountOptions" : "umask=0077", + "RdiffBaseImage" : "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4" }, { "ID": "rootfs", - "MountPoint": "/" + "MountPoint": "/", + "OverlayBaseImage" : "../out/images/core-efi/core-efi-rootfs-1.0.20200918.1751.ext4" } ], "PackageLists": [ diff --git a/toolkit/tools/imagegen/configuration/configuration_test.go b/toolkit/tools/imagegen/configuration/configuration_test.go index 5bb9838e96..8d1e925d47 100644 --- a/toolkit/tools/imagegen/configuration/configuration_test.go +++ b/toolkit/tools/imagegen/configuration/configuration_test.go @@ -268,15 +268,17 @@ var expectedConfiguration Config = Config{ Name: "SmallerDisk", IsDefault: true, PartitionSettings: []PartitionSetting{ - { - ID: "MyBoot", - MountPoint: "/boot", - MountOptions: "ro,exec", + PartitionSetting{ + ID: "MyBoot", + MountPoint: "/boot", + MountOptions: "ro,exec", + RdiffBaseImage: "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4", }, - { - ID: "MyRootfs", - MountPoint: "/", - RemoveDocs: true, + PartitionSetting{ + ID: "MyRootfs", + MountPoint: "/", + RemoveDocs: true, + OverlayBaseImage: "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4", }, }, PackageLists: []string{ diff --git a/toolkit/tools/imagegen/configuration/partitionsetting.go b/toolkit/tools/imagegen/configuration/partitionsetting.go index dd95fb254d..902524366b 100644 --- a/toolkit/tools/imagegen/configuration/partitionsetting.go +++ b/toolkit/tools/imagegen/configuration/partitionsetting.go @@ -12,10 +12,12 @@ import ( // PartitionSetting holds the mounting information for each partition. type PartitionSetting struct { - RemoveDocs bool `json:"RemoveDocs"` - ID string `json:"ID"` - MountOptions string `json:"MountOptions"` - MountPoint string `json:"MountPoint"` + RemoveDocs bool `json:"RemoveDocs"` + ID string `json:"ID"` + MountOptions string `json:"MountOptions"` + MountPoint string `json:"MountPoint"` + OverlayBaseImage string `json:"OverlayBaseImage"` + RdiffBaseImage string `json:"RdiffBaseImage"` } // IsValid returns an error if the PartitionSetting is not valid diff --git a/toolkit/tools/imagegen/configuration/testdata/test_configuration.json b/toolkit/tools/imagegen/configuration/testdata/test_configuration.json index 7973a9186d..1e10639416 100644 --- a/toolkit/tools/imagegen/configuration/testdata/test_configuration.json +++ b/toolkit/tools/imagegen/configuration/testdata/test_configuration.json @@ -111,12 +111,14 @@ { "ID": "MyBoot", "MountPoint": "/boot", - "MountOptions": "ro,exec" + "MountOptions": "ro,exec", + "RdiffBaseImage": "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4" }, { "ID": "MyRootfs", "MountPoint": "/", - "RemoveDocs": true + "RemoveDocs": true, + "OverlayBaseImage": "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4" } ], "PackageLists": [ diff --git a/toolkit/tools/imagegen/installutils/installutils.go b/toolkit/tools/imagegen/installutils/installutils.go index a56b10781b..67eb168861 100644 --- a/toolkit/tools/imagegen/installutils/installutils.go +++ b/toolkit/tools/imagegen/installutils/installutils.go @@ -27,6 +27,9 @@ import ( ) const ( + // NullDevice represents the /dev/null device used as a mount device for overlay images. + NullDevice = "/dev/null" + overlay = "overlay" rootMountPoint = "/" rootUser = "root" @@ -53,7 +56,8 @@ type PackageList struct { // - mountPointDevPathMap is a map of mountpoint to partition device path // - mountPointToFsTypeMap is a map of mountpoint to filesystem type // - mountPointToMountArgsMap is a map of mountpoint to mount arguments to be passed on a call to mount -func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]string, config configuration.SystemConfig) (mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string) { +// - diffDiskBuild is a flag that denotes whether this is a diffdisk build or not +func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]string, config configuration.SystemConfig) (mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, diffDiskBuild bool) { mountPointDevPathMap = make(map[string]string) mountPointToFsTypeMap = make(map[string]string) mountPointToMountArgsMap = make(map[string]string) @@ -63,9 +67,13 @@ func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]s logger.Log.Tracef("%v[%v]", partitionSetting.ID, partitionSetting.MountPoint) partDevPath, ok := partDevPathMap[partitionSetting.ID] if ok { - mountPointDevPathMap[partitionSetting.MountPoint] = partDevPath - mountPointToFsTypeMap[partitionSetting.MountPoint] = partIDToFsTypeMap[partitionSetting.ID] - mountPointToMountArgsMap[partitionSetting.MountPoint] = partitionSetting.MountOptions + if partitionSetting.OverlayBaseImage == "" { + mountPointDevPathMap[partitionSetting.MountPoint] = partDevPath + mountPointToFsTypeMap[partitionSetting.MountPoint] = partIDToFsTypeMap[partitionSetting.ID] + mountPointToMountArgsMap[partitionSetting.MountPoint] = partitionSetting.MountOptions + } else { + diffDiskBuild = true + } } logger.Log.Tracef("%v", mountPointDevPathMap) } @@ -94,18 +102,69 @@ func sortMountPoints(mountPointMap *map[string]string, sortForUnmount bool) (rem // Reverse the sorting so we unmount in the opposite order sort.Sort(sort.Reverse(sort.StringSlice(remainingMounts))) } + return +} +// UpdatePartitionMapWithOverlays Creates Overlay map and updates the partition map with required parameters. +// - partDevPathMap is a map of partition IDs to partition device paths +// - partIDToFsTypeMap is a map of partition IDs to filesystem type +// - mountPointDevPathMap is a map of mountpoint to partition device path +// - mountPointToFsTypeMap is a map of mountpoint to filesystem type +// - mountPointToMountArgsMap is a map of mountpoint to mount arguments to be passed on a call to mount +// - config is the SystemConfig from a config file +// Output +// - mountPointToOverlayMap is a map of mountpoint to overlay data +func UpdatePartitionMapWithOverlays(partDevPathMap, partIDToFsTypeMap, mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, config configuration.SystemConfig) (mountPointToOverlayMap map[string]*Overlay, err error) { + mountPointToOverlayMap = make(map[string]*Overlay) + + // Go through each PartitionSetting + for _, partitionSetting := range config.PartitionSettings { + logger.Log.Tracef("%v[%v]", partitionSetting.ID, partitionSetting.MountPoint) + if partitionSetting.OverlayBaseImage != "" { + err = createOverlayPartition(partitionSetting, mountPointDevPathMap, mountPointToMountArgsMap, mountPointToFsTypeMap, mountPointToOverlayMap) + if err != nil { + return + } + } + } + return +} + +func createOverlayPartition(partitionSetting configuration.PartitionSetting, mountPointDevPathMap, mountPointToMountArgsMap, mountPointToFsTypeMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) { + //Mount the base image + //Create a temp upper dir + //Add to the mount args + devicePath, err := diskutils.SetupLoopbackDevice(partitionSetting.OverlayBaseImage) + + if err != nil { + logger.Log.Errorf("Could not setup loop back device for mount (%s)", partitionSetting.OverlayBaseImage) + + return + } + + overlayMount := NewOverlay(devicePath) + + mountPointToOverlayMap[partitionSetting.MountPoint] = &overlayMount + + // For overlays the device to be mounted is /dev/null. The actual device is synthesized from the lower and upper dir args + // These args are passed to the mount command using -o + mountPointDevPathMap[partitionSetting.MountPoint] = NullDevice + mountPointToMountArgsMap[partitionSetting.MountPoint] = overlayMount.getMountArgs() + mountPointToFsTypeMap[partitionSetting.MountPoint] = overlay return } // CreateInstallRoot walks through the map of mountpoints and mounts the partitions into installroot // - installRoot is the destination path to mount these partitions // - mountPointMap is the map of mountpoint to partition device path -func CreateInstallRoot(installRoot string, mountPointMap, mountPointToMountArgsMap map[string]string) (installMap map[string]string, err error) { +// - mountPointToFsTypeMap is the map of mountpoint to the file type +// - mountPointToMountArgsMap is the map of mountpoint to the parameters sent to +// - mountPointToOverlayMap is the map of mountpoint to the overlay structure containing the base image +func CreateInstallRoot(installRoot string, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (installMap map[string]string, err error) { installMap = make(map[string]string) for _, mountPoint := range sortMountPoints(&mountPointMap, false) { device := mountPointMap[mountPoint] - err = mountSingleMountPoint(installRoot, mountPoint, device, mountPointToMountArgsMap[mountPoint]) + err = mountSingleMountPoint(installRoot, mountPoint, device, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint], mountPointToOverlayMap[mountPoint]) if err != nil { return } @@ -117,7 +176,12 @@ func CreateInstallRoot(installRoot string, mountPointMap, mountPointToMountArgsM // DestroyInstallRoot unmounts each of the installroot mountpoints in order, ensuring that the root mountpoint is last // - installRoot is the path to the root where the mountpoints exist // - mountPointMap is the map of mountpoints to partition device paths -func DestroyInstallRoot(installRoot string, mountPointMap map[string]string) (err error) { +// - mountPointToOverlayMap is the map of mountpoints to overlay devices +func DestroyInstallRoot(installRoot string, mountPointMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) { + logger.Log.Trace("Destroying InstallRoot") + + defer OverlayUnmount(mountPointToOverlayMap) + logger.Log.Trace("Destroying InstallRoot") // Reverse order for unmounting for _, mountPoint := range sortMountPoints(&mountPointMap, true) { @@ -135,36 +199,62 @@ func DestroyInstallRoot(installRoot string, mountPointMap map[string]string) (er return } -func mountSingleMountPoint(installRoot, mountPoint, device, extraOptions string) (err error) { - logger.Log.Debugf("Mounting %s to %s", device, mountPoint) +// OverlayUnmount unmounts the overlay devices that are stored in the map, It ignores the errors and returns the last error. +// - mountPointToOverlayMap is the map of mountpoints to overlay devices +func OverlayUnmount(mountPointToOverlayMap map[string]*Overlay) (err error) { + for _, overlay := range mountPointToOverlayMap { + temperr := overlay.unmount() + if temperr != nil { + // Log a warning on error. Return the last error. + logger.Log.Warnf("Failed to unmount the overlay (%+v). Continuing without unmounting: (%v)", overlay, temperr) + err = temperr + } + } + return +} + +func mountSingleMountPoint(installRoot, mountPoint, device, fsType, extraOptions string, overlayDevice *Overlay) (err error) { mountPath := filepath.Join(installRoot, mountPoint) err = os.MkdirAll(mountPath, os.ModePerm) if err != nil { logger.Log.Warnf("Failed to create mountpoint: %v", err) return } - err = mount(mountPath, device, extraOptions) + + if overlayDevice != nil { + err = overlayDevice.setupFolders() + if err != nil { + logger.Log.Errorf("Failed to create mount for overlay device: %v", err) + return + } + } + err = mount(mountPath, device, fsType, extraOptions) return } func unmountSingleMountPoint(installRoot, mountPoint string) (err error) { + mountPath := filepath.Join(installRoot, mountPoint) err = umount(mountPath) return } -func mount(path, device, extraOptions string) (err error) { +func mount(path, device, fsType, extraOptions string) (err error) { const squashErrors = false - if extraOptions == "" { - err = shell.ExecuteLive(squashErrors, "mount", device, path) - } else { - err = shell.ExecuteLive(squashErrors, "mount", "-o", extraOptions, device, path) + var mountArgs []string + + if fsType != "" { + mountArgs = append(mountArgs, "-t", fsType) } - if err != nil { - return + if extraOptions != "" { + mountArgs = append(mountArgs, "-o", extraOptions) } + + mountArgs = append(mountArgs, device, path) + + err = shell.ExecuteLive(squashErrors, "mount", mountArgs...) return } @@ -268,7 +358,8 @@ func PackageNamesFromConfig(config configuration.Config) (packageList []*pkgjson // - mountPointToMountArgsMap is a map of mountpoints to mount options // - isRootFS specifies if the installroot is either backed by a directory (rootfs) or a raw disk // - encryptedRoot stores information about the encrypted root device if root encryption is enabled -func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []string, config configuration.SystemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice) (err error) { +//- diffDiskBuild is a flag that denotes whether this is a diffdisk build or not +func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []string, config configuration.SystemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, diffDiskBuild bool) (err error) { const ( filesystemPkg = "filesystem" ) @@ -280,7 +371,7 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s installRoot := filepath.Join(rootMountPoint, installChroot.RootDir()) // Initialize RPM Database so we can install RPMs into the installroot - err = initializeRpmDatabase(installRoot) + err = initializeRpmDatabase(installRoot, diffDiskBuild) if err != nil { return } @@ -315,7 +406,7 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s } hostname := config.Hostname - if !isRootFS { + if !isRootFS && mountPointToFsTypeMap[rootMountPoint] != overlay { // Add /etc/hostname err = updateHostname(installChroot.RootDir(), hostname) if err != nil { @@ -377,15 +468,21 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s return } -func initializeRpmDatabase(installRoot string) (err error) { - stdout, stderr, err := shell.Execute("rpm", "--root", installRoot, "--initdb") - if err != nil { - logger.Log.Warnf("Failed to create rpm database: %v", err) - logger.Log.Warn(stdout) - logger.Log.Warn(stderr) - return - } +func initializeRpmDatabase(installRoot string, diffDiskBuild bool) (err error) { + if !diffDiskBuild { + var ( + stdout string + stderr string + ) + stdout, stderr, err = shell.Execute("rpm", "--root", installRoot, "--initdb") + if err != nil { + logger.Log.Warnf("Failed to create rpm database: %v", err) + logger.Log.Warn(stdout) + logger.Log.Warn(stderr) + return err + } + } err = initializeTdnfConfiguration(installRoot) return } @@ -653,7 +750,7 @@ func updateFstab(installRoot string, installMap, mountPointToFsTypeMap, mountPoi ReportAction("Configuring fstab") for mountPoint, devicePath := range installMap { - if mountPoint != "" { + if mountPoint != "" && devicePath != NullDevice { err = addEntryToFstab(installRoot, mountPoint, devicePath, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint]) if err != nil { return @@ -1495,6 +1592,7 @@ func runPostInstallScripts(installChroot *safechroot.Chroot, config configuratio logger.Log.Infof("Running post-install script: %s", script.Path) err = installChroot.UnsafeRun(func() error { err := shell.ExecuteLive(squashErrors, shell.ShellProgram, "-c", scriptPath, script.Args) + if err != nil { return err } @@ -1711,31 +1809,79 @@ func setGrubCfgRootDevice(rootDevice, grubPath, luksUUID string) (err error) { } // ExtractPartitionArtifacts scans through the SystemConfig and generates all the partition-based artifacts specified. +// - setupChrootDirPath is the path to the setup root dir where the build takes place // - workDirPath is the directory to place the artifacts +// - diskIndex is the index of the disk this is added to the parition artifact generated +// - disk configuration settings for the disk +// - systemConfig system configration corresponding to the disk configuration // - partIDToDevPathMap is a map of partition IDs to partition device paths -func ExtractPartitionArtifacts(workDirPath string, diskIndex int, disk configuration.Disk, partIDToDevPathMap map[string]string) (err error) { +// - mountPointToOverlayMap is a map of mountpoints to the overlay details for this mount if any +func ExtractPartitionArtifacts(setupChrootDirPath, workDirPath string, diskIndex int, disk configuration.Disk, systemConfig configuration.SystemConfig, partIDToDevPathMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) { const ( - ext4ArtifactType = "ext4" + ext4ArtifactType = "ext4" + diffArtifactType = "diff" + rdiffArtifactType = "rdiff" ) - // Scan each partition for Artifacts for i, partition := range disk.Partitions { for _, artifact := range partition.Artifacts { - if artifact.Type == ext4ArtifactType { - devPath := partIDToDevPathMap[partition.ID] + devPath := partIDToDevPathMap[partition.ID] + switch artifact.Type { + case ext4ArtifactType: // Ext4 artifact type output is a .raw of the partition finalName := fmt.Sprintf("disk%d.partition%d.raw", diskIndex, i) err = createRawArtifact(workDirPath, devPath, finalName) if err != nil { return err } + case diffArtifactType: + for _, setting := range systemConfig.PartitionSettings { + if setting.ID == partition.ID { + if setting.OverlayBaseImage != "" { + // Diff artifact type output + finalName := fmt.Sprintf("disk%d.partition%d.diff", diskIndex, i) + err = createDiffArtifact(setupChrootDirPath, workDirPath, finalName, mountPointToOverlayMap[setting.MountPoint]) + } + break + } + } + + case rdiffArtifactType: + for _, setting := range systemConfig.PartitionSettings { + if setting.ID == partition.ID { + if setting.RdiffBaseImage != "" { + // Diff artifact type output + finalName := fmt.Sprintf("disk%d.partition%d.rdiff", diskIndex, i) + err = createRDiffArtifact(workDirPath, devPath, setting.RdiffBaseImage, finalName) + } + break + } + } } } } return } +func createDiffArtifact(setupChrootDirPath, workDirPath, name string, overlay *Overlay) (err error) { + const ( + squashErrors = true + ) + + fullPath := filepath.Join(workDirPath, name) + + upperDir := overlay.getUpperDir() + upperDir = filepath.Join(setupChrootDirPath, upperDir) + tarArgs := []string{ + "cvf", + fullPath, + "-C", + upperDir, + "."} + + return shell.ExecuteLive(squashErrors, "tar", tarArgs...) +} func createRawArtifact(workDirPath, devPath, name string) (err error) { const ( defaultBlockSize = 1024 * 1024 // 1MB @@ -1753,6 +1899,39 @@ func createRawArtifact(workDirPath, devPath, name string) (err error) { return shell.ExecuteLive(squashErrors, "dd", ddArgs...) } +func createRDiffArtifact(workDirPath, devPath, rDiffBaseImage, name string) (err error) { + const ( + signatureFileName = "./signature" + squashErrors = true + ) + + fullPath := filepath.Join(workDirPath, name) + + // rdiff expectes the signature file path to be relative. + rdiffArgs := []string{ + "signature", + rDiffBaseImage, + signatureFileName, + } + + err = shell.ExecuteLive(squashErrors, "rdiff", rdiffArgs...) + if err != nil { + return + } + + signatureFileFullPath := filepath.Join(workDirPath, signatureFileName) + defer os.Remove(signatureFileFullPath) + + rdiffArgs = []string{ + "delta", + signatureFileName, + devPath, + fullPath, + } + + return shell.ExecuteLive(squashErrors, "rdiff", rdiffArgs...) +} + // isRunningInHyperV checks if the program is running in a Hyper-V Virtual Machine. func isRunningInHyperV() (isHyperV bool, err error) { const ( diff --git a/toolkit/tools/imagegen/installutils/overlay.go b/toolkit/tools/imagegen/installutils/overlay.go new file mode 100644 index 0000000000..c0799f6a43 --- /dev/null +++ b/toolkit/tools/imagegen/installutils/overlay.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package installutils + +import ( + "fmt" + "os" + "path/filepath" + + "microsoft.com/pkggen/imagegen/diskutils" + "microsoft.com/pkggen/internal/logger" +) + +// Overlay Struct representing an overlay mount +type Overlay struct { + DevicePath string + lowerDir string + upperDir string + workDir string +} + +func (o Overlay) getMountArgs() string { + + return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", o.lowerDir, o.upperDir, o.workDir) +} + +func (o Overlay) setupFolders() (err error) { + err = os.MkdirAll(o.lowerDir, os.ModePerm) + if err != nil { + logger.Log.Errorf("Could not create directory (%s)", o.lowerDir) + return + } + err = os.MkdirAll(o.upperDir, os.ModePerm) + if err != nil { + logger.Log.Errorf("Could not create directory (%s)", o.upperDir) + return + } + err = os.MkdirAll(o.workDir, os.ModePerm) + if err != nil { + logger.Log.Errorf("Could not create directory (%s)", o.workDir) + return + } + err = mount(o.lowerDir, o.DevicePath, "", "") + if err != nil { + logger.Log.Errorf("Could not mount %s to %s", o.DevicePath, o.lowerDir) + } + return +} + +func (o Overlay) getUpperDir() (upperDir string) { + return o.upperDir +} + +func (o Overlay) unmount() (err error) { + err = umount(o.lowerDir) + if err != nil { + logger.Log.Warnf("Unmount of loopback(%s) failed. Still continuing", o.lowerDir) + } + + temperr := diskutils.DetachLoopbackDevice(o.DevicePath) + if temperr != nil { + logger.Log.Warnf("Losetup of loopback(%s) failed. Still continuing", o.lowerDir) + err = temperr + } + return +} + +// NewOverlay Creates the overlay struct +func NewOverlay(devicePath string) Overlay { + _, deviceName := filepath.Split(devicePath) + lowerDir := "/lowerdir" + deviceName + upperDir := "/upperdir" + deviceName + workDir := "/workdir" + deviceName + + o := Overlay{ + devicePath, + lowerDir, + upperDir, + workDir, + } + + return o +} diff --git a/toolkit/tools/imager/imager.go b/toolkit/tools/imager/imager.go index c29f71421f..23afa9475c 100644 --- a/toolkit/tools/imager/imager.go +++ b/toolkit/tools/imager/imager.go @@ -91,17 +91,18 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur ) var ( - isRootFS bool - isLoopDevice bool - isOfflineInstall bool - diskDevPath string - kernelPkg string - encryptedRoot diskutils.EncryptedRootDevice - readOnlyRoot diskutils.VerityDevice - partIDToDevPathMap map[string]string - partIDToFsTypeMap map[string]string - extraMountPoints []*safechroot.MountPoint - extraDirectories []string + isRootFS bool + isLoopDevice bool + isOfflineInstall bool + diskDevPath string + kernelPkg string + encryptedRoot diskutils.EncryptedRootDevice + readOnlyRoot diskutils.VerityDevice + partIDToDevPathMap map[string]string + partIDToFsTypeMap map[string]string + mountPointToOverlayMap map[string]*installutils.Overlay + extraMountPoints []*safechroot.MountPoint + extraDirectories []string ) // Get list of packages to install into image @@ -167,8 +168,19 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur packagesToInstall = append([]string{kernelPkg}, packagesToInstall...) } + setupChrootDir := filepath.Join(buildDir, setupRoot) + // Create Parition to Mountpoint map - mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap := installutils.CreateMountPointPartitionMap(partIDToDevPathMap, partIDToFsTypeMap, systemConfig) + mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, diffDiskBuild := installutils.CreateMountPointPartitionMap(partIDToDevPathMap, partIDToFsTypeMap, systemConfig) + if diffDiskBuild { + mountPointToOverlayMap, err = installutils.UpdatePartitionMapWithOverlays(partIDToDevPathMap, partIDToFsTypeMap, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, systemConfig) + // Schedule unmount of overlays after the upper layers are unmounted. + defer installutils.OverlayUnmount(mountPointToOverlayMap) + if err != nil { + logger.Log.Error("Failed to create the partition map") + return + } + } if isOfflineInstall { // Create setup chroot @@ -179,7 +191,6 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur } extraMountPoints = append(extraMountPoints, additionalExtraMountPoints...) - setupChrootDir := filepath.Join(buildDir, setupRoot) setupChroot := safechroot.NewChroot(setupChrootDir, existingChrootDir) err = setupChroot.Initialize(*tdnfTar, extraDirectories, extraMountPoints) if err != nil { @@ -197,7 +208,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur } err = setupChroot.Run(func() error { - return buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot) + return buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot, diffDiskBuild) }) if err != nil { logger.Log.Error("Failed to build image") @@ -211,7 +222,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur } // Create any partition-based artifacts - err = installutils.ExtractPartitionArtifacts(outputDir, defaultDiskIndex, disks[defaultDiskIndex], partIDToDevPathMap) + err = installutils.ExtractPartitionArtifacts(setupChrootDir, outputDir, defaultDiskIndex, disks[defaultDiskIndex], systemConfig, partIDToDevPathMap, mountPointToOverlayMap) if err != nil { return } @@ -229,7 +240,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur } } } else { - err = buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot) + err = buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot, diffDiskBuild) if err != nil { logger.Log.Error("Failed to build image") return @@ -440,8 +451,7 @@ func cleanupExtraFilesInChroot(chroot *safechroot.Chroot) (err error) { }) return } - -func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, packagesToInstall []string, systemConfig configuration.SystemConfig, diskDevPath string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, readOnlyRoot diskutils.VerityDevice) (err error) { +func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, mountPointToOverlayMap map[string]*installutils.Overlay, packagesToInstall []string, systemConfig configuration.SystemConfig, diskDevPath string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, readOnlyRoot diskutils.VerityDevice, diffDiskBuild bool) (err error) { const ( installRoot = "/installroot" verityWorkingDir = "verityworkingdir" @@ -455,12 +465,12 @@ func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap m // Only invoke CreateInstallRoot for a raw disk. This call will result in mount points being created from a raw disk // into the install root. A rootfs will not have these. if !isRootFS { - installMap, err = installutils.CreateInstallRoot(installRoot, mountPointMap, mountPointToMountArgsMap) + installMap, err = installutils.CreateInstallRoot(installRoot, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap) if err != nil { err = fmt.Errorf("failed to create install root: %s", err) return } - defer installutils.DestroyInstallRoot(installRoot, installMap) + defer installutils.DestroyInstallRoot(installRoot, installMap, mountPointToOverlayMap) } if systemConfig.ReadOnlyVerityRoot.Enable { @@ -488,7 +498,7 @@ func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap m defer installChroot.Close(leaveChrootOnDisk) // Populate image contents - err = installutils.PopulateInstallRoot(installChroot, packagesToInstall, systemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap, isRootFS, encryptedRoot) + err = installutils.PopulateInstallRoot(installChroot, packagesToInstall, systemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap, isRootFS, encryptedRoot, diffDiskBuild) if err != nil { err = fmt.Errorf("failed to populate image contents: %s", err) return @@ -540,6 +550,11 @@ func configureDiskBootloader(systemConfig configuration.SystemConfig, installChr // If we do not have a seperate boot partition we will need to add a prefix to all paths used in the configs. bootPrefix = "/boot" } + + if installMap[rootMountPoint] == installutils.NullDevice { + // In case of overlay device being mounted at root, no need to change the bootloader. + return + } bootUUID, err := installutils.GetUUID(bootDevice) if err != nil { err = fmt.Errorf("failed to get UUID: %s", err) diff --git a/toolkit/tools/roast/formats/diff.go b/toolkit/tools/roast/formats/diff.go new file mode 100644 index 0000000000..a19286da06 --- /dev/null +++ b/toolkit/tools/roast/formats/diff.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package formats + +import ( + "fmt" + + "microsoft.com/pkggen/internal/file" +) + +// DiffType represents the diff file system format +const DiffType = "diff" + +// Diff implements Converter interface for Diff partitions +type Diff struct { +} + +// Convert simply makes a copy of the RAW image and renames the extension to .diff +func (e *Diff) Convert(input, output string, isInputFile bool) (err error) { + if !isInputFile { + return fmt.Errorf("overlay diff conversion requires a RAW file as an input") + } + err = file.Copy(input, output) + return +} + +// Extension returns the filetype extension produced by this converter. +func (e *Diff) Extension() string { + return DiffType +} + +// NewDiff returns a new xz format encoder +func NewDiff() *Diff { + return &Diff{} +} diff --git a/toolkit/tools/roast/formats/rdiff.go b/toolkit/tools/roast/formats/rdiff.go new file mode 100644 index 0000000000..e1ea50381e --- /dev/null +++ b/toolkit/tools/roast/formats/rdiff.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package formats + +import ( + "fmt" + + "microsoft.com/pkggen/internal/file" +) + +// RdiffType represents the rdiff file system format +const RdiffType = "rdiff" + +// Rdiff implements Converter interface for Rdiff partitions +type Rdiff struct { +} + +// Convert simply makes a copy of the RAW image and renames the extension to .diff +func (e *Rdiff) Convert(input, output string, isInputFile bool) (err error) { + if !isInputFile { + return fmt.Errorf("rdiff conversion requires a RAW file as an input") + } + err = file.Copy(input, output) + return +} + +// Extension returns the filetype extension produced by this converter. +func (e *Rdiff) Extension() string { + return RdiffType +} + +// NewRdiff returns a new xz format encoder +func NewRdiff() *Rdiff { + return &Rdiff{} +} diff --git a/toolkit/tools/roast/roast.go b/toolkit/tools/roast/roast.go index f597b3b53e..573b58b544 100644 --- a/toolkit/tools/roast/roast.go +++ b/toolkit/tools/roast/roast.go @@ -93,6 +93,8 @@ func main() { } func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag, tmpDir string, config configuration.Config) (err error) { + const defaultSystemConfig = 0 + err = os.MkdirAll(tmpDir, os.ModePerm) if err != nil { return @@ -133,7 +135,8 @@ func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag for j, partition := range disk.Partitions { for _, artifact := range partition.Artifacts { - inputName, isFile := partitionArtifactInput(i, j) + // Currently only process 1 system config + inputName, isFile := partitionArtifactInput(i, j, retrievePartitionSettings(&config.SystemConfigs[defaultSystemConfig], partition.ID)) convertRequests <- &convertRequest{ inputPath: filepath.Join(inDir, inputName), isInputFile: isFile, @@ -162,6 +165,17 @@ func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag return } +func retrievePartitionSettings(systemConfig *configuration.SystemConfig, searchedID string) (foundSetting *configuration.PartitionSetting) { + for i := range systemConfig.PartitionSettings { + if systemConfig.PartitionSettings[i].ID == searchedID { + foundSetting = &systemConfig.PartitionSettings[i] + return + } + } + logger.Log.Warningf("Couldn't find partition setting '%s' under system config '%s'", searchedID, systemConfig.Name) + return +} + func artifactConverterWorker(convertRequests chan *convertRequest, convertedResults chan *convertResult, releaseVersion, tmpDir, imageTag, outDir string) { const ( initrdArtifactType = "initrd" @@ -259,6 +273,10 @@ func converterFactory(formatType string) (converter formats.Converter, err error converter = formats.NewRaw() case formats.Ext4Type: converter = formats.NewExt4() + case formats.DiffType: + converter = formats.NewDiff() + case formats.RdiffType: + converter = formats.NewRdiff() case formats.GzipType: converter = formats.NewGzip() case formats.TarGzipType: @@ -298,9 +316,15 @@ func diskArtifactInput(diskIndex int, disk configuration.Disk) (input string, is return } -func partitionArtifactInput(diskIndex, partitionIndex int) (input string, isFile bool) { +func partitionArtifactInput(diskIndex, partitionIndex int, partitionSetting *configuration.PartitionSetting) (input string, isFile bool) { // Currently all file artifacts have a raw file for input - input = fmt.Sprintf("disk%d.partition%d.raw", diskIndex, partitionIndex) + if partitionSetting.OverlayBaseImage != "" { + input = fmt.Sprintf("disk%d.partition%d.diff", diskIndex, partitionIndex) + } else if partitionSetting.RdiffBaseImage != "" { + input = fmt.Sprintf("disk%d.partition%d.rdiff", diskIndex, partitionIndex) + } else { + input = fmt.Sprintf("disk%d.partition%d.raw", diskIndex, partitionIndex) + } isFile = true return }