Overlay based diff image creation cherry pick (#611)
* Overlay based diff image creation prototype Here is a link to the spec https://microsoft-my.sharepoint.com/:w:/g/personal/arvindka_microsoft_com1/ESrYHTpWUPBOgdi7LjDsE14Bf1mHSLG702551XctkFX1mA?e=CyCc2j. This is for early feedback on the approach. It introduces a new element, BaseImage for each partition. Instead of creating a complete new partition image, a new diff layer is created using overlay file system. Overlay file system is a simple implementation of union file system. The changes files are completely copied in the upper level overlay. The implementation then copies the higher level files in a tgz. This tgz can be transferred to the ADU agent which first rehydrates the base image and then uses SWUpdate to do the A/B switch. Co-authored-by: Arvind Kandhare <arvindka@microsoft.com>
This commit is contained in:
arvindkandhare
GitHub
Родитель
fad9eb35df
Коммит
7bd75d547d
|
|
@ -102,6 +102,36 @@ A sample PartitionSettings entry, designating an EFI and a root partitions:
|
|||
],
|
||||
```
|
||||
|
||||
It is possible to use `PartitionSettings` to configure diff disk image creation. Two types of diffs are possible.
|
||||
`rdiff` and `overlay` diff.
|
||||
|
||||
For small and deterministic images `rdiff` is a better algorithm.
|
||||
For large images based on `ext4` `overlay` diff is a better algorithm.
|
||||
|
||||
A sample `ParitionSettings` entry using `rdiff` algorithm:
|
||||
|
||||
``` json
|
||||
{
|
||||
"ID": "boot",
|
||||
"MountPoint": "/boot/efi",
|
||||
"MountOptions" : "umask=0077",
|
||||
"RdiffBaseImage" : "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4"
|
||||
},
|
||||
```
|
||||
|
||||
A sample `ParitionSettings` entry using `overlay` algorithm:
|
||||
|
||||
``` json
|
||||
{
|
||||
"ID": "rootfs",
|
||||
"MountPoint": "/",
|
||||
"OverlayBaseImage" : "../out/images/core-efi/core-efi-rootfs-1.0.20200918.1751.ext4"
|
||||
}
|
||||
|
||||
```
|
||||
`RdiffBaseImage` represents the base image when `rdiff` algorithm is used.
|
||||
`OverlayBaseImage` represents the base image when `overlay` algorithm is used.
|
||||
|
||||
### PackageLists
|
||||
|
||||
PackageLists key consists of an array of relative paths to the package lists (JSON files).
|
||||
|
|
@ -301,11 +331,13 @@ A sample image configuration, producing a VHDX disk image:
|
|||
{
|
||||
"ID": "boot",
|
||||
"MountPoint": "/boot/efi",
|
||||
"MountOptions" : "umask=0077"
|
||||
"MountOptions" : "umask=0077",
|
||||
"RdiffBaseImage" : "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4"
|
||||
},
|
||||
{
|
||||
"ID": "rootfs",
|
||||
"MountPoint": "/"
|
||||
"MountPoint": "/",
|
||||
"OverlayBaseImage" : "../out/images/core-efi/core-efi-rootfs-1.0.20200918.1751.ext4"
|
||||
}
|
||||
],
|
||||
"PackageLists": [
|
||||
|
|
|
|||
|
|
@ -268,15 +268,17 @@ var expectedConfiguration Config = Config{
|
|||
Name: "SmallerDisk",
|
||||
IsDefault: true,
|
||||
PartitionSettings: []PartitionSetting{
|
||||
{
|
||||
ID: "MyBoot",
|
||||
MountPoint: "/boot",
|
||||
MountOptions: "ro,exec",
|
||||
PartitionSetting{
|
||||
ID: "MyBoot",
|
||||
MountPoint: "/boot",
|
||||
MountOptions: "ro,exec",
|
||||
RdiffBaseImage: "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4",
|
||||
},
|
||||
{
|
||||
ID: "MyRootfs",
|
||||
MountPoint: "/",
|
||||
RemoveDocs: true,
|
||||
PartitionSetting{
|
||||
ID: "MyRootfs",
|
||||
MountPoint: "/",
|
||||
RemoveDocs: true,
|
||||
OverlayBaseImage: "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4",
|
||||
},
|
||||
},
|
||||
PackageLists: []string{
|
||||
|
|
|
|||
|
|
@ -12,10 +12,12 @@ import (
|
|||
|
||||
// PartitionSetting holds the mounting information for each partition.
|
||||
type PartitionSetting struct {
|
||||
RemoveDocs bool `json:"RemoveDocs"`
|
||||
ID string `json:"ID"`
|
||||
MountOptions string `json:"MountOptions"`
|
||||
MountPoint string `json:"MountPoint"`
|
||||
RemoveDocs bool `json:"RemoveDocs"`
|
||||
ID string `json:"ID"`
|
||||
MountOptions string `json:"MountOptions"`
|
||||
MountPoint string `json:"MountPoint"`
|
||||
OverlayBaseImage string `json:"OverlayBaseImage"`
|
||||
RdiffBaseImage string `json:"RdiffBaseImage"`
|
||||
}
|
||||
|
||||
// IsValid returns an error if the PartitionSetting is not valid
|
||||
|
|
|
|||
|
|
@ -111,12 +111,14 @@
|
|||
{
|
||||
"ID": "MyBoot",
|
||||
"MountPoint": "/boot",
|
||||
"MountOptions": "ro,exec"
|
||||
"MountOptions": "ro,exec",
|
||||
"RdiffBaseImage": "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4"
|
||||
},
|
||||
{
|
||||
"ID": "MyRootfs",
|
||||
"MountPoint": "/",
|
||||
"RemoveDocs": true
|
||||
"RemoveDocs": true,
|
||||
"OverlayBaseImage": "../out/images/core-efi/core-efi-1.0.20200918.1751.ext4"
|
||||
}
|
||||
],
|
||||
"PackageLists": [
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// NullDevice represents the /dev/null device used as a mount device for overlay images.
|
||||
NullDevice = "/dev/null"
|
||||
overlay = "overlay"
|
||||
rootMountPoint = "/"
|
||||
rootUser = "root"
|
||||
|
||||
|
|
@ -53,7 +56,8 @@ type PackageList struct {
|
|||
// - mountPointDevPathMap is a map of mountpoint to partition device path
|
||||
// - mountPointToFsTypeMap is a map of mountpoint to filesystem type
|
||||
// - mountPointToMountArgsMap is a map of mountpoint to mount arguments to be passed on a call to mount
|
||||
func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]string, config configuration.SystemConfig) (mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string) {
|
||||
// - diffDiskBuild is a flag that denotes whether this is a diffdisk build or not
|
||||
func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]string, config configuration.SystemConfig) (mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, diffDiskBuild bool) {
|
||||
mountPointDevPathMap = make(map[string]string)
|
||||
mountPointToFsTypeMap = make(map[string]string)
|
||||
mountPointToMountArgsMap = make(map[string]string)
|
||||
|
|
@ -63,9 +67,13 @@ func CreateMountPointPartitionMap(partDevPathMap, partIDToFsTypeMap map[string]s
|
|||
logger.Log.Tracef("%v[%v]", partitionSetting.ID, partitionSetting.MountPoint)
|
||||
partDevPath, ok := partDevPathMap[partitionSetting.ID]
|
||||
if ok {
|
||||
mountPointDevPathMap[partitionSetting.MountPoint] = partDevPath
|
||||
mountPointToFsTypeMap[partitionSetting.MountPoint] = partIDToFsTypeMap[partitionSetting.ID]
|
||||
mountPointToMountArgsMap[partitionSetting.MountPoint] = partitionSetting.MountOptions
|
||||
if partitionSetting.OverlayBaseImage == "" {
|
||||
mountPointDevPathMap[partitionSetting.MountPoint] = partDevPath
|
||||
mountPointToFsTypeMap[partitionSetting.MountPoint] = partIDToFsTypeMap[partitionSetting.ID]
|
||||
mountPointToMountArgsMap[partitionSetting.MountPoint] = partitionSetting.MountOptions
|
||||
} else {
|
||||
diffDiskBuild = true
|
||||
}
|
||||
}
|
||||
logger.Log.Tracef("%v", mountPointDevPathMap)
|
||||
}
|
||||
|
|
@ -94,18 +102,69 @@ func sortMountPoints(mountPointMap *map[string]string, sortForUnmount bool) (rem
|
|||
// Reverse the sorting so we unmount in the opposite order
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(remainingMounts)))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UpdatePartitionMapWithOverlays Creates Overlay map and updates the partition map with required parameters.
|
||||
// - partDevPathMap is a map of partition IDs to partition device paths
|
||||
// - partIDToFsTypeMap is a map of partition IDs to filesystem type
|
||||
// - mountPointDevPathMap is a map of mountpoint to partition device path
|
||||
// - mountPointToFsTypeMap is a map of mountpoint to filesystem type
|
||||
// - mountPointToMountArgsMap is a map of mountpoint to mount arguments to be passed on a call to mount
|
||||
// - config is the SystemConfig from a config file
|
||||
// Output
|
||||
// - mountPointToOverlayMap is a map of mountpoint to overlay data
|
||||
func UpdatePartitionMapWithOverlays(partDevPathMap, partIDToFsTypeMap, mountPointDevPathMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, config configuration.SystemConfig) (mountPointToOverlayMap map[string]*Overlay, err error) {
|
||||
mountPointToOverlayMap = make(map[string]*Overlay)
|
||||
|
||||
// Go through each PartitionSetting
|
||||
for _, partitionSetting := range config.PartitionSettings {
|
||||
logger.Log.Tracef("%v[%v]", partitionSetting.ID, partitionSetting.MountPoint)
|
||||
if partitionSetting.OverlayBaseImage != "" {
|
||||
err = createOverlayPartition(partitionSetting, mountPointDevPathMap, mountPointToMountArgsMap, mountPointToFsTypeMap, mountPointToOverlayMap)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func createOverlayPartition(partitionSetting configuration.PartitionSetting, mountPointDevPathMap, mountPointToMountArgsMap, mountPointToFsTypeMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) {
|
||||
//Mount the base image
|
||||
//Create a temp upper dir
|
||||
//Add to the mount args
|
||||
devicePath, err := diskutils.SetupLoopbackDevice(partitionSetting.OverlayBaseImage)
|
||||
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Could not setup loop back device for mount (%s)", partitionSetting.OverlayBaseImage)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
overlayMount := NewOverlay(devicePath)
|
||||
|
||||
mountPointToOverlayMap[partitionSetting.MountPoint] = &overlayMount
|
||||
|
||||
// For overlays the device to be mounted is /dev/null. The actual device is synthesized from the lower and upper dir args
|
||||
// These args are passed to the mount command using -o
|
||||
mountPointDevPathMap[partitionSetting.MountPoint] = NullDevice
|
||||
mountPointToMountArgsMap[partitionSetting.MountPoint] = overlayMount.getMountArgs()
|
||||
mountPointToFsTypeMap[partitionSetting.MountPoint] = overlay
|
||||
return
|
||||
}
|
||||
|
||||
// CreateInstallRoot walks through the map of mountpoints and mounts the partitions into installroot
|
||||
// - installRoot is the destination path to mount these partitions
|
||||
// - mountPointMap is the map of mountpoint to partition device path
|
||||
func CreateInstallRoot(installRoot string, mountPointMap, mountPointToMountArgsMap map[string]string) (installMap map[string]string, err error) {
|
||||
// - mountPointToFsTypeMap is the map of mountpoint to the file type
|
||||
// - mountPointToMountArgsMap is the map of mountpoint to the parameters sent to
|
||||
// - mountPointToOverlayMap is the map of mountpoint to the overlay structure containing the base image
|
||||
func CreateInstallRoot(installRoot string, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (installMap map[string]string, err error) {
|
||||
installMap = make(map[string]string)
|
||||
for _, mountPoint := range sortMountPoints(&mountPointMap, false) {
|
||||
device := mountPointMap[mountPoint]
|
||||
err = mountSingleMountPoint(installRoot, mountPoint, device, mountPointToMountArgsMap[mountPoint])
|
||||
err = mountSingleMountPoint(installRoot, mountPoint, device, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint], mountPointToOverlayMap[mountPoint])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -117,7 +176,12 @@ func CreateInstallRoot(installRoot string, mountPointMap, mountPointToMountArgsM
|
|||
// DestroyInstallRoot unmounts each of the installroot mountpoints in order, ensuring that the root mountpoint is last
|
||||
// - installRoot is the path to the root where the mountpoints exist
|
||||
// - mountPointMap is the map of mountpoints to partition device paths
|
||||
func DestroyInstallRoot(installRoot string, mountPointMap map[string]string) (err error) {
|
||||
// - mountPointToOverlayMap is the map of mountpoints to overlay devices
|
||||
func DestroyInstallRoot(installRoot string, mountPointMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) {
|
||||
logger.Log.Trace("Destroying InstallRoot")
|
||||
|
||||
defer OverlayUnmount(mountPointToOverlayMap)
|
||||
|
||||
logger.Log.Trace("Destroying InstallRoot")
|
||||
// Reverse order for unmounting
|
||||
for _, mountPoint := range sortMountPoints(&mountPointMap, true) {
|
||||
|
|
@ -135,36 +199,62 @@ func DestroyInstallRoot(installRoot string, mountPointMap map[string]string) (er
|
|||
return
|
||||
}
|
||||
|
||||
func mountSingleMountPoint(installRoot, mountPoint, device, extraOptions string) (err error) {
|
||||
logger.Log.Debugf("Mounting %s to %s", device, mountPoint)
|
||||
// OverlayUnmount unmounts the overlay devices that are stored in the map, It ignores the errors and returns the last error.
|
||||
// - mountPointToOverlayMap is the map of mountpoints to overlay devices
|
||||
func OverlayUnmount(mountPointToOverlayMap map[string]*Overlay) (err error) {
|
||||
for _, overlay := range mountPointToOverlayMap {
|
||||
temperr := overlay.unmount()
|
||||
if temperr != nil {
|
||||
// Log a warning on error. Return the last error.
|
||||
logger.Log.Warnf("Failed to unmount the overlay (%+v). Continuing without unmounting: (%v)", overlay, temperr)
|
||||
err = temperr
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func mountSingleMountPoint(installRoot, mountPoint, device, fsType, extraOptions string, overlayDevice *Overlay) (err error) {
|
||||
mountPath := filepath.Join(installRoot, mountPoint)
|
||||
err = os.MkdirAll(mountPath, os.ModePerm)
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Failed to create mountpoint: %v", err)
|
||||
return
|
||||
}
|
||||
err = mount(mountPath, device, extraOptions)
|
||||
|
||||
if overlayDevice != nil {
|
||||
err = overlayDevice.setupFolders()
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Failed to create mount for overlay device: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
err = mount(mountPath, device, fsType, extraOptions)
|
||||
return
|
||||
}
|
||||
|
||||
func unmountSingleMountPoint(installRoot, mountPoint string) (err error) {
|
||||
|
||||
mountPath := filepath.Join(installRoot, mountPoint)
|
||||
err = umount(mountPath)
|
||||
return
|
||||
}
|
||||
|
||||
func mount(path, device, extraOptions string) (err error) {
|
||||
func mount(path, device, fsType, extraOptions string) (err error) {
|
||||
const squashErrors = false
|
||||
|
||||
if extraOptions == "" {
|
||||
err = shell.ExecuteLive(squashErrors, "mount", device, path)
|
||||
} else {
|
||||
err = shell.ExecuteLive(squashErrors, "mount", "-o", extraOptions, device, path)
|
||||
var mountArgs []string
|
||||
|
||||
if fsType != "" {
|
||||
mountArgs = append(mountArgs, "-t", fsType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
if extraOptions != "" {
|
||||
mountArgs = append(mountArgs, "-o", extraOptions)
|
||||
}
|
||||
|
||||
mountArgs = append(mountArgs, device, path)
|
||||
|
||||
err = shell.ExecuteLive(squashErrors, "mount", mountArgs...)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -268,7 +358,8 @@ func PackageNamesFromConfig(config configuration.Config) (packageList []*pkgjson
|
|||
// - mountPointToMountArgsMap is a map of mountpoints to mount options
|
||||
// - isRootFS specifies if the installroot is either backed by a directory (rootfs) or a raw disk
|
||||
// - encryptedRoot stores information about the encrypted root device if root encryption is enabled
|
||||
func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []string, config configuration.SystemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice) (err error) {
|
||||
//- diffDiskBuild is a flag that denotes whether this is a diffdisk build or not
|
||||
func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []string, config configuration.SystemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, diffDiskBuild bool) (err error) {
|
||||
const (
|
||||
filesystemPkg = "filesystem"
|
||||
)
|
||||
|
|
@ -280,7 +371,7 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s
|
|||
installRoot := filepath.Join(rootMountPoint, installChroot.RootDir())
|
||||
|
||||
// Initialize RPM Database so we can install RPMs into the installroot
|
||||
err = initializeRpmDatabase(installRoot)
|
||||
err = initializeRpmDatabase(installRoot, diffDiskBuild)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -315,7 +406,7 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s
|
|||
}
|
||||
|
||||
hostname := config.Hostname
|
||||
if !isRootFS {
|
||||
if !isRootFS && mountPointToFsTypeMap[rootMountPoint] != overlay {
|
||||
// Add /etc/hostname
|
||||
err = updateHostname(installChroot.RootDir(), hostname)
|
||||
if err != nil {
|
||||
|
|
@ -377,15 +468,21 @@ func PopulateInstallRoot(installChroot *safechroot.Chroot, packagesToInstall []s
|
|||
return
|
||||
}
|
||||
|
||||
func initializeRpmDatabase(installRoot string) (err error) {
|
||||
stdout, stderr, err := shell.Execute("rpm", "--root", installRoot, "--initdb")
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Failed to create rpm database: %v", err)
|
||||
logger.Log.Warn(stdout)
|
||||
logger.Log.Warn(stderr)
|
||||
return
|
||||
}
|
||||
func initializeRpmDatabase(installRoot string, diffDiskBuild bool) (err error) {
|
||||
if !diffDiskBuild {
|
||||
var (
|
||||
stdout string
|
||||
stderr string
|
||||
)
|
||||
|
||||
stdout, stderr, err = shell.Execute("rpm", "--root", installRoot, "--initdb")
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Failed to create rpm database: %v", err)
|
||||
logger.Log.Warn(stdout)
|
||||
logger.Log.Warn(stderr)
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = initializeTdnfConfiguration(installRoot)
|
||||
return
|
||||
}
|
||||
|
|
@ -653,7 +750,7 @@ func updateFstab(installRoot string, installMap, mountPointToFsTypeMap, mountPoi
|
|||
ReportAction("Configuring fstab")
|
||||
|
||||
for mountPoint, devicePath := range installMap {
|
||||
if mountPoint != "" {
|
||||
if mountPoint != "" && devicePath != NullDevice {
|
||||
err = addEntryToFstab(installRoot, mountPoint, devicePath, mountPointToFsTypeMap[mountPoint], mountPointToMountArgsMap[mountPoint])
|
||||
if err != nil {
|
||||
return
|
||||
|
|
@ -1495,6 +1592,7 @@ func runPostInstallScripts(installChroot *safechroot.Chroot, config configuratio
|
|||
logger.Log.Infof("Running post-install script: %s", script.Path)
|
||||
err = installChroot.UnsafeRun(func() error {
|
||||
err := shell.ExecuteLive(squashErrors, shell.ShellProgram, "-c", scriptPath, script.Args)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1711,31 +1809,79 @@ func setGrubCfgRootDevice(rootDevice, grubPath, luksUUID string) (err error) {
|
|||
}
|
||||
|
||||
// ExtractPartitionArtifacts scans through the SystemConfig and generates all the partition-based artifacts specified.
|
||||
// - setupChrootDirPath is the path to the setup root dir where the build takes place
|
||||
// - workDirPath is the directory to place the artifacts
|
||||
// - diskIndex is the index of the disk this is added to the parition artifact generated
|
||||
// - disk configuration settings for the disk
|
||||
// - systemConfig system configration corresponding to the disk configuration
|
||||
// - partIDToDevPathMap is a map of partition IDs to partition device paths
|
||||
func ExtractPartitionArtifacts(workDirPath string, diskIndex int, disk configuration.Disk, partIDToDevPathMap map[string]string) (err error) {
|
||||
// - mountPointToOverlayMap is a map of mountpoints to the overlay details for this mount if any
|
||||
func ExtractPartitionArtifacts(setupChrootDirPath, workDirPath string, diskIndex int, disk configuration.Disk, systemConfig configuration.SystemConfig, partIDToDevPathMap map[string]string, mountPointToOverlayMap map[string]*Overlay) (err error) {
|
||||
const (
|
||||
ext4ArtifactType = "ext4"
|
||||
ext4ArtifactType = "ext4"
|
||||
diffArtifactType = "diff"
|
||||
rdiffArtifactType = "rdiff"
|
||||
)
|
||||
|
||||
// Scan each partition for Artifacts
|
||||
for i, partition := range disk.Partitions {
|
||||
for _, artifact := range partition.Artifacts {
|
||||
if artifact.Type == ext4ArtifactType {
|
||||
devPath := partIDToDevPathMap[partition.ID]
|
||||
devPath := partIDToDevPathMap[partition.ID]
|
||||
|
||||
switch artifact.Type {
|
||||
case ext4ArtifactType:
|
||||
// Ext4 artifact type output is a .raw of the partition
|
||||
finalName := fmt.Sprintf("disk%d.partition%d.raw", diskIndex, i)
|
||||
err = createRawArtifact(workDirPath, devPath, finalName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case diffArtifactType:
|
||||
for _, setting := range systemConfig.PartitionSettings {
|
||||
if setting.ID == partition.ID {
|
||||
if setting.OverlayBaseImage != "" {
|
||||
// Diff artifact type output
|
||||
finalName := fmt.Sprintf("disk%d.partition%d.diff", diskIndex, i)
|
||||
err = createDiffArtifact(setupChrootDirPath, workDirPath, finalName, mountPointToOverlayMap[setting.MountPoint])
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
case rdiffArtifactType:
|
||||
for _, setting := range systemConfig.PartitionSettings {
|
||||
if setting.ID == partition.ID {
|
||||
if setting.RdiffBaseImage != "" {
|
||||
// Diff artifact type output
|
||||
finalName := fmt.Sprintf("disk%d.partition%d.rdiff", diskIndex, i)
|
||||
err = createRDiffArtifact(workDirPath, devPath, setting.RdiffBaseImage, finalName)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func createDiffArtifact(setupChrootDirPath, workDirPath, name string, overlay *Overlay) (err error) {
|
||||
const (
|
||||
squashErrors = true
|
||||
)
|
||||
|
||||
fullPath := filepath.Join(workDirPath, name)
|
||||
|
||||
upperDir := overlay.getUpperDir()
|
||||
upperDir = filepath.Join(setupChrootDirPath, upperDir)
|
||||
tarArgs := []string{
|
||||
"cvf",
|
||||
fullPath,
|
||||
"-C",
|
||||
upperDir,
|
||||
"."}
|
||||
|
||||
return shell.ExecuteLive(squashErrors, "tar", tarArgs...)
|
||||
}
|
||||
func createRawArtifact(workDirPath, devPath, name string) (err error) {
|
||||
const (
|
||||
defaultBlockSize = 1024 * 1024 // 1MB
|
||||
|
|
@ -1753,6 +1899,39 @@ func createRawArtifact(workDirPath, devPath, name string) (err error) {
|
|||
return shell.ExecuteLive(squashErrors, "dd", ddArgs...)
|
||||
}
|
||||
|
||||
func createRDiffArtifact(workDirPath, devPath, rDiffBaseImage, name string) (err error) {
|
||||
const (
|
||||
signatureFileName = "./signature"
|
||||
squashErrors = true
|
||||
)
|
||||
|
||||
fullPath := filepath.Join(workDirPath, name)
|
||||
|
||||
// rdiff expectes the signature file path to be relative.
|
||||
rdiffArgs := []string{
|
||||
"signature",
|
||||
rDiffBaseImage,
|
||||
signatureFileName,
|
||||
}
|
||||
|
||||
err = shell.ExecuteLive(squashErrors, "rdiff", rdiffArgs...)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
signatureFileFullPath := filepath.Join(workDirPath, signatureFileName)
|
||||
defer os.Remove(signatureFileFullPath)
|
||||
|
||||
rdiffArgs = []string{
|
||||
"delta",
|
||||
signatureFileName,
|
||||
devPath,
|
||||
fullPath,
|
||||
}
|
||||
|
||||
return shell.ExecuteLive(squashErrors, "rdiff", rdiffArgs...)
|
||||
}
|
||||
|
||||
// isRunningInHyperV checks if the program is running in a Hyper-V Virtual Machine.
|
||||
func isRunningInHyperV() (isHyperV bool, err error) {
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
package installutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"microsoft.com/pkggen/imagegen/diskutils"
|
||||
"microsoft.com/pkggen/internal/logger"
|
||||
)
|
||||
|
||||
// Overlay Struct representing an overlay mount
|
||||
type Overlay struct {
|
||||
DevicePath string
|
||||
lowerDir string
|
||||
upperDir string
|
||||
workDir string
|
||||
}
|
||||
|
||||
func (o Overlay) getMountArgs() string {
|
||||
|
||||
return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", o.lowerDir, o.upperDir, o.workDir)
|
||||
}
|
||||
|
||||
func (o Overlay) setupFolders() (err error) {
|
||||
err = os.MkdirAll(o.lowerDir, os.ModePerm)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Could not create directory (%s)", o.lowerDir)
|
||||
return
|
||||
}
|
||||
err = os.MkdirAll(o.upperDir, os.ModePerm)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Could not create directory (%s)", o.upperDir)
|
||||
return
|
||||
}
|
||||
err = os.MkdirAll(o.workDir, os.ModePerm)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Could not create directory (%s)", o.workDir)
|
||||
return
|
||||
}
|
||||
err = mount(o.lowerDir, o.DevicePath, "", "")
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Could not mount %s to %s", o.DevicePath, o.lowerDir)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (o Overlay) getUpperDir() (upperDir string) {
|
||||
return o.upperDir
|
||||
}
|
||||
|
||||
func (o Overlay) unmount() (err error) {
|
||||
err = umount(o.lowerDir)
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Unmount of loopback(%s) failed. Still continuing", o.lowerDir)
|
||||
}
|
||||
|
||||
temperr := diskutils.DetachLoopbackDevice(o.DevicePath)
|
||||
if temperr != nil {
|
||||
logger.Log.Warnf("Losetup of loopback(%s) failed. Still continuing", o.lowerDir)
|
||||
err = temperr
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewOverlay Creates the overlay struct
|
||||
func NewOverlay(devicePath string) Overlay {
|
||||
_, deviceName := filepath.Split(devicePath)
|
||||
lowerDir := "/lowerdir" + deviceName
|
||||
upperDir := "/upperdir" + deviceName
|
||||
workDir := "/workdir" + deviceName
|
||||
|
||||
o := Overlay{
|
||||
devicePath,
|
||||
lowerDir,
|
||||
upperDir,
|
||||
workDir,
|
||||
}
|
||||
|
||||
return o
|
||||
}
|
||||
|
|
@ -91,17 +91,18 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
)
|
||||
|
||||
var (
|
||||
isRootFS bool
|
||||
isLoopDevice bool
|
||||
isOfflineInstall bool
|
||||
diskDevPath string
|
||||
kernelPkg string
|
||||
encryptedRoot diskutils.EncryptedRootDevice
|
||||
readOnlyRoot diskutils.VerityDevice
|
||||
partIDToDevPathMap map[string]string
|
||||
partIDToFsTypeMap map[string]string
|
||||
extraMountPoints []*safechroot.MountPoint
|
||||
extraDirectories []string
|
||||
isRootFS bool
|
||||
isLoopDevice bool
|
||||
isOfflineInstall bool
|
||||
diskDevPath string
|
||||
kernelPkg string
|
||||
encryptedRoot diskutils.EncryptedRootDevice
|
||||
readOnlyRoot diskutils.VerityDevice
|
||||
partIDToDevPathMap map[string]string
|
||||
partIDToFsTypeMap map[string]string
|
||||
mountPointToOverlayMap map[string]*installutils.Overlay
|
||||
extraMountPoints []*safechroot.MountPoint
|
||||
extraDirectories []string
|
||||
)
|
||||
|
||||
// Get list of packages to install into image
|
||||
|
|
@ -167,8 +168,19 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
packagesToInstall = append([]string{kernelPkg}, packagesToInstall...)
|
||||
}
|
||||
|
||||
setupChrootDir := filepath.Join(buildDir, setupRoot)
|
||||
|
||||
// Create Parition to Mountpoint map
|
||||
mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap := installutils.CreateMountPointPartitionMap(partIDToDevPathMap, partIDToFsTypeMap, systemConfig)
|
||||
mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, diffDiskBuild := installutils.CreateMountPointPartitionMap(partIDToDevPathMap, partIDToFsTypeMap, systemConfig)
|
||||
if diffDiskBuild {
|
||||
mountPointToOverlayMap, err = installutils.UpdatePartitionMapWithOverlays(partIDToDevPathMap, partIDToFsTypeMap, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, systemConfig)
|
||||
// Schedule unmount of overlays after the upper layers are unmounted.
|
||||
defer installutils.OverlayUnmount(mountPointToOverlayMap)
|
||||
if err != nil {
|
||||
logger.Log.Error("Failed to create the partition map")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if isOfflineInstall {
|
||||
// Create setup chroot
|
||||
|
|
@ -179,7 +191,6 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
}
|
||||
extraMountPoints = append(extraMountPoints, additionalExtraMountPoints...)
|
||||
|
||||
setupChrootDir := filepath.Join(buildDir, setupRoot)
|
||||
setupChroot := safechroot.NewChroot(setupChrootDir, existingChrootDir)
|
||||
err = setupChroot.Initialize(*tdnfTar, extraDirectories, extraMountPoints)
|
||||
if err != nil {
|
||||
|
|
@ -197,7 +208,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
}
|
||||
|
||||
err = setupChroot.Run(func() error {
|
||||
return buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot)
|
||||
return buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot, diffDiskBuild)
|
||||
})
|
||||
if err != nil {
|
||||
logger.Log.Error("Failed to build image")
|
||||
|
|
@ -211,7 +222,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
}
|
||||
|
||||
// Create any partition-based artifacts
|
||||
err = installutils.ExtractPartitionArtifacts(outputDir, defaultDiskIndex, disks[defaultDiskIndex], partIDToDevPathMap)
|
||||
err = installutils.ExtractPartitionArtifacts(setupChrootDir, outputDir, defaultDiskIndex, disks[defaultDiskIndex], systemConfig, partIDToDevPathMap, mountPointToOverlayMap)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -229,7 +240,7 @@ func buildSystemConfig(systemConfig configuration.SystemConfig, disks []configur
|
|||
}
|
||||
}
|
||||
} else {
|
||||
err = buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot)
|
||||
err = buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap, packagesToInstall, systemConfig, diskDevPath, isRootFS, encryptedRoot, readOnlyRoot, diffDiskBuild)
|
||||
if err != nil {
|
||||
logger.Log.Error("Failed to build image")
|
||||
return
|
||||
|
|
@ -440,8 +451,7 @@ func cleanupExtraFilesInChroot(chroot *safechroot.Chroot) (err error) {
|
|||
})
|
||||
return
|
||||
}
|
||||
|
||||
func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, packagesToInstall []string, systemConfig configuration.SystemConfig, diskDevPath string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, readOnlyRoot diskutils.VerityDevice) (err error) {
|
||||
func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap map[string]string, mountPointToOverlayMap map[string]*installutils.Overlay, packagesToInstall []string, systemConfig configuration.SystemConfig, diskDevPath string, isRootFS bool, encryptedRoot diskutils.EncryptedRootDevice, readOnlyRoot diskutils.VerityDevice, diffDiskBuild bool) (err error) {
|
||||
const (
|
||||
installRoot = "/installroot"
|
||||
verityWorkingDir = "verityworkingdir"
|
||||
|
|
@ -455,12 +465,12 @@ func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap m
|
|||
// Only invoke CreateInstallRoot for a raw disk. This call will result in mount points being created from a raw disk
|
||||
// into the install root. A rootfs will not have these.
|
||||
if !isRootFS {
|
||||
installMap, err = installutils.CreateInstallRoot(installRoot, mountPointMap, mountPointToMountArgsMap)
|
||||
installMap, err = installutils.CreateInstallRoot(installRoot, mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap, mountPointToOverlayMap)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to create install root: %s", err)
|
||||
return
|
||||
}
|
||||
defer installutils.DestroyInstallRoot(installRoot, installMap)
|
||||
defer installutils.DestroyInstallRoot(installRoot, installMap, mountPointToOverlayMap)
|
||||
}
|
||||
|
||||
if systemConfig.ReadOnlyVerityRoot.Enable {
|
||||
|
|
@ -488,7 +498,7 @@ func buildImage(mountPointMap, mountPointToFsTypeMap, mountPointToMountArgsMap m
|
|||
defer installChroot.Close(leaveChrootOnDisk)
|
||||
|
||||
// Populate image contents
|
||||
err = installutils.PopulateInstallRoot(installChroot, packagesToInstall, systemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap, isRootFS, encryptedRoot)
|
||||
err = installutils.PopulateInstallRoot(installChroot, packagesToInstall, systemConfig, installMap, mountPointToFsTypeMap, mountPointToMountArgsMap, isRootFS, encryptedRoot, diffDiskBuild)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to populate image contents: %s", err)
|
||||
return
|
||||
|
|
@ -540,6 +550,11 @@ func configureDiskBootloader(systemConfig configuration.SystemConfig, installChr
|
|||
// If we do not have a seperate boot partition we will need to add a prefix to all paths used in the configs.
|
||||
bootPrefix = "/boot"
|
||||
}
|
||||
|
||||
if installMap[rootMountPoint] == installutils.NullDevice {
|
||||
// In case of overlay device being mounted at root, no need to change the bootloader.
|
||||
return
|
||||
}
|
||||
bootUUID, err := installutils.GetUUID(bootDevice)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("failed to get UUID: %s", err)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
package formats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"microsoft.com/pkggen/internal/file"
|
||||
)
|
||||
|
||||
// DiffType represents the diff file system format
|
||||
const DiffType = "diff"
|
||||
|
||||
// Diff implements Converter interface for Diff partitions
|
||||
type Diff struct {
|
||||
}
|
||||
|
||||
// Convert simply makes a copy of the RAW image and renames the extension to .diff
|
||||
func (e *Diff) Convert(input, output string, isInputFile bool) (err error) {
|
||||
if !isInputFile {
|
||||
return fmt.Errorf("overlay diff conversion requires a RAW file as an input")
|
||||
}
|
||||
err = file.Copy(input, output)
|
||||
return
|
||||
}
|
||||
|
||||
// Extension returns the filetype extension produced by this converter.
|
||||
func (e *Diff) Extension() string {
|
||||
return DiffType
|
||||
}
|
||||
|
||||
// NewDiff returns a new xz format encoder
|
||||
func NewDiff() *Diff {
|
||||
return &Diff{}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
package formats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"microsoft.com/pkggen/internal/file"
|
||||
)
|
||||
|
||||
// RdiffType represents the rdiff file system format
|
||||
const RdiffType = "rdiff"
|
||||
|
||||
// Rdiff implements Converter interface for Rdiff partitions
|
||||
type Rdiff struct {
|
||||
}
|
||||
|
||||
// Convert simply makes a copy of the RAW image and renames the extension to .diff
|
||||
func (e *Rdiff) Convert(input, output string, isInputFile bool) (err error) {
|
||||
if !isInputFile {
|
||||
return fmt.Errorf("rdiff conversion requires a RAW file as an input")
|
||||
}
|
||||
err = file.Copy(input, output)
|
||||
return
|
||||
}
|
||||
|
||||
// Extension returns the filetype extension produced by this converter.
|
||||
func (e *Rdiff) Extension() string {
|
||||
return RdiffType
|
||||
}
|
||||
|
||||
// NewRdiff returns a new xz format encoder
|
||||
func NewRdiff() *Rdiff {
|
||||
return &Rdiff{}
|
||||
}
|
||||
|
|
@ -93,6 +93,8 @@ func main() {
|
|||
}
|
||||
|
||||
func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag, tmpDir string, config configuration.Config) (err error) {
|
||||
const defaultSystemConfig = 0
|
||||
|
||||
err = os.MkdirAll(tmpDir, os.ModePerm)
|
||||
if err != nil {
|
||||
return
|
||||
|
|
@ -133,7 +135,8 @@ func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag
|
|||
|
||||
for j, partition := range disk.Partitions {
|
||||
for _, artifact := range partition.Artifacts {
|
||||
inputName, isFile := partitionArtifactInput(i, j)
|
||||
// Currently only process 1 system config
|
||||
inputName, isFile := partitionArtifactInput(i, j, retrievePartitionSettings(&config.SystemConfigs[defaultSystemConfig], partition.ID))
|
||||
convertRequests <- &convertRequest{
|
||||
inputPath: filepath.Join(inDir, inputName),
|
||||
isInputFile: isFile,
|
||||
|
|
@ -162,6 +165,17 @@ func generateImageArtifacts(workers int, inDir, outDir, releaseVersion, imageTag
|
|||
return
|
||||
}
|
||||
|
||||
func retrievePartitionSettings(systemConfig *configuration.SystemConfig, searchedID string) (foundSetting *configuration.PartitionSetting) {
|
||||
for i := range systemConfig.PartitionSettings {
|
||||
if systemConfig.PartitionSettings[i].ID == searchedID {
|
||||
foundSetting = &systemConfig.PartitionSettings[i]
|
||||
return
|
||||
}
|
||||
}
|
||||
logger.Log.Warningf("Couldn't find partition setting '%s' under system config '%s'", searchedID, systemConfig.Name)
|
||||
return
|
||||
}
|
||||
|
||||
func artifactConverterWorker(convertRequests chan *convertRequest, convertedResults chan *convertResult, releaseVersion, tmpDir, imageTag, outDir string) {
|
||||
const (
|
||||
initrdArtifactType = "initrd"
|
||||
|
|
@ -259,6 +273,10 @@ func converterFactory(formatType string) (converter formats.Converter, err error
|
|||
converter = formats.NewRaw()
|
||||
case formats.Ext4Type:
|
||||
converter = formats.NewExt4()
|
||||
case formats.DiffType:
|
||||
converter = formats.NewDiff()
|
||||
case formats.RdiffType:
|
||||
converter = formats.NewRdiff()
|
||||
case formats.GzipType:
|
||||
converter = formats.NewGzip()
|
||||
case formats.TarGzipType:
|
||||
|
|
@ -298,9 +316,15 @@ func diskArtifactInput(diskIndex int, disk configuration.Disk) (input string, is
|
|||
return
|
||||
}
|
||||
|
||||
func partitionArtifactInput(diskIndex, partitionIndex int) (input string, isFile bool) {
|
||||
func partitionArtifactInput(diskIndex, partitionIndex int, partitionSetting *configuration.PartitionSetting) (input string, isFile bool) {
|
||||
// Currently all file artifacts have a raw file for input
|
||||
input = fmt.Sprintf("disk%d.partition%d.raw", diskIndex, partitionIndex)
|
||||
if partitionSetting.OverlayBaseImage != "" {
|
||||
input = fmt.Sprintf("disk%d.partition%d.diff", diskIndex, partitionIndex)
|
||||
} else if partitionSetting.RdiffBaseImage != "" {
|
||||
input = fmt.Sprintf("disk%d.partition%d.rdiff", diskIndex, partitionIndex)
|
||||
} else {
|
||||
input = fmt.Sprintf("disk%d.partition%d.raw", diskIndex, partitionIndex)
|
||||
}
|
||||
isFile = true
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче