Add option to build a package for a specific architecture (#3565)
This change allows you to build a package for either arm64 for x86_64 by appending TARGET_ARCH to the build-packages make command. This change combined with the cross-toolchain allows developers to cross-build RPM packages.
This commit is contained in:
Dallas Delaney
GitHub
Родитель
0df9d872be
Коммит
abf6ba7cdf
|
|
@ -49,6 +49,7 @@ USE_PACKAGE_BUILD_CACHE ?= y
|
|||
REBUILD_DEP_CHAINS ?= y
|
||||
HYDRATED_BUILD ?= n
|
||||
DELTA_BUILD ?= n
|
||||
TARGET_ARCH ?=
|
||||
|
||||
# Folder defines
|
||||
toolkit_root := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
|
||||
|
|
|
|||
|
|
@ -688,6 +688,7 @@ To reproduce an ISO build, run the same make invocation as before, but set:
|
|||
| USE_PACKAGE_BUILD_CACHE | y | Skip building a package if it and its dependencies are already built.
|
||||
| NUM_OF_ANALYTICS_RESULTS | 10 | The number of entries to print when using the `graphanalytics` tool. If set to 0 this will print all available results.
|
||||
| REBUILD_DEP_CHAINS | y | Rebuild packages if their dependencies need to be built, even though the package has already been built.
|
||||
| TARGET_ARCH | | The architecture of the machine that will run the package binaries.
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,8 @@ $(specs_file): $(chroot_worker) $(BUILD_SPECS_DIR) $(build_specs) $(build_spec_d
|
|||
--worker-tar $(chroot_worker) \
|
||||
$(if $(filter y,$(RUN_CHECK)),--run-check) \
|
||||
$(logging_command) \
|
||||
--output $@
|
||||
$(if $(TARGET_ARCH),--target-arch="$(TARGET_ARCH)") \
|
||||
--output $@
|
||||
|
||||
# Convert the dependency information in the json file into a graph structure
|
||||
# We require all the toolchain RPMs to be available here to help resolve unfixable cyclic dependencies
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ package rpm
|
|||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
|
||||
|
|
@ -14,6 +15,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// TargetArgument specifies to build for a target platform (i.e., aarch64-mariner-linux)
|
||||
TargetArgument = "--target"
|
||||
|
||||
// BuildRequiresArgument specifies the build requires argument to be used with rpm tools
|
||||
BuildRequiresArgument = "--buildrequires"
|
||||
|
||||
|
|
@ -54,6 +58,20 @@ const (
|
|||
rpmBuildProgram = "rpmbuild"
|
||||
)
|
||||
|
||||
var goArchToRpmArch = map[string]string{
|
||||
"amd64": "x86_64",
|
||||
"arm64": "aarch64",
|
||||
}
|
||||
|
||||
// GetRpmArch converts the GOARCH arch into an RPM arch
|
||||
func GetRpmArch(goArch string) (rpmArch string, err error) {
|
||||
rpmArch, ok := goArchToRpmArch[goArch]
|
||||
if !ok {
|
||||
err = fmt.Errorf("Unknown GOARCH detected (%s)", goArch)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
// Output from 'rpm' prints installed RPMs in a line with the following format:
|
||||
//
|
||||
|
|
@ -164,13 +182,16 @@ func GetInstalledPackages() (result []string, err error) {
|
|||
}
|
||||
|
||||
// QuerySPEC queries a SPEC file with queryFormat. Returns the output split by line and trimmed.
|
||||
func QuerySPEC(specFile, sourceDir, queryFormat string, defines map[string]string, extraArgs ...string) (result []string, err error) {
|
||||
func QuerySPEC(specFile, sourceDir, queryFormat, arch string, defines map[string]string, extraArgs ...string) (result []string, err error) {
|
||||
const queryArg = "-q"
|
||||
|
||||
var allDefines map[string]string
|
||||
|
||||
extraArgs = append(extraArgs, queryArg)
|
||||
|
||||
// Apply --target arch argument
|
||||
extraArgs = append(extraArgs, TargetArgument, arch)
|
||||
|
||||
// To query some SPECs the source directory must be set
|
||||
// since the SPEC file may use `%include` on a source file
|
||||
if sourceDir == "" {
|
||||
|
|
@ -189,13 +210,13 @@ func QuerySPEC(specFile, sourceDir, queryFormat string, defines map[string]strin
|
|||
}
|
||||
|
||||
// QuerySPECForBuiltRPMs queries a SPEC file with queryFormat. Returns only the subpackages, which generate a .rpm file.
|
||||
func QuerySPECForBuiltRPMs(specFile, sourceDir string, defines map[string]string) (result []string, err error) {
|
||||
func QuerySPECForBuiltRPMs(specFile, sourceDir, arch string, defines map[string]string) (result []string, err error) {
|
||||
const (
|
||||
builtRPMsSwitch = "--builtrpms"
|
||||
queryFormat = ""
|
||||
)
|
||||
|
||||
return QuerySPEC(specFile, sourceDir, queryFormat, defines, builtRPMsSwitch)
|
||||
return QuerySPEC(specFile, sourceDir, queryFormat, arch, defines, builtRPMsSwitch)
|
||||
}
|
||||
|
||||
// QueryPackage queries an RPM or SRPM file with queryFormat. Returns the output split by line and trimmed.
|
||||
|
|
@ -209,14 +230,29 @@ func QueryPackage(packageFile, queryFormat string, defines map[string]string, ex
|
|||
}
|
||||
|
||||
// BuildRPMFromSRPM builds an RPM from the given SRPM file
|
||||
func BuildRPMFromSRPM(srpmFile string, defines map[string]string, extraArgs ...string) (err error) {
|
||||
func BuildRPMFromSRPM(srpmFile, outArch string, defines map[string]string, extraArgs ...string) (err error) {
|
||||
const (
|
||||
queryFormat = ""
|
||||
squashErrors = true
|
||||
vendor = "mariner"
|
||||
os = "linux"
|
||||
)
|
||||
|
||||
extraArgs = append(extraArgs, "--rebuild", "--nodeps")
|
||||
|
||||
// buildArch is the arch of the build machine
|
||||
// outArch is the arch of the machine that will run the resulting binary
|
||||
buildArch, err := GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if buildArch != outArch && "noarch" != outArch {
|
||||
tuple := outArch + "-" + vendor + "-" + os
|
||||
logger.Log.Debugf("Applying RPM target tuple (%s)", tuple)
|
||||
extraArgs = append(extraArgs, TargetArgument, tuple)
|
||||
}
|
||||
|
||||
args := formatCommandArgs(extraArgs, srpmFile, queryFormat, defines)
|
||||
return shell.ExecuteLive(squashErrors, rpmBuildProgram, args...)
|
||||
}
|
||||
|
|
@ -320,7 +356,7 @@ func ResolveCompetingPackages(rootDir string, rpmPaths ...string) (resolvedRPMs
|
|||
}
|
||||
|
||||
// SpecExclusiveArchIsCompatible verifies the "ExclusiveArch" tag is compatible with the current machine's architecture.
|
||||
func SpecExclusiveArchIsCompatible(specfile, sourcedir string, defines map[string]string) (isCompatible bool, err error) {
|
||||
func SpecExclusiveArchIsCompatible(specfile, sourcedir, arch string, defines map[string]string) (isCompatible bool, err error) {
|
||||
const queryExclusiveArch = "%{ARCH}\n[%{EXCLUSIVEARCH} ]\n"
|
||||
|
||||
const (
|
||||
|
|
@ -330,7 +366,7 @@ func SpecExclusiveArchIsCompatible(specfile, sourcedir string, defines map[strin
|
|||
)
|
||||
|
||||
// Sanity check that this SPEC is meant to be built for the current machine architecture
|
||||
exclusiveArchList, err := QuerySPEC(specfile, sourcedir, queryExclusiveArch, defines, QueryHeaderArgument)
|
||||
exclusiveArchList, err := QuerySPEC(specfile, sourcedir, queryExclusiveArch, arch, defines, QueryHeaderArgument)
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Failed to query SPEC (%s), error: %s", specfile, err)
|
||||
return
|
||||
|
|
@ -351,7 +387,7 @@ func SpecExclusiveArchIsCompatible(specfile, sourcedir string, defines map[strin
|
|||
}
|
||||
|
||||
// SpecExcludeArchIsCompatible verifies the "ExcludeArch" tag is compatible with the current machine's architecture.
|
||||
func SpecExcludeArchIsCompatible(specfile, sourcedir string, defines map[string]string) (isCompatible bool, err error) {
|
||||
func SpecExcludeArchIsCompatible(specfile, sourcedir, arch string, defines map[string]string) (isCompatible bool, err error) {
|
||||
const queryExclusiveArch = "%{ARCH}\n[%{EXCLUDEARCH} ]\n"
|
||||
|
||||
const (
|
||||
|
|
@ -360,7 +396,7 @@ func SpecExcludeArchIsCompatible(specfile, sourcedir string, defines map[string]
|
|||
minimumFieldsCount = iota
|
||||
)
|
||||
|
||||
excludedArchList, err := QuerySPEC(specfile, sourcedir, queryExclusiveArch, defines, QueryHeaderArgument)
|
||||
excludedArchList, err := QuerySPEC(specfile, sourcedir, queryExclusiveArch, arch, defines, QueryHeaderArgument)
|
||||
if err != nil {
|
||||
logger.Log.Warnf("Failed to query SPEC (%s), error: %s", specfile, err)
|
||||
return
|
||||
|
|
@ -378,14 +414,14 @@ func SpecExcludeArchIsCompatible(specfile, sourcedir string, defines map[string]
|
|||
}
|
||||
|
||||
// SpecArchIsCompatible verifies the spec is buildable on the current machine's architecture.
|
||||
func SpecArchIsCompatible(specfile, sourcedir string, defines map[string]string) (isCompatible bool, err error) {
|
||||
isCompatible, err = SpecExclusiveArchIsCompatible(specfile, sourcedir, defines)
|
||||
func SpecArchIsCompatible(specfile, sourcedir, arch string, defines map[string]string) (isCompatible bool, err error) {
|
||||
isCompatible, err = SpecExclusiveArchIsCompatible(specfile, sourcedir, arch, defines)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if isCompatible {
|
||||
return SpecExcludeArchIsCompatible(specfile, sourcedir, defines)
|
||||
return SpecExcludeArchIsCompatible(specfile, sourcedir, arch, defines)
|
||||
}
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ package rpm
|
|||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/logger"
|
||||
|
|
@ -14,6 +15,8 @@ import (
|
|||
|
||||
const specsDir = "testdata"
|
||||
|
||||
var buildArch = goArchToRpmArch[runtime.GOARCH]
|
||||
|
||||
var defines = map[string]string{
|
||||
"dist": ".cmX",
|
||||
"with_check": "1",
|
||||
|
|
@ -27,7 +30,7 @@ func TestMain(m *testing.M) {
|
|||
func TestExclusiveArchCheckShouldSucceedForSupportedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "supported_unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -35,7 +38,7 @@ func TestExclusiveArchCheckShouldSucceedForSupportedArchitectures(t *testing.T)
|
|||
func TestExclusiveArchCheckShouldSucceedForNoExclusiveArch(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "no_exclusive_architecture.spec")
|
||||
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -43,7 +46,7 @@ func TestExclusiveArchCheckShouldSucceedForNoExclusiveArch(t *testing.T) {
|
|||
func TestExclusiveArchCheckShouldFailForUnsupportedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExclusiveArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, matches)
|
||||
}
|
||||
|
|
@ -51,7 +54,7 @@ func TestExclusiveArchCheckShouldFailForUnsupportedArchitectures(t *testing.T) {
|
|||
func TestExcludedArchCheckShouldSucceedForSupportedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "supported_unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -59,7 +62,7 @@ func TestExcludedArchCheckShouldSucceedForSupportedArchitectures(t *testing.T) {
|
|||
func TestExcludedArchShouldSucceedForNoExcludedArch(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "no_exclusive_architecture.spec")
|
||||
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -67,7 +70,7 @@ func TestExcludedArchShouldSucceedForNoExcludedArch(t *testing.T) {
|
|||
func TestExcludedArchShouldFailForExcludedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecExcludeArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, matches)
|
||||
}
|
||||
|
|
@ -75,7 +78,7 @@ func TestExcludedArchShouldFailForExcludedArchitectures(t *testing.T) {
|
|||
func TestArchCheckShouldSucceedForSupportedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "supported_unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -83,7 +86,7 @@ func TestArchCheckShouldSucceedForSupportedArchitectures(t *testing.T) {
|
|||
func TestArchShouldSucceedForNoExcludedArch(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "no_exclusive_architecture.spec")
|
||||
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, matches)
|
||||
}
|
||||
|
|
@ -91,7 +94,7 @@ func TestArchShouldSucceedForNoExcludedArch(t *testing.T) {
|
|||
func TestArchShouldFailForExcludedArchitectures(t *testing.T) {
|
||||
specFilePath := filepath.Join(specsDir, "unsupported_architectures.spec")
|
||||
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, defines)
|
||||
matches, err := SpecArchIsCompatible(specFilePath, specsDir, buildArch, defines)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, matches)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/file"
|
||||
"github.com/microsoft/CBL-Mariner/toolkit/tools/internal/jsonutils"
|
||||
|
|
@ -158,10 +159,15 @@ func (s *SnapshotGenerator) convertResultsToRepoContents(allBuiltRPMs []string)
|
|||
func (s *SnapshotGenerator) filterCompatibleSpecs(allSpecFilePaths []string, defines map[string]string) (specPaths []string, err error) {
|
||||
var specCompatible bool
|
||||
|
||||
buildArch, err := rpm.GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, specFilePath := range allSpecFilePaths {
|
||||
specDirPath := filepath.Dir(specFilePath)
|
||||
|
||||
specCompatible, err = rpm.SpecArchIsCompatible(specFilePath, specDirPath, defines)
|
||||
specCompatible, err = rpm.SpecArchIsCompatible(specFilePath, specDirPath, buildArch, defines)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Failed while querrying spec (%s). Error: %v.", specFilePath, err)
|
||||
return
|
||||
|
|
@ -237,12 +243,17 @@ func (s *SnapshotGenerator) initializeChroot(specsDirPath string) (err error) {
|
|||
func (s *SnapshotGenerator) readBuiltRPMs(specPaths []string, defines map[string]string) (allBuiltRPMs []string, err error) {
|
||||
var builtRPMs []string
|
||||
|
||||
buildArch, err := rpm.GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, specPath := range specPaths {
|
||||
logger.Log.Debugf("Parsing spec (%s).", specPath)
|
||||
|
||||
specDirPath := filepath.Dir(specPath)
|
||||
|
||||
builtRPMs, err = rpm.QuerySPECForBuiltRPMs(specPath, specDirPath, defines)
|
||||
builtRPMs, err = rpm.QuerySPECForBuiltRPMs(specPath, specDirPath, buildArch, defines)
|
||||
if err != nil {
|
||||
logger.Log.Errorf("Failed to query built RPMs from (%s). Error: %v.", specPath, err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ var (
|
|||
rpmmacrosFile = app.Flag("rpmmacros-file", "Optional file path to an rpmmacros file for rpmbuild to use").ExistingFile()
|
||||
runCheck = app.Flag("run-check", "Run the check during package build").Bool()
|
||||
packagesToInstall = app.Flag("install-package", "Filepaths to RPM packages that should be installed before building.").Strings()
|
||||
outArch = app.Flag("out-arch", "Architecture of resulting package").String()
|
||||
|
||||
logFile = exe.LogFileFlag(app)
|
||||
logLevel = exe.LogLevelFlag(app)
|
||||
|
|
@ -82,7 +83,7 @@ func main() {
|
|||
defines[rpm.DistroBuildNumberDefine] = *distroBuildNumber
|
||||
defines[rpm.MarinerModuleLdflagsDefine] = "-Wl,-dT,%{_topdir}/BUILD/module_info.ld"
|
||||
|
||||
builtRPMs, err := buildSRPMInChroot(chrootDir, rpmsDirAbsPath, *workerTar, *srpmFile, *repoFile, *rpmmacrosFile, defines, *noCleanup, *runCheck, *packagesToInstall)
|
||||
builtRPMs, err := buildSRPMInChroot(chrootDir, rpmsDirAbsPath, *workerTar, *srpmFile, *repoFile, *rpmmacrosFile, *outArch, defines, *noCleanup, *runCheck, *packagesToInstall)
|
||||
logger.PanicOnError(err, "Failed to build SRPM '%s'. For details see log file: %s .", *srpmFile, *logFile)
|
||||
|
||||
err = copySRPMToOutput(*srpmFile, srpmsDirAbsPath)
|
||||
|
|
@ -104,7 +105,7 @@ func copySRPMToOutput(srpmFilePath, srpmOutputDirPath string) (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
func buildSRPMInChroot(chrootDir, rpmDirPath, workerTar, srpmFile, repoFile, rpmmacrosFile string, defines map[string]string, noCleanup, runCheck bool, packagesToInstall []string) (builtRPMs []string, err error) {
|
||||
func buildSRPMInChroot(chrootDir, rpmDirPath, workerTar, srpmFile, repoFile, rpmmacrosFile, outArch string, defines map[string]string, noCleanup, runCheck bool, packagesToInstall []string) (builtRPMs []string, err error) {
|
||||
const (
|
||||
buildHeartbeatTimeout = 30 * time.Minute
|
||||
|
||||
|
|
@ -159,7 +160,7 @@ func buildSRPMInChroot(chrootDir, rpmDirPath, workerTar, srpmFile, repoFile, rpm
|
|||
}
|
||||
|
||||
err = chroot.Run(func() (err error) {
|
||||
return buildRPMFromSRPMInChroot(srpmFileInChroot, runCheck, defines, packagesToInstall)
|
||||
return buildRPMFromSRPMInChroot(srpmFileInChroot, outArch, runCheck, defines, packagesToInstall)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
|
|
@ -171,7 +172,7 @@ func buildSRPMInChroot(chrootDir, rpmDirPath, workerTar, srpmFile, repoFile, rpm
|
|||
return
|
||||
}
|
||||
|
||||
func buildRPMFromSRPMInChroot(srpmFile string, runCheck bool, defines map[string]string, packagesToInstall []string) (err error) {
|
||||
func buildRPMFromSRPMInChroot(srpmFile, outArch string, runCheck bool, defines map[string]string, packagesToInstall []string) (err error) {
|
||||
// Convert /localrpms into a repository that a package manager can use.
|
||||
err = rpmrepomanager.CreateRepo(chrootLocalRpmsDir)
|
||||
if err != nil {
|
||||
|
|
@ -195,9 +196,9 @@ func buildRPMFromSRPMInChroot(srpmFile string, runCheck bool, defines map[string
|
|||
|
||||
// Build the SRPM
|
||||
if runCheck {
|
||||
err = rpm.BuildRPMFromSRPM(srpmFile, defines)
|
||||
err = rpm.BuildRPMFromSRPM(srpmFile, outArch, defines)
|
||||
} else {
|
||||
err = rpm.BuildRPMFromSRPM(srpmFile, defines, "--nocheck")
|
||||
err = rpm.BuildRPMFromSRPM(srpmFile, outArch, defines, "--nocheck")
|
||||
}
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -34,8 +34,9 @@ func (c *ChrootAgent) Initialize(config *BuildAgentConfig) (err error) {
|
|||
// BuildPackage builds a given file and returns the output files or error.
|
||||
// - inputFile is the SRPM to build.
|
||||
// - logName is the file name to save the package build log to.
|
||||
// - outArch is the target architecture to build for.
|
||||
// - dependencies is a list of dependencies that need to be installed before building.
|
||||
func (c *ChrootAgent) BuildPackage(inputFile, logName string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
func (c *ChrootAgent) BuildPackage(inputFile, logName, outArch string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
// On success, pkgworker will print a comma-seperated list of all RPMs built to stdout.
|
||||
// This will be the last stdout line written.
|
||||
const delimiter = ","
|
||||
|
|
@ -52,7 +53,7 @@ func (c *ChrootAgent) BuildPackage(inputFile, logName string, dependencies []str
|
|||
logger.Log.Trace(lastStdoutLine)
|
||||
}
|
||||
|
||||
args := serializeChrootBuildAgentConfig(c.config, inputFile, logFile, dependencies)
|
||||
args := serializeChrootBuildAgentConfig(c.config, inputFile, logFile, outArch, dependencies)
|
||||
err = shell.ExecuteLiveWithCallback(onStdout, logger.Log.Trace, true, c.config.Program, args...)
|
||||
|
||||
if err == nil && lastStdoutLine != "" {
|
||||
|
|
@ -73,7 +74,7 @@ func (c *ChrootAgent) Close() (err error) {
|
|||
}
|
||||
|
||||
// serializeChrootBuildAgentConfig serializes a BuildAgentConfig into arguments usable by pkgworker.
|
||||
func serializeChrootBuildAgentConfig(config *BuildAgentConfig, inputFile, logFile string, dependencies []string) (serializedArgs []string) {
|
||||
func serializeChrootBuildAgentConfig(config *BuildAgentConfig, inputFile, logFile, outArch string, dependencies []string) (serializedArgs []string) {
|
||||
serializedArgs = []string{
|
||||
fmt.Sprintf("--input=%s", inputFile),
|
||||
fmt.Sprintf("--work-dir=%s", config.WorkDir),
|
||||
|
|
@ -87,6 +88,7 @@ func serializeChrootBuildAgentConfig(config *BuildAgentConfig, inputFile, logFil
|
|||
fmt.Sprintf("--distro-build-number=%s", config.DistroBuildNumber),
|
||||
fmt.Sprintf("--log-file=%s", logFile),
|
||||
fmt.Sprintf("--log-level=%s", config.LogLevel),
|
||||
fmt.Sprintf("--out-arch=%s", outArch),
|
||||
}
|
||||
|
||||
if config.RpmmacrosFile != "" {
|
||||
|
|
|
|||
|
|
@ -36,8 +36,9 @@ type BuildAgent interface {
|
|||
// BuildPackage builds a given file and returns the output files or error.
|
||||
// - inputFile is the SRPM to build.
|
||||
// - logName is the file name to save the package build log to.
|
||||
// - outArch is the machine architecture where the output binary will run
|
||||
// - dependencies is a list of dependencies that need to be installed before building.
|
||||
BuildPackage(inputFile, logName string, dependencies []string) ([]string, string, error)
|
||||
BuildPackage(inputFile, logName, outArch string, dependencies []string) ([]string, string, error)
|
||||
|
||||
// Config returns a copy of the agent's configuration.
|
||||
Config() BuildAgentConfig
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ func (t *TestAgent) Initialize(config *BuildAgentConfig) (err error) {
|
|||
}
|
||||
|
||||
// BuildPackage simply sleeps and then returns success for TestAgent.
|
||||
func (t *TestAgent) BuildPackage(inputFile, logName string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
func (t *TestAgent) BuildPackage(inputFile, logName, outArch string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
const sleepDuration = time.Second * 5
|
||||
time.Sleep(sleepDuration)
|
||||
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ func buildBuildNode(node *pkggraph.PkgNode, pkgGraph *pkggraph.PkgGraph, graphMu
|
|||
dependencies := getBuildDependencies(node, pkgGraph, graphMutex)
|
||||
|
||||
logger.Log.Infof("Building %s", baseSrpmName)
|
||||
builtFiles, logFile, err = buildSRPMFile(agent, buildAttempts, node.SrpmPath, dependencies)
|
||||
builtFiles, logFile, err = buildSRPMFile(agent, buildAttempts, node.SrpmPath, node.Architecture, dependencies)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -190,14 +190,14 @@ func getBuildDependencies(node *pkggraph.PkgNode, pkgGraph *pkggraph.PkgGraph, g
|
|||
}
|
||||
|
||||
// buildSRPMFile sends an SRPM to a build agent to build.
|
||||
func buildSRPMFile(agent buildagents.BuildAgent, buildAttempts int, srpmFile string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
func buildSRPMFile(agent buildagents.BuildAgent, buildAttempts int, srpmFile, outArch string, dependencies []string) (builtFiles []string, logFile string, err error) {
|
||||
const (
|
||||
retryDuration = time.Second
|
||||
)
|
||||
|
||||
logBaseName := filepath.Base(srpmFile) + ".log"
|
||||
err = retry.Run(func() (buildErr error) {
|
||||
builtFiles, logFile, buildErr = agent.BuildPackage(srpmFile, logBaseName, dependencies)
|
||||
builtFiles, logFile, buildErr = agent.BuildPackage(srpmFile, logBaseName, outArch, dependencies)
|
||||
return
|
||||
}, buildAttempts, retryDuration)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -38,18 +39,19 @@ type parseResult struct {
|
|||
}
|
||||
|
||||
var (
|
||||
app = kingpin.New("specreader", "A tool to parse spec dependencies into JSON")
|
||||
specsDir = exe.InputDirFlag(app, "Directory to scan for SPECS")
|
||||
output = exe.OutputFlag(app, "Output file to export the JSON")
|
||||
workers = app.Flag("workers", "Number of concurrent goroutines to parse with").Default(defaultWorkerCount).Int()
|
||||
buildDir = app.Flag("build-dir", "Directory to store temporary files while parsing.").String()
|
||||
srpmsDir = app.Flag("srpm-dir", "Directory containing SRPMs.").Required().ExistingDir()
|
||||
rpmsDir = app.Flag("rpm-dir", "Directory containing built RPMs.").Required().ExistingDir()
|
||||
distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String()
|
||||
workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz. If this argument is empty, specs will be parsed in the host environment.").ExistingFile()
|
||||
runCheck = app.Flag("run-check", "Whether or not to run the spec file's check section during package build.").Bool()
|
||||
logFile = exe.LogFileFlag(app)
|
||||
logLevel = exe.LogLevelFlag(app)
|
||||
app = kingpin.New("specreader", "A tool to parse spec dependencies into JSON")
|
||||
specsDir = exe.InputDirFlag(app, "Directory to scan for SPECS")
|
||||
output = exe.OutputFlag(app, "Output file to export the JSON")
|
||||
workers = app.Flag("workers", "Number of concurrent goroutines to parse with").Default(defaultWorkerCount).Int()
|
||||
buildDir = app.Flag("build-dir", "Directory to store temporary files while parsing.").String()
|
||||
srpmsDir = app.Flag("srpm-dir", "Directory containing SRPMs.").Required().ExistingDir()
|
||||
rpmsDir = app.Flag("rpm-dir", "Directory containing built RPMs.").Required().ExistingDir()
|
||||
distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String()
|
||||
workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz. If this argument is empty, specs will be parsed in the host environment.").ExistingFile()
|
||||
targetArch = app.Flag("target-arch", "The architecture of the machine the RPM binaries run on").String()
|
||||
runCheck = app.Flag("run-check", "Whether or not to run the spec file's check section during package build.").Bool()
|
||||
logFile = exe.LogFileFlag(app)
|
||||
logLevel = exe.LogLevelFlag(app)
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
|
@ -82,9 +84,28 @@ func parseSPECsWrapper(buildDir, specsDir, rpmsDir, srpmsDir, distTag, outputFil
|
|||
defer chroot.Close(leaveFilesOnDisk)
|
||||
}
|
||||
|
||||
buildArch, err := rpm.GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
doParse := func() error {
|
||||
var parseError error
|
||||
packageRepo, parseError = parseSPECs(specsDir, rpmsDir, srpmsDir, distTag, workers, runCheck)
|
||||
|
||||
if *targetArch == "" {
|
||||
packageRepo, parseError = parseSPECs(specsDir, rpmsDir, srpmsDir, distTag, buildArch, workers, runCheck)
|
||||
if parseError != nil {
|
||||
err := fmt.Errorf("Failed to parse native specs (%w)", parseError)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
packageRepo, parseError = parseSPECs(specsDir, rpmsDir, srpmsDir, distTag, *targetArch, workers, runCheck)
|
||||
if parseError != nil {
|
||||
err := fmt.Errorf("Failed to parse cross specs (%w)", parseError)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return parseError
|
||||
}
|
||||
|
||||
|
|
@ -161,7 +182,7 @@ func createChroot(workerTar, buildDir, specsDir, srpmsDir string) (chroot *safec
|
|||
}
|
||||
|
||||
// parseSPECs will parse all specs in specsDir and return a summary of the SPECs.
|
||||
func parseSPECs(specsDir, rpmsDir, srpmsDir, distTag string, workers int, runCheck bool) (packageRepo *pkgjson.PackageRepo, err error) {
|
||||
func parseSPECs(specsDir, rpmsDir, srpmsDir, distTag, arch string, workers int, runCheck bool) (packageRepo *pkgjson.PackageRepo, err error) {
|
||||
var (
|
||||
packageList []*pkgjson.Package
|
||||
wg sync.WaitGroup
|
||||
|
|
@ -187,7 +208,7 @@ func parseSPECs(specsDir, rpmsDir, srpmsDir, distTag string, workers int, runChe
|
|||
// Start the workers now so they begin working as soon as a new job is buffered.
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go readSpecWorker(requests, results, cancel, &wg, distTag, rpmsDir, srpmsDir, runCheck)
|
||||
go readSpecWorker(requests, results, cancel, &wg, distTag, rpmsDir, srpmsDir, runCheck, arch)
|
||||
}
|
||||
|
||||
for _, specFile := range specFiles {
|
||||
|
|
@ -247,7 +268,7 @@ func sortPackages(packageRepo *pkgjson.PackageRepo) {
|
|||
// readspec is a goroutine that takes a full filepath to a spec file and scrapes it into the Specdef structure
|
||||
// Concurrency is limited by the size of the semaphore channel passed in. Too many goroutines at once can deplete
|
||||
// available filehandles.
|
||||
func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel <-chan struct{}, wg *sync.WaitGroup, distTag, rpmsDir, srpmsDir string, runCheck bool) {
|
||||
func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel <-chan struct{}, wg *sync.WaitGroup, distTag, rpmsDir, srpmsDir string, runCheck bool, arch string) {
|
||||
const (
|
||||
emptyQueryFormat = ``
|
||||
querySrpm = `%{NAME}-%{VERSION}-%{RELEASE}.src.rpm`
|
||||
|
|
@ -274,7 +295,7 @@ func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel
|
|||
sourcedir := filepath.Dir(specfile)
|
||||
|
||||
// Find the SRPM associated with the SPEC.
|
||||
srpmResults, err := rpm.QuerySPEC(specfile, sourcedir, querySrpm, defines, rpm.QueryHeaderArgument)
|
||||
srpmResults, err := rpm.QuerySPEC(specfile, sourcedir, querySrpm, arch, defines, rpm.QueryHeaderArgument)
|
||||
if err != nil {
|
||||
result.err = err
|
||||
results <- result
|
||||
|
|
@ -283,7 +304,7 @@ func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel
|
|||
|
||||
srpmPath := filepath.Join(srpmsDir, srpmResults[0])
|
||||
|
||||
isCompatible, err := rpm.SpecArchIsCompatible(specfile, sourcedir, defines)
|
||||
isCompatible, err := rpm.SpecArchIsCompatible(specfile, sourcedir, arch, defines)
|
||||
if err != nil {
|
||||
result.err = err
|
||||
results <- result
|
||||
|
|
@ -297,7 +318,7 @@ func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel
|
|||
}
|
||||
|
||||
// Find every package that the spec provides
|
||||
queryResults, err := rpm.QuerySPEC(specfile, sourcedir, queryProvidedPackages, defines, rpm.QueryBuiltRPMHeadersArgument)
|
||||
queryResults, err := rpm.QuerySPEC(specfile, sourcedir, queryProvidedPackages, arch, defines, rpm.QueryBuiltRPMHeadersArgument)
|
||||
if err == nil && len(queryResults) != 0 {
|
||||
providerList, err = parseProvides(rpmsDir, srpmPath, queryResults)
|
||||
if err != nil {
|
||||
|
|
@ -308,7 +329,7 @@ func readSpecWorker(requests <-chan string, results chan<- *parseResult, cancel
|
|||
}
|
||||
|
||||
// Query the BuildRequires fields from this spec and turn them into an array of PackageVersions
|
||||
queryResults, err = rpm.QuerySPEC(specfile, sourcedir, emptyQueryFormat, defines, rpm.BuildRequiresArgument)
|
||||
queryResults, err = rpm.QuerySPEC(specfile, sourcedir, emptyQueryFormat, arch, defines, rpm.BuildRequiresArgument)
|
||||
if err == nil && len(queryResults) != 0 {
|
||||
buildRequiresList, err = parsePackageVersionList(queryResults)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -399,10 +400,15 @@ func calculateSPECsToRepack(specFiles []string, distTag, outDir string, nestedSo
|
|||
|
||||
logger.Log.Infof("Calculating SPECs to repack")
|
||||
|
||||
arch, err := rpm.GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Start the workers now so they begin working as soon as a new job is buffered.
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go specsToPackWorker(requests, results, cancel, &wg, distTag, outDir, nestedSourcesDir, repackAll, runCheck)
|
||||
go specsToPackWorker(requests, results, cancel, &wg, distTag, outDir, arch, nestedSourcesDir, repackAll, runCheck)
|
||||
}
|
||||
|
||||
for _, specFile := range specFiles {
|
||||
|
|
@ -452,7 +458,7 @@ func calculateSPECsToRepack(specFiles []string, distTag, outDir string, nestedSo
|
|||
}
|
||||
|
||||
// specsToPackWorker will process a channel of spec files that should be checked if packing is needed.
|
||||
func specsToPackWorker(requests <-chan string, results chan<- *specState, cancel <-chan struct{}, wg *sync.WaitGroup, distTag, outDir string, nestedSourcesDir, repackAll, runCheck bool) {
|
||||
func specsToPackWorker(requests <-chan string, results chan<- *specState, cancel <-chan struct{}, wg *sync.WaitGroup, distTag, outDir string, arch string, nestedSourcesDir, repackAll, runCheck bool) {
|
||||
const (
|
||||
queryFormat = `%{NAME}-%{VERSION}-%{RELEASE}.src.rpm`
|
||||
nestedSourceDirName = "SOURCES"
|
||||
|
|
@ -489,7 +495,7 @@ func specsToPackWorker(requests <-chan string, results chan<- *specState, cancel
|
|||
if nestedSourcesDir {
|
||||
sourceDir = filepath.Join(sourceDir, nestedSourceDirName)
|
||||
}
|
||||
specQueryResults, err := rpm.QuerySPEC(specFile, sourceDir, queryFormat, defines, rpm.QueryHeaderArgument)
|
||||
specQueryResults, err := rpm.QuerySPEC(specFile, sourceDir, queryFormat, arch, defines, rpm.QueryHeaderArgument)
|
||||
|
||||
if err != nil {
|
||||
if err.Error() == rpm.NoCompatibleArchError {
|
||||
|
|
@ -520,7 +526,7 @@ func specsToPackWorker(requests <-chan string, results chan<- *specState, cancel
|
|||
}
|
||||
|
||||
// Sanity check that SRPMS is meant to be built for the machine architecture
|
||||
isCompatible, err := rpm.SpecArchIsCompatible(specFile, sourceDir, defines)
|
||||
isCompatible, err := rpm.SpecArchIsCompatible(specFile, sourceDir, arch, defines)
|
||||
if err != nil {
|
||||
result.err = err
|
||||
results <- result
|
||||
|
|
@ -796,9 +802,9 @@ func createRPMBuildFolderStructure(workingDir string) (err error) {
|
|||
|
||||
// readSPECTagArray will return an array of tag values from the given specfile.
|
||||
// (e.g. all SOURCE entries)
|
||||
func readSPECTagArray(specFile, sourceDir, tag string, defines map[string]string) (tagValues []string, err error) {
|
||||
func readSPECTagArray(specFile, sourceDir, tag string, arch string, defines map[string]string) (tagValues []string, err error) {
|
||||
queryFormat := fmt.Sprintf(`[%%{%s}\n]`, tag)
|
||||
return rpm.QuerySPEC(specFile, sourceDir, queryFormat, defines, rpm.QueryHeaderArgument)
|
||||
return rpm.QuerySPEC(specFile, sourceDir, queryFormat, arch, defines, rpm.QueryHeaderArgument)
|
||||
}
|
||||
|
||||
// hydrateFiles will attempt to retrieve all sources needed to build an SRPM from a SPEC.
|
||||
|
|
@ -837,8 +843,15 @@ func hydrateFiles(fileTypeToHydrate fileType, specFile, workingDir string, srcCo
|
|||
newSourceDir := filepath.Join(workingDir, srpmSOURCESDir)
|
||||
fileHydrationState := make(map[string]bool)
|
||||
|
||||
// Only consult the current build system's arch
|
||||
// We don't care about the target arch since SRPMs should be packaged in an architecture agnostic manner
|
||||
arch, err := rpm.GetRpmArch(runtime.GOARCH)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Collect a list of files of type `specTag` needed for this SRPM
|
||||
filesNeeded, err := readSPECTagArray(specFile, srcConfig.localSourceDir, specTag, defines)
|
||||
filesNeeded, err := readSPECTagArray(specFile, srcConfig.localSourceDir, specTag, arch, defines)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче