diff --git a/toolkit/resources/manifests/package/toolchain_aarch64.txt b/toolkit/resources/manifests/package/toolchain_aarch64.txt index d7a2b0e6ab..b0021c2f3d 100644 --- a/toolkit/resources/manifests/package/toolchain_aarch64.txt +++ b/toolkit/resources/manifests/package/toolchain_aarch64.txt @@ -235,8 +235,8 @@ mariner-repos-debug-2.0-8.cm2.noarch.rpm mariner-repos-debug-preview-2.0-8.cm2.noarch.rpm mariner-repos-extended-2.0-8.cm2.noarch.rpm mariner-repos-extended-debug-2.0-8.cm2.noarch.rpm -mariner-repos-extended-preview-2.0-8.cm2.noarch.rpm mariner-repos-extended-debug-preview-2.0-8.cm2.noarch.rpm +mariner-repos-extended-preview-2.0-8.cm2.noarch.rpm mariner-repos-extras-2.0-8.cm2.noarch.rpm mariner-repos-extras-preview-2.0-8.cm2.noarch.rpm mariner-repos-microsoft-2.0-8.cm2.noarch.rpm @@ -571,10 +571,10 @@ texinfo-debuginfo-6.8-1.cm2.aarch64.rpm unzip-6.0-20.cm2.aarch64.rpm unzip-debuginfo-6.0-20.cm2.aarch64.rpm util-linux-2.37.4-6.cm2.aarch64.rpm -util-linux-libs-2.37.4-6.cm2.aarch64.rpm util-linux-debuginfo-2.37.4-6.cm2.aarch64.rpm util-linux-devel-2.37.4-6.cm2.aarch64.rpm util-linux-lang-2.37.4-6.cm2.aarch64.rpm +util-linux-libs-2.37.4-6.cm2.aarch64.rpm which-2.21-8.cm2.aarch64.rpm which-debuginfo-2.21-8.cm2.aarch64.rpm xz-5.2.5-1.cm2.aarch64.rpm diff --git a/toolkit/resources/manifests/package/toolchain_x86_64.txt b/toolkit/resources/manifests/package/toolchain_x86_64.txt index 71ff1cb7ae..865cf640cf 100644 --- a/toolkit/resources/manifests/package/toolchain_x86_64.txt +++ b/toolkit/resources/manifests/package/toolchain_x86_64.txt @@ -235,8 +235,8 @@ mariner-repos-debug-2.0-8.cm2.noarch.rpm mariner-repos-debug-preview-2.0-8.cm2.noarch.rpm mariner-repos-extended-2.0-8.cm2.noarch.rpm mariner-repos-extended-debug-2.0-8.cm2.noarch.rpm -mariner-repos-extended-preview-2.0-8.cm2.noarch.rpm mariner-repos-extended-debug-preview-2.0-8.cm2.noarch.rpm +mariner-repos-extended-preview-2.0-8.cm2.noarch.rpm mariner-repos-extras-2.0-8.cm2.noarch.rpm mariner-repos-extras-preview-2.0-8.cm2.noarch.rpm mariner-repos-microsoft-2.0-8.cm2.noarch.rpm @@ -571,10 +571,10 @@ texinfo-debuginfo-6.8-1.cm2.x86_64.rpm unzip-6.0-20.cm2.x86_64.rpm unzip-debuginfo-6.0-20.cm2.x86_64.rpm util-linux-2.37.4-6.cm2.x86_64.rpm -util-linux-libs-2.37.4-6.cm2.x86_64.rpm util-linux-debuginfo-2.37.4-6.cm2.x86_64.rpm util-linux-devel-2.37.4-6.cm2.x86_64.rpm util-linux-lang-2.37.4-6.cm2.x86_64.rpm +util-linux-libs-2.37.4-6.cm2.x86_64.rpm which-2.21-8.cm2.x86_64.rpm which-debuginfo-2.21-8.cm2.x86_64.rpm xz-5.2.5-1.cm2.x86_64.rpm diff --git a/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh b/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh index 989789e14a..ec1a23e853 100755 --- a/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh +++ b/toolkit/scripts/toolchain/build_official_toolchain_rpms.sh @@ -28,12 +28,13 @@ source $(dirname $0)/../timestamp.sh begin_timestamp start_record_timestamp "prep_files" - MARINER_LOGS=$MARINER_BUILD_DIR/logs TOOLCHAIN_LOGS=$MARINER_LOGS/toolchain TOOLCHAIN_BUILD_LIST=$TOOLCHAIN_LOGS/build_list.txt +TOOLCHAIN_BUILT_RPMS_LIST=$TOOLCHAIN_LOGS/built_rpms_list.txt +TOOLCHAIN_BUILT_SPECS_LIST=$TOOLCHAIN_LOGS/built_specs_list.txt TOOLCHAIN_FAILURES=$TOOLCHAIN_LOGS/failures.txt -set -x +set -ex export LFS=$MARINER_BUILD_DIR/toolchain/populated_toolchain TOPDIR=/usr/src/mariner @@ -77,6 +78,21 @@ mkdir -pv $CHROOT_RPMS_DIR mkdir -pv $CHROOT_RPMS_DIR_ARCH mkdir -pv $CHROOT_RPMS_DIR_NOARCH +TEMP_DIR=$(mktemp -d -t) +TEMP_BUILT_RPMS_LIST="$(mktemp --tmpdir="$TEMP_DIR")" +TEMP_BUILT_SPECS_LIST="$(mktemp --tmpdir="$TEMP_DIR")" +function clean_up { + # Removing duplicates during clean-up to simplify appends during run-time. + echo "Copying build lists to log output..." + sort "$TEMP_BUILT_RPMS_LIST" | uniq > "$TOOLCHAIN_BUILT_RPMS_LIST" + sort "$TEMP_BUILT_SPECS_LIST" | uniq > "$TOOLCHAIN_BUILT_SPECS_LIST" + + echo "Cleaning up..." + chroot_unmount + rm -rf "$TEMP_DIR" +} +trap clean_up EXIT + # Remove artifacts from previous toolchain builds sudo rm -f $TOOLCHAIN_BUILD_LIST sudo rm -f $TOOLCHAIN_FAILURES @@ -112,7 +128,6 @@ fi stop_record_timestamp "hydrate" chroot_mount () { - trap chroot_unmount EXIT mount --bind /dev $LFS/dev mount -t devpts devpts $LFS/dev/pts -o gid=5,mode=620 mount -t proc proc $LFS/proc @@ -122,11 +137,15 @@ chroot_mount () { blocking_unmount () { # $1 mountpoint - umount -l $1 + if ! mountpoint -q "$1"; then + return + fi + + umount -l $1 || true while mountpoint -q $1; do echo $1 is still busy... sleep 1 - umount -l $1 + umount -l $1 || true done } @@ -137,7 +156,6 @@ chroot_unmount () { blocking_unmount $LFS/run blocking_unmount $LFS/proc blocking_unmount $LFS/sys - trap - EXIT } chroot_and_print_installed_rpms () { @@ -164,7 +182,7 @@ chroot_and_install_rpms () { # $1 = spec name (or rpm name if $2 is omitted) # $2 = qualified package name # Clean and then copy the RPM into the chroot directory for installation below - rm -v $CHROOT_INSTALL_RPM_DIR/* + rm -fv $CHROOT_INSTALL_RPM_DIR/* if [[ -n $2 ]]; then # If we're using the qualified package name, there's probably naming conflicts # that prevent us from simply globbing for RPMs with a prefix of the qualified name. @@ -230,17 +248,18 @@ chroot_and_run_rpmbuild () { # $INCREMENTAL_TOOLCHAIN is set to "y". build_rpm_in_chroot_no_install () { start_record_timestamp "build packages/build/$1" - # $1 = SRPM name + # $1 = spec name - # Find all the associated RPMs for the SRPM and check if they are in the chroot RPM directory specPath=$(find $SPECROOT -name "$1.spec" -print -quit) specDir=$(dirname $specPath) + rpmMacros=(-D "with_check 1" -D "_sourcedir $specDir" -D "dist $PARAM_DIST_TAG") + builtRpms="$(rpmspec -q $specPath --builtrpms "${rpmMacros[@]}" --queryformat="%{nvra}.rpm\n")" + # Find all the associated RPMs for the SRPM and check if they are in the chroot RPM directory foundAllRPMs="false" if [ "$INCREMENTAL_TOOLCHAIN" = "y" ]; then foundAllRPMs="true" - all_rpms=$(rpmspec -q $specPath --srpm --define="with_check 1" --define="_sourcedir $specDir" --define="dist $PARAM_DIST_TAG" --builtrpms --queryformat '%{nvra}.rpm ' | xargs) - for rpm in ${all_rpms}; do + for rpm in $builtRpms; do rpmPath=$(find $CHROOT_RPMS_DIR -name "$rpm" -print -quit) if [ -z "$rpmPath" ]; then echo "Did not find incremental toolchain rpm '$rpm' in '$CHROOT_RPMS_DIR', must rebuild." @@ -251,26 +270,35 @@ build_rpm_in_chroot_no_install () { fi done fi + if [ "$foundAllRPMs" = "false" ]; then echo only building RPM $1 within the chroot - srpmName=$(rpmspec -q $specPath --srpm --define="with_check 1" --define="_sourcedir $specDir" --define="dist $PARAM_DIST_TAG" --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) + srpmName=$(rpmspec -q $specPath --srpm "${rpmMacros[@]}" --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) srpmPath=$MARINER_INPUT_SRPMS_DIR/$srpmName cp $srpmPath $CHROOT_SRPMS_DIR chroot_and_run_rpmbuild $srpmName 2>&1 | awk '{ print strftime("time=\"%Y-%m-%dT%T%Z\""), $0; fflush(); }' | tee $TOOLCHAIN_LOGS/$srpmName.log - cp $CHROOT_RPMS_DIR_ARCH/$1* $FINISHED_RPM_DIR - cp $CHROOT_RPMS_DIR_NOARCH/$1* $FINISHED_RPM_DIR + copy_built_rpms $builtRpms cp $srpmPath $MARINER_OUTPUT_SRPMS_DIR + echo "$1" >> $TEMP_BUILT_SPECS_LIST echo NOT installing the package $srpmName fi + echo "$1" >> $TOOLCHAIN_BUILD_LIST stop_record_timestamp "build packages/build/$1" } -# Copy RPM subpackages that have a different prefix -copy_rpm_subpackage () { - echo cache $1 RPMS - cp $CHROOT_RPMS_DIR_ARCH/$1* $FINISHED_RPM_DIR - cp $CHROOT_RPMS_DIR_NOARCH/$1* $FINISHED_RPM_DIR +# Log the built RPMs and copy them to the finished RPMs directory. +copy_built_rpms () { + for builtRpm in "$@"; do + rpmPath="$(find "$CHROOT_RPMS_DIR" -name "$builtRpm" -print -quit)" + if [[ ! -f "$rpmPath" ]]; then + echo ERROR: could not find expected built RPM "$builtRpm" in "$CHROOT_RPMS_DIR". >&2 + return 1 + fi + + cp "$rpmPath" "$FINISHED_RPM_DIR" + echo "$builtRpm" >> "$TEMP_BUILT_RPMS_LIST" + done } start_record_timestamp "build prep" @@ -284,10 +312,8 @@ mkdir -pv $LFS/usr/lib/rpm/mariner cp -v $SPECROOT/mariner-rpm-macros/gen-ld-script.sh $LFS/usr/lib/rpm/mariner/gen-ld-script.sh cp -v $SPECROOT/mariner-rpm-macros/generate-package-note.py $LFS/usr/lib/rpm/mariner/generate-package-note.py cp -v $SPECROOT/mariner-rpm-macros/verify-package-notes.sh $LFS/usr/lib/rpm/mariner/verify-package-notes.sh -cp -v $SPECROOT/rpm/brp* $LFS/usr/lib/rpm mkdir -pv $LFS/usr/lib/rpm/macros.d cp -v $MARINER_TOOLCHAIN_MANIFESTS_DIR/macros.override $LFS/usr/lib/rpm/macros.d/macros.override -chmod +x $LFS/usr/lib/rpm/brp* cp /etc/resolv.conf $LFS/etc/ chroot_and_print_installed_rpms @@ -299,7 +325,6 @@ start_record_timestamp "build packages/install" echo Building final list of toolchain RPMs build_rpm_in_chroot_no_install mariner-rpm-macros -copy_rpm_subpackage mariner-check-macros chroot_and_install_rpms mariner-rpm-macros chroot_and_install_rpms mariner-check-macros build_rpm_in_chroot_no_install filesystem @@ -403,7 +428,6 @@ chroot_and_install_rpms python3 python3 # libxml2 is required for at least: libxslt, createrepo_c build_rpm_in_chroot_no_install libxml2 -copy_rpm_subpackage python3-libxml2 chroot_and_install_rpms libxml2 # Download JDK rpms @@ -576,7 +600,6 @@ build_rpm_in_chroot_no_install libselinux # libcap-ng needs: swig, python3 build_rpm_in_chroot_no_install libcap-ng -copy_rpm_subpackage python3-libcap-ng # util-linux and rpm require libselinux and libcap-ng chroot_and_install_rpms libselinux @@ -590,10 +613,8 @@ build_rpm_in_chroot_no_install rpm # python-jinja2 needs python3-markupsafe # python3-setuptools, python3-libs are also needed but already installed build_rpm_in_chroot_no_install python-markupsafe -copy_rpm_subpackage python3-markupsafe chroot_and_install_rpms python3-markupsafe build_rpm_in_chroot_no_install python-jinja2 -copy_rpm_subpackage python3-jinja2 # systemd-bootstrap requires libcap, xz, kbd, kmod, util-linux, meson, intltool, python3-jinja2 # gperf is also needed, but is installed earlier @@ -643,7 +664,6 @@ build_rpm_in_chroot_no_install pyproject-rpm-macros # Without it, audit's systemd macros won't expand and install/uninstall # will fail. build_rpm_in_chroot_no_install audit -copy_rpm_subpackage python3-audit stop_record_timestamp "build packages" start_record_timestamp "finalize" @@ -656,7 +676,6 @@ cp -v $CHROOT_RPMS_DIR_ARCH/* $FINISHED_RPM_DIR cp -v $CHROOT_RPMS_DIR_NOARCH/* $FINISHED_RPM_DIR echo Finished building final list of toolchain RPMs -chroot_unmount ls -la $FINISHED_RPM_DIR ls -la $FINISHED_RPM_DIR | wc