Updated package building pipeline templates to support external repos. (#5961)

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -102,8 +102,7 @@ extends:
                     steps:
                       - template: .pipelines/templates/PackageBuild.yml@self
                         parameters:
-                          customToolchainArtifactBuild: $(toolchainArtifactNameBase)_${{ configuration.name }}
-                          customToolchainEnabled: true
+                          customToolchainArtifactName: $(toolchainArtifactNameBase)_${{ configuration.name }}
                           isCheckBuild: ${{ packageBuildJob.runCheck }}
                           isDeltaBuild: true
                           outputArtifactsFolder: $(ob_outputDirectory)

.pipelines/templates/PackageBuild.yml

@@ -1,11 +1,19 @@
 parameters:
-  - name: customToolchainArtifactBuild
+  - name: buildRepoRoot
+    type: string
+    default: "$(Build.SourcesDirectory)"
+
+  - name: concurrentPackageBuilds
+    type: number
+    default: 12
+
+  - name: customToolchainArtifactName
     type: string
     default: ""
 
-  - name: customToolchainEnabled
-    type: boolean
-    default: false
+  - name: extraPackageRepos
+    type: string
+    default: ""
 
   - name: isCheckBuild
     type: boolean
@@ -19,9 +27,9 @@ parameters:
     type: string
     default: "$(Build.ArtifactStagingDirectory)"
 
-  - name: buildRepoRoot
+  - name: rpmsCacheArtifactName
     type: string
-    default: "$(Build.SourcesDirectory)"
+    default: ""
 
   - name: rpmsTarballName
     type: string
@@ -31,6 +39,10 @@ parameters:
     type: string
     default: "CBL-Mariner"
 
+  - name: specsFolderPath
+    type: string
+    default: "SPECS"
+
   - name: srpmsTarballName
     type: string
     default: "srpms.tar.gz"
@@ -40,11 +52,11 @@ steps:
     parameters:
       buildRepoRoot: ${{ parameters.buildRepoRoot }}
 
-  - ${{ if eq(parameters.customToolchainEnabled, true) }}:
+  - ${{ if parameters.customToolchainArtifactName }}:
       - task: DownloadPipelineArtifact@2
         displayName: "Download toolchain"
         inputs:
-          artifact: "${{ parameters.customToolchainArtifactBuild }}"
+          artifact: "${{ parameters.customToolchainArtifactName }}"
           patterns: "**/toolchain_built_rpms_all.tar.gz"
 
       - script: |
@@ -57,6 +69,23 @@ steps:
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive"
         displayName: "Populate toolchain"
 
+  - ${{ if parameters.rpmsCacheArtifactName }}:
+      - task: DownloadPipelineArtifact@2
+        displayName: "Download RPM cache"
+        inputs:
+          artifact: "${{ parameters.rpmsCacheArtifactName }}"
+          patterns: "**/rpms*.tar.gz"
+
+      - script: |
+          rpms_archive="$(find "$(Pipeline.Workspace)" -name rpms*.tar.gz)"
+          if [[ ! -f "$rpms_archive" ]]; then
+            echo "ERROR: RPMs cache archive not found!" >&2
+            exit 1
+          fi
+
+          sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" hydrate-cached-rpms CACHED_PACKAGES_ARCHIVE="$rpms_archive"
+        displayName: "Populate RPMs cache"
+
   - script: |
       delta_fetch_arg="DELTA_FETCH=n"
       if [[ ${{ parameters.isDeltaBuild }} =~ [Tt]rue ]]; then
@@ -69,9 +98,12 @@ steps:
       fi
 
       sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" build-packages -j$(nproc) \
+        CONCURRENT_PACKAGE_BUILDS=${{ parameters.concurrentPackageBuilds }} \
         CONFIG_FILE="" \
-        $delta_fetch_arg \
         REBUILD_TOOLS=y \
+        REPO_LIST="${{ parameters.extraPackageRepos }}" \
+        SPECS_DIR="${{ parameters.buildRepoRoot }}/${{ parameters.specsFolderPath }}" \
+        $delta_fetch_arg \
         $run_check_arg
     displayName: "Build packages"
 

.pipelines/templates/ToolchainBuild.yml

@@ -34,7 +34,7 @@ steps:
   - bash: |
       published_artifacts_dir="${{ parameters.outputArtifactsFolder }}/ARTIFACTS"
       mkdir -p "$published_artifacts_dir"
-      cp "${{ parameters.buildRepoRoot }}"/build/toolchain/toolchain_built_{,s}rpms_all.tar.gz "$published_artifacts_dir"
+      cp "${{ parameters.buildRepoRoot }}"/build/toolchain/toolchain_built_{,s}rpms_*.tar.gz "$published_artifacts_dir"
     displayName: "Copy artifacts for publishing"
 
   - bash: |

toolkit/Makefile

@@ -59,6 +59,7 @@ REBUILD_DEP_CHAINS              ?= y
 HYDRATED_BUILD                  ?= n
 TARGET_ARCH                     ?=
 ALLOW_TOOLCHAIN_REBUILDS        ?= n
+CACHED_PACKAGES_ARCHIVE         ?=
 
 # Tracing & Profiling support: https://go.dev/doc/diagnostics
 ENABLE_CPU_PROFILE              ?= n

toolkit/docs/building/building.md

@@ -772,6 +772,7 @@ To reproduce an ISO build, run the same make invocation as before, but set:
 | USE_PREVIEW_REPO              | n                                                                                                      | Pull missing packages from the upstream preview repository in addition to the base repository?
 | DISABLE_UPSTREAM_REPOS        | n                                                                                                      | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`.
 | DISABLE_DEFAULT_REPOS         | n                                                                                                      | Disable pulling packages from PMC. Use this option with `REPO_LIST` if you want to use your own repository exclusively.
+| CACHED_PACKAGES_ARCHIVE       |                                                                                                        | Use with `make hydrate-cached-rpms` to populate the external RPMs cache from an archive.
 
 ---
 

toolkit/scripts/pkggen.mk

@@ -26,7 +26,7 @@ pkggen_rpms     = $(call shell_real_build_only, find $(RPMS_DIR)/*  2>/dev/null
 cache_working_dir      = $(PKGBUILD_DIR)/tdnf_cache_worker
 parse_working_dir      = $(BUILD_DIR)/spec_parsing
 rpmbuilding_logs_dir   = $(LOGS_DIR)/pkggen/rpmbuilding
-rpm_cache_files        = $(call shell_real_build_only, find $(CACHED_RPMS_DIR)/)
+rpm_cache_files        = $(call shell_real_build_only, find $(CACHED_RPMS_DIR))
 validate-pkggen-config = $(STATUS_FLAGS_DIR)/validate-image-config-pkggen.flag
 
 # Outputs
@@ -154,10 +154,10 @@ $(cached_file): $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACK
 endif
 
 $(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(rpm_cache_files) $(TOOLCHAIN_MANIFEST) $(toolchain_rpms)
-	mkdir -p $(CACHED_RPMS_DIR)/cache && \
+	mkdir -p $(CACHED_RPMS_DIR) && \
 	$(go-graphpkgfetcher) \
 		--input=$(graph_file) \
-		--output-dir=$(CACHED_RPMS_DIR)/cache \
+		--output-dir=$(CACHED_RPMS_DIR) \
 		--rpm-dir=$(RPMS_DIR) \
 		--toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \
 		--tmp-dir=$(cache_working_dir) \
@@ -233,7 +233,7 @@ $(STATUS_FLAGS_DIR)/build-rpms.flag: $(preprocessed_file) $(chroot_worker) $(go-
 		--rpm-dir="$(RPMS_DIR)" \
 		--toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \
 		--srpm-dir="$(SRPMS_DIR)" \
-		--cache-dir="$(CACHED_RPMS_DIR)/cache" \
+		--cache-dir="$(CACHED_RPMS_DIR)" \
 		--ccache-dir="$(CCACHE_DIR)" \
 		--build-logs-dir="$(rpmbuilding_logs_dir)" \
 		--dist-tag="$(DIST_TAG)" \
@@ -280,8 +280,17 @@ compress-srpms:
 	tar -cvp -f $(BUILD_DIR)/temp_srpms_tarball.tar.gz -C $(SRPMS_DIR)/.. $(notdir $(SRPMS_DIR))
 	mv $(BUILD_DIR)/temp_srpms_tarball.tar.gz $(srpms_archive)
 
+# Seed the cached RPMs folder files from the archive.
+hydrate-cached-rpms:
+	$(if $(CACHED_PACKAGES_ARCHIVE),,$(error Must set CACHED_PACKAGES_ARCHIVE=<path>))
+	@echo Unpacking cache RPMs from $(CACHED_PACKAGES_ARCHIVE) into $(CACHED_RPMS_DIR)
+	@tar -xf $(CACHED_PACKAGES_ARCHIVE) -C $(CACHED_RPMS_DIR) --strip-components 1 --skip-old-files --touch --checkpoint=100000 --checkpoint-action=echo="%T"
+# The cached RPMs directory has a flat structure, so we need to move the RPMs into the cache's root directory.
+	@find $(CACHED_RPMS_DIR) -mindepth 2 -name "*.rpm" -exec mv {} $(CACHED_RPMS_DIR) \;
+	@find $(CACHED_RPMS_DIR) -mindepth 1 -type d -and ! -name repodata -exec rm -fr {} +
+
 # Seed the RPMs folder with the any missing files from the archive.
 hydrate-rpms:
-	$(if $(PACKAGE_ARCHIVE),,$(error Must set PACKAGE_ARCHIVE=))
+	$(if $(PACKAGE_ARCHIVE),,$(error Must set PACKAGE_ARCHIVE=<path>))
 	@echo Unpacking RPMs from $(PACKAGE_ARCHIVE) into $(RPMS_DIR)
 	tar -xf $(PACKAGE_ARCHIVE) -C $(RPMS_DIR) --strip-components 1 --skip-old-files --touch --checkpoint=100000 --checkpoint-action=echo="%T"

toolkit/scripts/toolchain/build_mariner_toolchain.sh

@@ -67,6 +67,14 @@ pushd $MARINER_BUILD_DIR/toolchain
 tar -C ./populated_toolchain/usr/src/mariner -cvf toolchain_built_srpms_all.tar.gz SRPMS
 popd
 
+if [ "$INCREMENTAL_TOOLCHAIN" = "y" ]; then
+    echo "Creating delta toolchain RPMs tarball."
+
+    tar -C "$MARINER_BUILD_DIR"/toolchain/built_rpms_all \
+        -T "$MARINER_BUILD_DIR"/logs/toolchain/built_rpms_list.txt \
+        -cvf "$MARINER_BUILD_DIR"/toolchain/toolchain_built_rpms_delta.tar.gz
+fi
+
 echo Printing list of built toolchain RPMS:
 ls -la $MARINER_BUILD_DIR/toolchain/built_rpms_all
 ls -la $MARINER_BUILD_DIR/toolchain/built_rpms_all | wc

toolkit/scripts/tools.mk

@@ -6,7 +6,7 @@
 #	- Chroot Utilities
 
 $(call create_folder,$(RPMS_DIR))
-$(call create_folder,$(CACHED_RPMS_DIR)/cache)
+$(call create_folder,$(CACHED_RPMS_DIR))
 $(call create_folder,$(CCACHE_DIR))
 $(call create_folder,$(TOOL_BINS_DIR))
 $(call create_folder,$(BUILD_DIR)/tools)
@@ -193,7 +193,7 @@ macro_manifest = $(TOOLCHAIN_MANIFESTS_DIR)/macro_packages.txt
 macro-tools: $(macro_rpmrc)
 
 $(macro_rpmrc): $(toolchain_rpms)
-	$(SCRIPTS_DIR)/preparemacros.sh $(MACRO_DIR) $(CACHED_RPMS_DIR)/cache $(macro_manifest)
+	$(SCRIPTS_DIR)/preparemacros.sh $(MACRO_DIR) $(CACHED_RPMS_DIR) $(macro_manifest)
 
 clean: clean-macro-tools
 clean-macro-tools: