[AUTO-CHERRYPICK] Adding support for multiple cache inputs - branch main (#7464)

Co-authored-by: Pawel Winogrodzki <pawelwi@microsoft.com>

.pipelines/templates/PackageBuild.yml

@@ -30,6 +30,13 @@ parameters:
     type: boolean
     default: true
 
+  - name: inputCacheArtifacts
+    type: object
+    default: []
+    # Sample:
+    # - name: build-artifacts
+    #   rpmsTarball: cache.tar.gz
+
   - name: isCheckBuild
     type: string
     default: "default"
@@ -86,6 +93,18 @@ parameters:
     type: string
     default: ""
 
+  - name: outputRPMsCacheTarballName
+    type: string
+    default: ""
+
+  - name: outputRPMsTarballName
+    type: string
+    default: "rpms.tar.gz"
+
+  - name: outputSRPMsTarballName
+    type: string
+    default: "srpms.tar.gz"
+
   - name: pipArtifactFeeds
     type: string
     default: ""
@@ -94,18 +113,6 @@ parameters:
     type: boolean
     default: true
 
-  - name: rpmsCacheArtifactName
-    type: string
-    default: ""
-
-  - name: rpmsCacheTarballName
-    type: string
-    default: "rpms.tar.gz"
-
-  - name: rpmsTarballName
-    type: string
-    default: "rpms.tar.gz"
-
   - name: selfRepoName
     type: string
     default: "CBL-Mariner"
@@ -118,10 +125,6 @@ parameters:
     type: string
     default: ""
 
-  - name: srpmsTarballName
-    type: string
-    default: "srpms.tar.gz"
-
   - name: testSuiteName
     type: string
     default: "Package test"
@@ -155,22 +158,22 @@ steps:
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive"
         displayName: "Populate toolchain"
 
-  - ${{ if parameters.rpmsCacheArtifactName }}:
+  - ${{ each inputCacheArtifact in parameters.inputCacheArtifacts }}:
       - task: DownloadPipelineArtifact@2
-        displayName: "Download RPM cache"
+        displayName: "Download input cache RPM from ${{ inputCacheArtifact.name }}"
         inputs:
-          artifact: "${{ parameters.rpmsCacheArtifactName }}"
+          artifact: "${{ inputCacheArtifact.name }}"
-          patterns: "**/${{ parameters.rpmsCacheTarballName }}"
+          patterns: "**/${{ inputCacheArtifact.rpmsTarball }}"
 
       - script: |
-          rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ parameters.rpmsCacheTarballName }}" -print -quit)"
+          rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ inputCacheArtifact.rpmsTarball }}" -print -quit)"
           if [[ ! -f "$rpms_archive" ]]; then
-            echo "ERROR: RPMs cache archive not found!" >&2
+            echo "ERROR: cache RPMs archive '${{ inputCacheArtifact.rpmsTarball }}' not found!" >&2
             exit 1
           fi
 
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" hydrate-cached-rpms CACHED_PACKAGES_ARCHIVE="$rpms_archive"
-        displayName: "Populate RPMs cache"
+        displayName: "Populate cache RPMs"
 
   - script: |
       if [[ ${{ parameters.isDeltaBuild }} == "true" ]]; then
@@ -219,7 +222,18 @@ steps:
     displayName: "Build packages"
 
   - ${{ if parameters.outputArtifactsFolder }}:
-      - ${{ if parameters.rpmsTarballName }}:
+      - ${{ if parameters.outputRPMsCacheTarballName }}:
+          - script: |
+              sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" compress-cached-rpms
+            displayName: "Compress cached RPMs"
+
+          - bash: |
+              published_artifacts_dir="${{ parameters.outputArtifactsFolder }}/${{ parameters.outputArtifactsPackagesSubfolder }}"
+              mkdir -p "$published_artifacts_dir"
+              cp "${{ parameters.buildRepoRoot }}"/out/cache.tar.gz "$published_artifacts_dir/${{ parameters.outputRPMsCacheTarballName }}"
+            displayName: "Copy cached RPMs for publishing"
+
+      - ${{ if parameters.outputRPMsTarballName }}:
           - script: |
               sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" compress-rpms
             displayName: "Compress RPMs"
@@ -227,10 +241,10 @@ steps:
           - bash: |
               published_artifacts_dir="${{ parameters.outputArtifactsFolder }}/${{ parameters.outputArtifactsPackagesSubfolder }}"
               mkdir -p "$published_artifacts_dir"
-              cp "${{ parameters.buildRepoRoot }}"/out/rpms.tar.gz "$published_artifacts_dir/${{ parameters.rpmsTarballName }}"
+              cp "${{ parameters.buildRepoRoot }}"/out/rpms.tar.gz "$published_artifacts_dir/${{ parameters.outputRPMsTarballName }}"
             displayName: "Copy RPMs for publishing"
 
-      - ${{ if parameters.srpmsTarballName }}:
+      - ${{ if parameters.outputSRPMsTarballName }}:
           - script: |
               sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" compress-srpms
             displayName: "Compress SRPMs"
@@ -238,7 +252,7 @@ steps:
           - bash: |
               published_artifacts_dir="${{ parameters.outputArtifactsFolder }}/${{ parameters.outputArtifactsPackagesSubfolder }}"
               mkdir -p "$published_artifacts_dir"
-              cp "${{ parameters.buildRepoRoot }}"/out/srpms.tar.gz "$published_artifacts_dir/${{ parameters.srpmsTarballName }}"
+              cp "${{ parameters.buildRepoRoot }}"/out/srpms.tar.gz "$published_artifacts_dir/${{ parameters.outputSRPMsTarballName }}"
             displayName: "Copy SRPMs for publishing"
 
       - ${{ if parameters.publishLogs }}:

toolkit/scripts/pkggen.mk

@@ -233,6 +233,7 @@ $(preprocessed_file): $(cached_file) $(go-graphPreprocessor)
 
 ######## PACKAGE BUILD ########
 
+cache_archive	= $(OUT_DIR)/cache.tar.gz
 pkggen_archive	= $(OUT_DIR)/rpms.tar.gz
 srpms_archive  	= $(OUT_DIR)/srpms.tar.gz
 
@@ -326,6 +327,10 @@ compress-rpms:
 	tar -cvp -f $(BUILD_DIR)/temp_rpms_tarball.tar.gz -C $(RPMS_DIR)/.. $(notdir $(RPMS_DIR))
 	mv $(BUILD_DIR)/temp_rpms_tarball.tar.gz $(pkggen_archive)
 
+##help:target:compress-cached-rpms=Compresses all cached RPMs in `build/rpm_cache/cache` into `out/cache.tar.gz`.
+compress-cached-rpms:
+	tar -cvp -f $(cache_archive) -C $(remote_rpms_cache_dir)/.. $(notdir $(remote_rpms_cache_dir))
+
 ##help:target:compress-srpms=Compresses all SRPMs in `../out/SRPMS` into `../out/srpms.tar.gz`.
 # use temp tarball to avoid tar warning "file changed as we read it"
 # that can sporadically occur when tarball is the dir that is compressed
@@ -333,7 +338,10 @@ compress-srpms:
 	tar -cvp -f $(BUILD_DIR)/temp_srpms_tarball.tar.gz -C $(SRPMS_DIR)/.. $(notdir $(SRPMS_DIR))
 	mv $(BUILD_DIR)/temp_srpms_tarball.tar.gz $(srpms_archive)
 
-# Seed the cached RPMs folder files from the archive.
+##help:target:hydrate-cached-rpms=Hydrates the external RPMs cache from the `CACHED_PACKAGES_ARCHIVE` file.
+# All of the '*.rpm' files inside the archive will be extracted into the cache directory in flat manner.
+# Any duplicates inside the archive's subdirectories will be overwritten by the last one.
+# Also see the `compress-cached-rpms` target.
 hydrate-cached-rpms:
 	$(if $(CACHED_PACKAGES_ARCHIVE),,$(error Must set CACHED_PACKAGES_ARCHIVE=<path>))
 	@mkdir -p $(remote_rpms_cache_dir)