[AUTO-CHERRYPICK] Fixed toolchain tests blocking non-toolchain packages' tests. - branch 3.0-dev (#10457)

Co-authored-by: Pawel Winogrodzki <pawelwi@microsoft.com>

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -39,8 +39,10 @@ variables:
     value: RPMs
   - name: toolchainArtifactNameBase
     value: Toolchain
+  - name: toolchainTestsArtifactNameBase
+    value: Toolchain_tests
   - name: system.debug
-    value: '${{ parameters.debug }}'
+    value: "${{ parameters.debug }}"
 
 extends:
   template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
@@ -74,6 +76,9 @@ extends:
                   # Toolchain package tests should be run through the full package build, calculate the list of packages that should be re-tested
                   # and make it available to the next stage via an output variable: 'CalculateToolchainPackageRetestList.toolchainPackageRetestList'
                   - template: .pipelines/templates/ToolchainCalculatePackageRetests.yml@self
+                    parameters:
+                      # GCC fails to build as a regular package.
+                      ignoredSpecs: ["gcc"]
 
                   - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
                     name: "ToolchainArtifactName"
@@ -99,9 +104,8 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
-                  ob_artifactBaseName: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}_$(System.JobAttempt)
+                  ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
-                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
                 steps:
                   - template: .pipelines/templates/PackageBuild.yml@self
@@ -111,11 +115,10 @@ extends:
                       isCheckBuild: true
                       isQuickRebuildPackages: true
                       isUseCCache: true
-                      outputArtifactsFolder: $(ob_outputDirectory)
                       maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
                       pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                       selfRepoName: self
-                      testRerunList: "$(testListFromToolchain)"
                       testSuiteName: "[${{ configuration.name }}] Package test"
 
                   - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
@@ -124,12 +127,50 @@ extends:
 
                   - task: PublishPipelineArtifact@1
                     inputs:
-                      artifact: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}_$(System.JobAttempt)
+                      artifact: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                       targetPath: $(ob_outputDirectory)
                     condition: always()
                     displayName: "Publish packages build artifacts"
 
-          - stage: sodiff_${{ configuration.name }}
+          - stage: Toolchain_tests_${{ configuration.name }}
+            dependsOn: Toolchain_${{ configuration.name }}
+            jobs:
+              - job: TestToolchainPackages
+                condition: stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList']
+                pool:
+                  type: linux
+                  isCustom: true
+                  name: ${{ configuration.agentPool }}
+                variables:
+                  ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                  ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
+                  toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                steps:
+                  - template: .pipelines/templates/PackageBuild.yml@self
+                    parameters:
+                      checkBuildRetries: "1"
+                      customToolchainArtifactName: $(toolchainArtifactName)
+                      isAllowToolchainRebuilds: true
+                      isCheckBuild: true
+                      isQuickRebuildPackages: true
+                      isUseCCache: true
+                      maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
+                      pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
+                      selfRepoName: self
+                      srpmPackList: "$(testListFromToolchain)"
+                      testRerunList: "$(testListFromToolchain)"
+                      testSuiteName: "[${{ configuration.name }}] Toolchain test"
+
+                  - task: PublishPipelineArtifact@1
+                    inputs:
+                      artifact: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                      targetPath: $(ob_outputDirectory)
+                    condition: always()
+                    displayName: "Publish toolchain build artifacts"
+
+          - stage: Sodiff_${{ configuration.name }}
             dependsOn: RPMs_${{ configuration.name }}
             jobs:
               - job: Sodiff_Check

.pipelines/templates/PackageBuild.yml

@@ -37,6 +37,14 @@ parameters:
     # - name: build-artifacts
     #   rpmsTarball: cache.tar.gz
 
+  - name: isAllowToolchainRebuilds
+    type: string
+    default: "default"
+    values:
+      - "default"
+      - "false"
+      - "true"
+
   - name: isCheckBuild
     type: string
     default: "default"
@@ -105,6 +113,14 @@ parameters:
     type: string
     default: "srpms.tar.gz"
 
+  - name: packageBuildList
+    type: string
+    default: ""
+
+  - name: packageRebuildList
+    type: string
+    default: ""
+
   - name: pipArtifactFeeds
     type: string
     default: ""
@@ -184,20 +200,14 @@ steps:
         check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}"
       fi
 
-      if [[ ${{ parameters.isDeltaBuild }} == "true" ]]; then
-        delta_fetch_arg="DELTA_FETCH=y"
-      elif [[ ${{ parameters.isDeltaBuild }} == "false" ]]; then
-        delta_fetch_arg="DELTA_FETCH=n"
+      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
+        toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
       fi
 
-      if [[ -n "${{ parameters.maxCascadingRebuilds }}" ]]; then
-        max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}"
-      fi
-
-      if [[ ${{ parameters.isQuickRebuildPackages }} == "true" ]]; then
-        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=y"
-      elif [[ ${{ parameters.isQuickRebuildPackages }} == "false" ]]; then
-        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=n"
+      if [[ ${{ parameters.isAllowToolchainRebuilds }} == "true" ]]; then
+        allow_toolchain_rebuilds_arg="ALLOW_TOOLCHAIN_REBUILDS=y"
+      elif [[ ${{ parameters.isAllowToolchainRebuilds }} == "false" ]]; then
+        allow_toolchain_rebuilds_arg="ALLOW_TOOLCHAIN_REBUILDS=n"
       fi
 
       if [[ ${{ parameters.isCheckBuild }} == "true" ]]; then
@@ -206,8 +216,16 @@ steps:
         run_check_arg="RUN_CHECK=n"
       fi
 
-      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
-        toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
+      if [[ ${{ parameters.isDeltaBuild }} == "true" ]]; then
+        delta_fetch_arg="DELTA_FETCH=y"
+      elif [[ ${{ parameters.isDeltaBuild }} == "false" ]]; then
+        delta_fetch_arg="DELTA_FETCH=n"
+      fi
+
+      if [[ ${{ parameters.isQuickRebuildPackages }} == "true" ]]; then
+        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=y"
+      elif [[ ${{ parameters.isQuickRebuildPackages }} == "false" ]]; then
+        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=n"
       fi
 
       if [[ ${{ parameters.isUseCCache }} == "true" ]]; then
@@ -216,15 +234,22 @@ steps:
         use_ccache_arg="USE_CCACHE=n"
       fi
 
+      if [[ -n "${{ parameters.maxCascadingRebuilds }}" ]]; then
+        max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}"
+      fi
+
       sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" build-packages -j$(nproc) \
         CONCURRENT_PACKAGE_BUILDS=${{ parameters.concurrentPackageBuilds }} \
         CONFIG_FILE="" \
         MAX_CPU="${{ parameters.maxCPU }}" \
+        PACKAGE_BUILD_LIST="${{ parameters.packageBuildList }}" \
+        PACKAGE_REBUILD_LIST="${{ parameters.packageRebuildList }}" \
         REBUILD_TOOLS=y \
         REPO_LIST="${{ parameters.extraPackageRepos }}" \
         SPECS_DIR="${{ parameters.buildRepoRoot }}/${{ parameters.specsFolderPath }}" \
         SRPM_PACK_LIST="${{ parameters.srpmPackList }}" \
         TEST_RERUN_LIST="${{ parameters.testRerunList }}" \
+        $allow_toolchain_rebuilds_arg \
         $check_build_retries_arg \
         $delta_fetch_arg \
         $max_cascading_rebuilds_arg \

.pipelines/templates/PackageTestResultsAnalysis.yml

@@ -281,14 +281,16 @@ steps:
 
   - ${{ if parameters.failOnTestFailures }}:
       - bash: |
-            report_path="${{ parameters.testsWorkspace }}/${{ parameters.reportFileName }}"
-            if [[ ! -f "$report_path" ]]; then
-                echo "##[error]Test report not found at '$report_path'."
-                exit 1
-            fi
+          report_path="${{ parameters.testsWorkspace }}/${{ parameters.reportFileName }}"
+          if [[ ! -f "$report_path" ]]; then
+              echo "##[error]Test report not found at '$report_path'."
+              exit 1
+          fi
 
-            if ! grep -q '^<testsuites.*failures="0"' "$report_path"; then
-                echo "##[error]Test report has failing tests. See the 'Tests' tab for details."
-                exit 1
-            fi
+          # The "failures" attribute indicates completed tests where at least one of the test cases failed.
+          # The "errors" attribute indicates tests, which failed to complete for whatever reason.
+          if grep -qP '^<testsuites.*(errors|failures)="(?!0)' "$report_path"; then
+              echo "##[error]Test report has failing tests. See the 'Tests' tab for details."
+              exit 1
+          fi
         displayName: "Verify all tests passed"

.pipelines/templates/ToolchainCalculatePackageRetests.yml

@@ -1,16 +1,38 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+# This template will calculate a list of toolchain packages that might need to be re-tested at a later stage.
+# It assumes a toolchain build has already been performed in the same job.
+
+# Output pipeline variables set by this template:
+# - CalculateToolchainPackageRetestList.toolchainPackageRetestList
+
 parameters:
   - name: buildRepoRoot
     type: string
     default: "$(Build.SourcesDirectory)"
 
+  - name: ignoredSpecs
+    type: object
+    default: []
+
 steps:
-  # This template will calculate a list of toolchain packages that might need to be re-tested at a later stage.
-  # It assumes a toolchain build has already been performed in the same job. The value will be made available as
-  # the variable 'CalculateToolchainPackageRetestList.toolchainPackageRetestList'.
   - bash: |
+      array_contains() {
+          local -n __array=$1
+          local value=$2
+
+          for item in "${__array[@]}"; do
+              if [[ "$item" == "$value" ]]; then
+                  return 0
+              fi
+          done
+
+          return 1
+      }
+
+      ignored_specs=(${{ join(parameters.ignoredSpecs, ' ') }})
+
       # Calculate the list of packages that should be re-tested during full package build.
       #   This list will be the contents of 'built_specs_list.txt' in the toolchain build logs directory, but only for
       #   packages that have a '%check' section in their spec file. The assumption is that all packages will have a 
@@ -24,14 +46,18 @@ steps:
               echo "##[error]ERROR: '${specs_dir}/${spec}/${spec}.spec' does not exist"
               exit 1
           fi
+
           if grep -q '^%check' "${specs_dir}/${spec}/${spec}.spec"; then
-              retest_list+=("${spec}")
+              if ! array_contains ignored_specs "${spec}"; then
+                retest_list+=("${spec}")
+              fi
           fi
         done < "$built_list"
       else
         echo "No file '$built_list' found, so no packages to re-test"
       fi
       # Default will be "", which is fine.
+
       echo "Setting 'CalculateToolchainPackageRetestList.toolchainPackageRetestList' to '${retest_list[*]}'"
       echo "##vso[task.setvariable variable=toolchainPackageRetestList;isOutput=true]${retest_list[*]}"
     name: "CalculateToolchainPackageRetestList"