Separated toolchain tests from non-toolchain package builds. (#10438)

2024-09-13 11:04:45 -07:00 · 2024-09-13 11:04:45 -07:00 · 7bb7a6b32a
--- a/.pipelines/prchecks/PackageBuildPRCheck.yml
+++ b/.pipelines/prchecks/PackageBuildPRCheck.yml
@ -39,8 +39,10 @@ variables:
    value: RPMs
  - name: toolchainArtifactNameBase
    value: Toolchain
+  - name: toolchainTestsArtifactNameBase
+    value: Toolchain_tests
  - name: system.debug
-    value: '${{ parameters.debug }}'
+    value: "${{ parameters.debug }}"

 extends:
  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
@ -74,6 +76,9 @@ extends:
                  # Toolchain package tests should be run through the full package build, calculate the list of packages that should be re-tested
                  # and make it available to the next stage via an output variable: 'CalculateToolchainPackageRetestList.toolchainPackageRetestList'
                  - template: .pipelines/templates/ToolchainCalculatePackageRetests.yml@self
+                    parameters:
+                      # GCC fails to build as a regular package.
+                      ignoredSpecs: ["gcc"]

                  - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
                    name: "ToolchainArtifactName"
@ -99,9 +104,8 @@ extends:
                  isCustom: true
                  name: ${{ configuration.agentPool }}
                variables:
-                  ob_artifactBaseName: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}_$(System.JobAttempt)
+                  ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                  ob_outputDirectory: $(Build.ArtifactStagingDirectory)
-                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
                  toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
                steps:
                  - template: .pipelines/templates/PackageBuild.yml@self
@ -111,12 +115,11 @@ extends:
                      isCheckBuild: true
                      isQuickRebuildPackages: true
                      isUseCCache: true
-                      outputArtifactsFolder: $(ob_outputDirectory)
                      maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
                      pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                      selfRepoName: self
                      testSuiteName: "[${{ configuration.name }}] Package test"
-                      testRerunList: "$(testListFromToolchain)"

                  - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
                    name: "RPMsArtifactName"
@ -124,12 +127,50 @@ extends:

                  - task: PublishPipelineArtifact@1
                    inputs:
-                      artifact: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}_$(System.JobAttempt)
+                      artifact: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                      targetPath: $(ob_outputDirectory)
                    condition: always()
                    displayName: "Publish packages build artifacts"

-          - stage: sodiff_${{ configuration.name }}
+          - stage: Toolchain_tests_${{ configuration.name }}
+            dependsOn: Toolchain_${{ configuration.name }}
+            jobs:
+              - job: TestToolchainPackages
+                condition: stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList']
+                pool:
+                  type: linux
+                  isCustom: true
+                  name: ${{ configuration.agentPool }}
+                variables:
+                  ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                  ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
+                  toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                steps:
+                  - template: .pipelines/templates/PackageBuild.yml@self
+                    parameters:
+                      checkBuildRetries: "1"
+                      customToolchainArtifactName: $(toolchainArtifactName)
+                      isAllowToolchainRebuilds: true
+                      isCheckBuild: true
+                      isQuickRebuildPackages: true
+                      isUseCCache: true
+                      maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
+                      pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
+                      selfRepoName: self
+                      srpmPackList: "$(testListFromToolchain)"
+                      testRerunList: "$(testListFromToolchain)"
+                      testSuiteName: "[${{ configuration.name }}] Toolchain test"
+
+                  - task: PublishPipelineArtifact@1
+                    inputs:
+                      artifact: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                      targetPath: $(ob_outputDirectory)
+                    condition: always()
+                    displayName: "Publish toolchain build artifacts"
+
+          - stage: Sodiff_${{ configuration.name }}
            dependsOn: RPMs_${{ configuration.name }}
            jobs:
              - job: Sodiff_Check
--- a/.pipelines/templates/PackageBuild.yml
+++ b/.pipelines/templates/PackageBuild.yml
@ -37,6 +37,14 @@ parameters:
    # - name: build-artifacts
    #   rpmsTarball: cache.tar.gz

+  - name: isAllowToolchainRebuilds
+    type: string
+    default: "default"
+    values:
+      - "default"
+      - "false"
+      - "true"
+
  - name: isCheckBuild
    type: string
    default: "default"
@ -105,6 +113,14 @@ parameters:
    type: string
    default: "srpms.tar.gz"

+  - name: packageBuildList
+    type: string
+    default: ""
+
+  - name: packageRebuildList
+    type: string
+    default: ""
+
  - name: pipArtifactFeeds
    type: string
    default: ""
@ -184,20 +200,14 @@ steps:
        check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}"
      fi

-      if [[ ${{ parameters.isDeltaBuild }} == "true" ]]; then
-        delta_fetch_arg="DELTA_FETCH=y"
-      elif [[ ${{ parameters.isDeltaBuild }} == "false" ]]; then
-        delta_fetch_arg="DELTA_FETCH=n"
+      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
+        toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
      fi

-      if [[ -n "${{ parameters.maxCascadingRebuilds }}" ]]; then
-        max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}"
-      fi
-
-      if [[ ${{ parameters.isQuickRebuildPackages }} == "true" ]]; then
-        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=y"
-      elif [[ ${{ parameters.isQuickRebuildPackages }} == "false" ]]; then
-        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=n"
+      if [[ ${{ parameters.isAllowToolchainRebuilds }} == "true" ]]; then
+        allow_toolchain_rebuilds_arg="ALLOW_TOOLCHAIN_REBUILDS=y"
+      elif [[ ${{ parameters.isAllowToolchainRebuilds }} == "false" ]]; then
+        allow_toolchain_rebuilds_arg="ALLOW_TOOLCHAIN_REBUILDS=n"
      fi

      if [[ ${{ parameters.isCheckBuild }} == "true" ]]; then
@ -206,8 +216,16 @@ steps:
        run_check_arg="RUN_CHECK=n"
      fi

-      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
-        toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
+      if [[ ${{ parameters.isDeltaBuild }} == "true" ]]; then
+        delta_fetch_arg="DELTA_FETCH=y"
+      elif [[ ${{ parameters.isDeltaBuild }} == "false" ]]; then
+        delta_fetch_arg="DELTA_FETCH=n"
+      fi
+
+      if [[ ${{ parameters.isQuickRebuildPackages }} == "true" ]]; then
+        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=y"
+      elif [[ ${{ parameters.isQuickRebuildPackages }} == "false" ]]; then
+        quick_rebuild_packages_arg="QUICK_REBUILD_PACKAGES=n"
      fi

      if [[ ${{ parameters.isUseCCache }} == "true" ]]; then
@ -216,15 +234,22 @@ steps:
        use_ccache_arg="USE_CCACHE=n"
      fi

+      if [[ -n "${{ parameters.maxCascadingRebuilds }}" ]]; then
+        max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}"
+      fi
+
      sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" build-packages -j$(nproc) \
        CONCURRENT_PACKAGE_BUILDS=${{ parameters.concurrentPackageBuilds }} \
        CONFIG_FILE="" \
        MAX_CPU="${{ parameters.maxCPU }}" \
+        PACKAGE_BUILD_LIST="${{ parameters.packageBuildList }}" \
+        PACKAGE_REBUILD_LIST="${{ parameters.packageRebuildList }}" \
        REBUILD_TOOLS=y \
        REPO_LIST="${{ parameters.extraPackageRepos }}" \
        SPECS_DIR="${{ parameters.buildRepoRoot }}/${{ parameters.specsFolderPath }}" \
        SRPM_PACK_LIST="${{ parameters.srpmPackList }}" \
        TEST_RERUN_LIST="${{ parameters.testRerunList }}" \
+        $allow_toolchain_rebuilds_arg \
        $check_build_retries_arg \
        $delta_fetch_arg \
        $max_cascading_rebuilds_arg \
--- a/.pipelines/templates/PackageTestResultsAnalysis.yml
+++ b/.pipelines/templates/PackageTestResultsAnalysis.yml
@ -281,14 +281,16 @@ steps:

  - ${{ if parameters.failOnTestFailures }}:
      - bash: |
-            report_path="${{ parameters.testsWorkspace }}/${{ parameters.reportFileName }}"
-            if [[ ! -f "$report_path" ]]; then
-                echo "##[error]Test report not found at '$report_path'."
-                exit 1
-            fi
+          report_path="${{ parameters.testsWorkspace }}/${{ parameters.reportFileName }}"
+          if [[ ! -f "$report_path" ]]; then
+              echo "##[error]Test report not found at '$report_path'."
+              exit 1
+          fi

-            if ! grep -q '^<testsuites.*failures="0"' "$report_path"; then
-                echo "##[error]Test report has failing tests. See the 'Tests' tab for details."
-                exit 1
-            fi
+          # The "failures" attribute indicates completed tests where at least one of the test cases failed.
+          # The "errors" attribute indicates tests, which failed to complete for whatever reason.
+          if grep -qP '^<testsuites.*(errors|failures)="(?!0)' "$report_path"; then
+              echo "##[error]Test report has failing tests. See the 'Tests' tab for details."
+              exit 1
+          fi
        displayName: "Verify all tests passed"
--- a/.pipelines/templates/ToolchainCalculatePackageRetests.yml
+++ b/.pipelines/templates/ToolchainCalculatePackageRetests.yml
@ -1,16 +1,38 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.

+# This template will calculate a list of toolchain packages that might need to be re-tested at a later stage.
+# It assumes a toolchain build has already been performed in the same job.
+
+# Output pipeline variables set by this template:
+# - CalculateToolchainPackageRetestList.toolchainPackageRetestList
+
 parameters:
  - name: buildRepoRoot
    type: string
    default: "$(Build.SourcesDirectory)"

+  - name: ignoredSpecs
+    type: object
+    default: []
+
 steps:
-  # This template will calculate a list of toolchain packages that might need to be re-tested at a later stage.
-  # It assumes a toolchain build has already been performed in the same job. The value will be made available as
-  # the variable 'CalculateToolchainPackageRetestList.toolchainPackageRetestList'.
  - bash: |
+      array_contains() {
+          local -n __array=$1
+          local value=$2
+
+          for item in "${__array[@]}"; do
+              if [[ "$item" == "$value" ]]; then
+                  return 0
+              fi
+          done
+
+          return 1
+      }
+
+      ignored_specs=(${{ join(parameters.ignoredSpecs, ' ') }})
+
      # Calculate the list of packages that should be re-tested during full package build.
      #   This list will be the contents of 'built_specs_list.txt' in the toolchain build logs directory, but only for
      #   packages that have a '%check' section in their spec file. The assumption is that all packages will have a 
@ -24,14 +46,18 @@ steps:
              echo "##[error]ERROR: '${specs_dir}/${spec}/${spec}.spec' does not exist"
              exit 1
          fi
+
          if grep -q '^%check' "${specs_dir}/${spec}/${spec}.spec"; then
-              retest_list+=("${spec}")
+              if ! array_contains ignored_specs "${spec}"; then
+                retest_list+=("${spec}")
+              fi
          fi
        done < "$built_list"
      else
        echo "No file '$built_list' found, so no packages to re-test"
      fi
      # Default will be "", which is fine.
+
      echo "Setting 'CalculateToolchainPackageRetestList.toolchainPackageRetestList' to '${retest_list[*]}'"
      echo "##vso[task.setvariable variable=toolchainPackageRetestList;isOutput=true]${retest_list[*]}"
    name: "CalculateToolchainPackageRetestList"