[release/8.0.1xx-xcode16.0] Fix the macOS tests in Azure DevOps. (#21362)

This is a backport of the following PRs: * [CI] Split the sim tests and the macOS tests. (#21215) * [CI] Add missing configuration stage for the macos tests pipeline. (#21303) * [CI] Fix the pipeline that runs the macOS tests. (6df33a86bb - no PR for this one?) * [CI] Use the full path for triggering resources. (#21338) --------- Co-authored-by: Manuel de la Pena <mandel@microsoft.com>
2024-10-03 19:02:22 +02:00 · 2024-10-03 19:02:22 +02:00 · 6149daa719
--- a/tools/devops/automation/run-post-ci-build-macos-tests.yml
+++ b/tools/devops/automation/run-post-ci-build-macos-tests.yml
@ -0,0 +1,20 @@
+# YAML pipeline for post build operations. 
+# This pipeline will execute the tests for the CI as soon as the workloads have been complited.
+
+trigger: none
+pr: none
+
+
+# we cannot use a template in a pipeline context
+resources:
+  pipelines:
+    - pipeline: macios
+      source: \Xamarin\Mac-iOS\ci pipelines\xamarin-macios-ci
+      trigger:
+        stages:
+          - build_macos_tests
+
+extends:
+  template: templates/pipelines/run-macos-tests-pipeline.yml
+  parameters:
+    isPR: false
--- a/tools/devops/automation/run-post-ci-build-tests.yml
+++ b/tools/devops/automation/run-post-ci-build-tests.yml
@ -4,16 +4,14 @@
 trigger: none
 pr: none

-
 # we cannot use a template in a pipeline context
 resources:
  pipelines:
    - pipeline: macios
-      source: xamarin-macios-ci
+      source: \Xamarin\Mac-iOS\ci pipelines\xamarin-macios-ci
      trigger:
        stages:
          - build_packages
-          - build_macos_tests

 extends:
  template: templates/pipelines/run-tests-pipeline.yml
--- a/tools/devops/automation/run-post-pr-build-macos-tests.yml
+++ b/tools/devops/automation/run-post-pr-build-macos-tests.yml
@ -0,0 +1,19 @@
+# YAML pipeline for post build operations. 
+# This pipeline will execute the tests for the CI on PR as soon as the workloads have been complited.
+
+trigger: none
+pr: none
+
+# we cannot use a template in a pipeline context
+resources:
+  pipelines:
+    - pipeline: macios
+      source: \Xamarin\Mac-iOS\pr pipelines\xamarin-macios-pr
+      trigger:
+        stages:
+          - build_macos_tests
+
+extends:
+  template: templates/pipelines/run-macos-tests-pipeline.yml
+  parameters:
+    isPR: true
--- a/tools/devops/automation/run-post-pr-build-tests.yml
+++ b/tools/devops/automation/run-post-pr-build-tests.yml
@ -8,11 +8,10 @@ pr: none
 resources:
  pipelines:
    - pipeline: macios
-      source: xamarin-macios-pr
+      source: \Xamarin\Mac-iOS\pr pipelines\xamarin-macios-pr
      trigger:
        stages:
          - build_packages
-          - build_macos_tests

 extends:
  template: templates/pipelines/run-tests-pipeline.yml
--- a/tools/devops/automation/templates/build/build-mac-tests-stage.yml
+++ b/tools/devops/automation/templates/build/build-mac-tests-stage.yml
@ -39,7 +39,7 @@ jobs:
 # This job builds the macOS tests.
 - job: build_macos_tests_job
  displayName: 'Build macOS tests'
-  timeoutInMinutes: 120
+  timeoutInMinutes: 180
  variables:
    DOTNET_PLATFORMS: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.DOTNET_PLATFORMS'] ]
    ENABLE_DOTNET: $[ stageDependencies.configure_build.configure.outputs['configure_platforms.ENABLE_DOTNET'] ]
--- a/tools/devops/automation/templates/mac/build.yml
+++ b/tools/devops/automation/templates/mac/build.yml
@ -144,9 +144,8 @@ steps:
 # Go get the tests!, this depends on how the test was triggered
 - ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion'), contains(variables['Build.DefinitionName'], 'xamarin-macios-ci-tests'), contains(variables['Build.DefinitionName'], 'xamarin-macios-pr-tests')) }}:
  - download: macios
-    artifact: mac-test-package
    displayName: Download Mac tests
-    patterns: '**/mac-test-package.7z'
+    artifact: 'mac-test-package'

  # the default location when downloading is $(Pipeline.Workspace)/<pipeline resource identifier>/<artifact name>
  - pwsh: |
@ -162,7 +161,7 @@ steps:
  - task: DownloadPipelineArtifact@2
    displayName: Download Mac tests
    inputs:
-      patterns: '**/mac-test-package.7z'
+      artifact: 'mac-test-package'
      allowFailedBuilds: true
      path: $(Build.SourcesDirectory)/artifacts/tmp

--- a/tools/devops/automation/templates/mac/stage.yml
+++ b/tools/devops/automation/templates/mac/stage.yml
@ -39,7 +39,13 @@ parameters:
  type: boolean
  default: false

+- name: stageDisplayNamePrefix
+  type: string
+  default: ''
+
+
 stages:
+
 - stage: ${{ parameters.stageName }}
  displayName: ${{ parameters.displayName }}
  dependsOn:
--- a/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml
+++ b/tools/devops/automation/templates/pipelines/run-macos-tests-pipeline.yml
@ -0,0 +1,282 @@
+# template to be extended by those pipelines that will run tests after a build.
+parameters:
+
+  - name: isPR
+    displayName: State if the tests are ran for a PR build
+    type: boolean
+    default: false
+
+  - name: repositoryAlias
+    type: string
+    default: self
+
+  - name: commit
+    type: string
+    default: HEAD
+
+  - name: stageDisplayNamePrefix
+    type: string
+    default: ''
+
+  - name: macTestsConfigurations
+    displayName: macOS test configurations to run
+    type: object
+    default: [
+      {
+        stageName: 'mac_11_m1',
+        displayName: 'M1 - Mac Big Sur (11)',
+        macPool: 'VSEng-VSMac-Xamarin-Shared',
+        useImage: false,
+        statusContext: 'M1 - Mac Big Sur (11)',
+        demands: [
+          "Agent.OS -equals Darwin",
+          "macOS.Name -equals BigSur",
+          "macOS.Architecture -equals arm64",
+          "Agent.HasDevices -equals False",
+          "Agent.IsPaired -equals False"
+        ]
+      },
+      {
+        stageName: 'mac_12_m1',
+        displayName: 'M1 - Mac Ventura (12)',
+        macPool: 'VSEng-VSMac-Xamarin-Shared',
+        useImage: false,
+        statusContext: 'M1 - Mac Monterey (12)',
+        demands: [
+          "Agent.OS -equals Darwin",
+          "macOS.Name -equals Monterey",
+          "macOS.Architecture -equals arm64",
+          "Agent.HasDevices -equals False",
+          "Agent.IsPaired -equals False"
+        ]
+      },
+      {
+        stageName: 'mac_13_m1',
+        displayName: 'M1 - Mac Ventura (13)',
+        macPool: 'VSEng-VSMac-Xamarin-Shared',
+        useImage: false,
+        statusContext: 'M1 - Mac Ventura (13)',
+        demands: [
+          "Agent.OS -equals Darwin",
+          "macOS.Name -equals Ventura",
+          "macOS.Architecture -equals arm64",
+          "Agent.HasDevices -equals False",
+          "Agent.IsPaired -equals False"
+        ]
+      },
+      {
+        stageName: 'mac_14_x64',
+        displayName: 'X64 - Mac Sonoma (14)',
+        macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted',
+        useImage: false,
+        statusContext: 'X64 - Mac Sonoma (14)',
+        demands: [
+          "Agent.OS -equals Darwin",
+          "macOS.Name -equals Sonoma",
+          "macOS.Architecture -equals x64",
+          "Agent.HasDevices -equals False",
+          "Agent.IsPaired -equals False"
+        ]
+      }]
+
+  - name: pool
+    type: string
+    default: automatic
+    values:
+      - pr
+      - ci
+      - automatic
+
+  - name: supportedPlatforms
+    type: object
+    default: [
+      {
+        platform: iOS,
+        isDotNetPlatform: true,
+      },
+      {
+        platform: macOS,
+        isDotNetPlatform: true,
+      },
+      {
+        platform: tvOS,
+        isDotNetPlatform: true,
+      },
+      {
+        platform: MacCatalyst,
+        isDotNetPlatform: true,
+      },
+      {
+        # when running platform-specific test runs, we also need a special test run that executes tests that only runs when multiple platforms are enabled
+        platform: Multiple,
+        isDotNetPlatform: true,
+      }
+    ]
+
+
+  - name: testConfigurations
+    type: object
+    default: [
+      # Disabled by default #
+      # {
+      #   label: bcl,
+      #   splitByPlatforms: false,
+      # },
+      {
+        label: cecil,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: dotnettests,
+        splitByPlatforms: true,
+        needsMultiplePlatforms: true,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: fsharp,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: framework,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: generator,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: interdependent-binding-projects,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: introspection,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: linker,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: mmp,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: monotouch,
+        splitByPlatforms: true,
+        needsMultiplePlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: msbuild,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: xcframework,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+      {
+        label: xtro,
+        splitByPlatforms: false,
+        testPrefix: 'simulator_tests',
+      },
+    ]
+
+resources:
+  repositories:
+    - repository: self
+      checkoutOptions:
+        submodules: true
+
+    - repository: yaml-templates
+      type: github
+      name: xamarin/yaml-templates
+      ref: refs/heads/main
+      endpoint: xamarin
+
+    - repository: maccore
+      type: github
+      name: xamarin/maccore
+      ref: refs/heads/main
+      endpoint: xamarin
+
+    - repository: release-scripts
+      type: github
+      name: xamarin/release-scripts
+      ref: refs/heads/only_codesign
+      endpoint: xamarin
+
+variables:
+  - ${{ if contains(variables['Build.DefinitionName'], 'private') }}:
+      - template: ../vsts-variables.yml
+  - template: ../variables.yml
+  - name: MaciosUploadPrefix
+    value: ''
+  - name: DisablePipelineConfigDetector
+    value: true
+
+stages:
+
+  - stage: configure_build
+    displayName: '${{ parameters.stageDisplayNamePrefix }}Configure'
+    jobs:
+
+      - ${{ if eq(parameters.pool, 'automatic') }}:
+          - job: AgentPoolSelector       # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml
+            pool:                        # Consider using an agentless (server) job here, but would need to host selection logic as an Azure function: https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#server
+              vmImage: ubuntu-latest
+            steps:
+              - checkout: none             # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout
+
+              # Selects appropriate agent pool based on trigger type (PR or CI); manually triggered builds target the PR pool
+              - template: azure-devops-pools/agent-pool-selector.yml@yaml-templates
+                parameters:
+                  agentPoolPR: $(PRBuildPool)
+                  agentPoolPRUrl: $(PRBuildPoolUrl)
+                  agentPoolCI: $(CIBuildPool)
+                  agentPoolCIUrl: $(CIBuildPoolUrl)
+
+      - job: configure
+        displayName: 'Configure build'
+        pool:
+          vmImage: windows-latest
+
+        variables:
+          isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
+          isScheduled: $[eq(variables['Build.Reason'], 'Schedule')]
+          BRANCH_NAME: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
+
+        steps:
+          - template: ../common/load_configuration.yml
+            parameters: 
+              repositoryAlias: ${{ parameters.repositoryAlias }}
+              commit: ${{ parameters.commit }}
+              testConfigurations: ${{ parameters.testConfigurations }}
+              supportedPlatforms: ${{ parameters.supportedPlatforms }}
+              testsLabels: '--label=skip-all-tests,run-ios-tests,run-ios-simulator-tests,run-tvos-tests,run-watchos-tests,run-mac-tests,run-maccatalyst-tests,run-dotnet-tests,run-system-permission-tests,run-legacy-xamarin-tests'
+              statusContext: 'VSTS: simulator tests' 
+              uploadArtifacts: true
+
+  - ${{ each config in parameters.macTestsConfigurations }}:
+      - template: ../mac/stage.yml
+        parameters:
+          isPR: ${{ parameters.isPR }}
+          repositoryAlias: ${{ parameters.repositoryAlias }}
+          commit: ${{ parameters.commit }}
+          stageName: ${{ config.stageName }}
+          displayName: ' ${{ parameters.stageDisplayNamePrefix }}${{ config.displayName }}'
+          macPool: ${{ config.macPool }}
+          useImage: ${{ config.useImage }}
+          statusContext: ${{ config.statusContext }}
+          keyringPass: $(pass--lab--mac--builder--keychain)
+          demands: ${{ config.demands }}
+          postPipeline: true
--- a/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml
+++ b/tools/devops/automation/templates/pipelines/run-tests-pipeline.yml
@ -84,68 +84,6 @@ parameters:
        ]
      }]

-  - name: macTestsConfigurations
-    displayName: macOS test configurations to run
-    type: object
-    default: [
-      {
-        stageName: 'mac_11_m1',
-        displayName: 'M1 - Mac Big Sur (11)',
-        macPool: 'VSEng-VSMac-Xamarin-Shared',
-        useImage: false,
-        statusContext: 'M1 - Mac Big Sur (11)',
-        demands: [
-          "Agent.OS -equals Darwin",
-          "macOS.Name -equals BigSur",
-          "macOS.Architecture -equals arm64",
-          "Agent.HasDevices -equals False",
-          "Agent.IsPaired -equals False"
-        ]
-      },
-      {
-        stageName: 'mac_12_m1',
-        displayName: 'M1 - Mac Ventura (12)',
-        macPool: 'VSEng-VSMac-Xamarin-Shared',
-        useImage: false,
-        statusContext: 'M1 - Mac Monterey (12)',
-        demands: [
-          "Agent.OS -equals Darwin",
-          "macOS.Name -equals Monterey",
-          "macOS.Architecture -equals arm64",
-          "Agent.HasDevices -equals False",
-          "Agent.IsPaired -equals False"
-        ]
-      },
-      {
-        stageName: 'mac_13_m1',
-        displayName: 'M1 - Mac Ventura (13)',
-        macPool: 'VSEng-VSMac-Xamarin-Shared',
-        useImage: false,
-        statusContext: 'M1 - Mac Ventura (13)',
-        demands: [
-          "Agent.OS -equals Darwin",
-          "macOS.Name -equals Ventura",
-          "macOS.Architecture -equals arm64",
-          "Agent.HasDevices -equals False",
-          "Agent.IsPaired -equals False"
-        ]
-      },
-      {
-        stageName: 'mac_14_x64',
-        displayName: 'X64 - Mac Sonoma (14)',
-        macPool: 'VSEng-Xamarin-RedmondMacBuildPool-iOS-Untrusted',
-        useImage: false,
-        statusContext: 'X64 - Mac Sonoma (14)',
-        demands: [
-          "Agent.OS -equals Darwin",
-          "macOS.Name -equals Sonoma",
-          "macOS.Architecture -equals x64",
-          "Agent.HasDevices -equals False",
-          "Agent.IsPaired -equals False"
-        ]
-      }]
-
-
 resources:
  repositories:
    - repository: self
@ -189,11 +127,9 @@ stages:
      pool: ${{ parameters.pool }}
      runTests: ${{ parameters.runTests }}
      runDeviceTests: ${{ parameters.runDeviceTests }}
-      runOldMacOSTests: ${{ parameters.runOldMacOSTests }}
      runWindowsIntegration: ${{ parameters.runWindowsIntegration }}
      runSamples: ${{ parameters.runSamples }}
      ${{ if ne(length(parameters.testConfigurations), 0)}}:
        testConfigurations: ${{ parameters.testConfigurations }}
      deviceTestsConfigurations: ${{ parameters.deviceTestsConfigurations }}
-      macTestsConfigurations: ${{ parameters.macTestsConfigurations }}

--- a/tools/devops/automation/templates/tests-stage.yml
+++ b/tools/devops/automation/templates/tests-stage.yml
@ -20,10 +20,6 @@ parameters:
  type: boolean
  default: false

- name: runOldMacOSTests
-  type: boolean
-  default: true
-
 - name: runWindowsIntegration
  type: boolean
  default: true
@ -157,9 +153,6 @@ parameters:
 - name: deviceTestsConfigurations
  type: object

- name: macTestsConfigurations
-  type: object
-
 - name: stageDisplayNamePrefix
  type: string
  default: ''
@ -265,22 +258,6 @@ stages:
    commit: ${{ parameters.commit }}
    postPipeline: true

- ${{ if eq(parameters.runOldMacOSTests, true) }}:
-  - ${{ each config in parameters.macTestsConfigurations }}:
-    - template: ./mac/stage.yml
-      parameters:
-        isPR: ${{ parameters.isPR }}
-        repositoryAlias: ${{ parameters.repositoryAlias }}
-        commit: ${{ parameters.commit }}
-        stageName: ${{ config.stageName }}
-        displayName: ' ${{ parameters.stageDisplayNamePrefix }}${{ config.displayName }}'
-        macPool: ${{ config.macPool }}
-        useImage: ${{ config.useImage }}
-        statusContext: ${{ config.statusContext }}
-        keyringPass: $(pass--lab--mac--builder--keychain)
-        demands: ${{ config.demands }}
-        postPipeline: true
-
 - ${{ if eq(parameters.runWindowsIntegration, true) }}:
  - template: ./windows/stage.yml
    parameters: