spark/azure-pipelines.yml

# Spark .NET build

trigger:
  batch: true
  branches:
    include:
    - main

variables:
  buildConfiguration: 'Release'
  _SignType: real
  _TeamName: DotNetSpark
  MSBUILDSINGLELOADCONTEXT: 1
  ArtifactPath: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries'

  backwardCompatibleRelease: '2.0.0'
  forwardCompatibleRelease: '2.0.0'

  backwardCompatibleTestOptions_Windows_2_4: ""
  forwardCompatibleTestOptions_Windows_2_4: ""
  backwardCompatibleTestOptions_Linux_2_4: ""
  forwardCompatibleTestOptions_Linux_2_4: ""

  backwardCompatibleTestOptions_Windows_3_0: ""
  forwardCompatibleTestOptions_Windows_3_0: ""
  backwardCompatibleTestOptions_Linux_3_0: ""
  forwardCompatibleTestOptions_Linux_3_0: ""

  backwardCompatibleTestOptions_Windows_3_1: ""
  forwardCompatibleTestOptions_Windows_3_1: ""
  backwardCompatibleTestOptions_Linux_3_1: ""
  forwardCompatibleTestOptions_Linux_3_1: ""

  # Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
  backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
  forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
  backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
  forwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)

  # Azure DevOps variables are transformed into environment variables, with these variables we
  # avoid the first time experience and telemetry to speed up the build.
  DOTNET_CLI_TELEMETRY_OPTOUT: 1
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1

resources:
  repositories:
  - repository: forwardCompatibleRelease
    type: github
    endpoint: public
    name: dotnet/spark
    ref: refs/tags/v$(forwardCompatibleRelease)

stages:
- stage: Build
  displayName: Build Sources
  jobs:
  - job: Build
    pool: 
      vmImage: 'windows-2022'

    variables:
      ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
        _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)

    steps:
    - task: Maven@3
      displayName: 'Maven build src'
      inputs:
        mavenPomFile: src/scala/pom.xml

    - task: Maven@3
      displayName: 'Maven build benchmark'
      inputs:
        mavenPomFile: benchmark/scala/pom.xml

    - script: build.cmd -pack
                -c $(buildConfiguration)
                -ci
                $(_OfficialBuildIdArgs)
                /p:PublishSparkWorker=true
                /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker
      displayName: '.NET build'

    - task: DotNetCoreCLI@2
      displayName: '.NET unit tests'
      inputs:
        command: test
        projects: '**/*UnitTest/*.csproj'
        arguments: '--configuration $(buildConfiguration)'

    - task: CopyFiles@2
      displayName: Stage Maven build jars
      inputs:
        sourceFolder: $(Build.SourcesDirectory)/src/scala
        contents: '**/*.jar'
        targetFolder: $(Build.ArtifactStagingDirectory)/Jars

    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
      - task: CopyFiles@2
        displayName: Stage .NET artifacts
        inputs:
          sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping
          contents: |
            **/*.nupkg
            **/*.snupkg            
          targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping

      - task: CopyFiles@2
        displayName: Stage build logs
        inputs:
          sourceFolder: $(Build.SourcesDirectory)/artifacts/log
          targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/log

    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
        artifactName:  Microsoft.Spark.Binaries

  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
    - job: Sign
      dependsOn:
        - Build
      displayName: Sign Artifacts
      pool:
        name: NetCore1ESPool-Internal
        demands: ImageOverride -equals build.windows.10.amd64.vs2019

      variables:
        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
          _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)

      steps:
      - task: DownloadBuildArtifacts@0
        displayName: Download Build Artifacts
        inputs:
          artifactName: Microsoft.Spark.Binaries
          downloadPath: $(Build.ArtifactStagingDirectory)

      - task: MicroBuildSigningPlugin@2
        displayName: Install MicroBuild plugin
        inputs:
          signType: $(_SignType)
          zipSources: false
          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
        env:
          TeamName: $(_TeamName)
        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))

      - task: PowerShell@2
        displayName: Sign artifacts
        inputs:
          filePath: eng\common\build.ps1
          arguments: -restore -sign
                     -c $(buildConfiguration)
                     -ci
                     $(_OfficialBuildIdArgs)
                     /p:DotNetSignType=$(_SignType)
                     /p:SparkPackagesDir=$(ArtifactPath)\BuildArtifacts\artifacts\packages
                     /p:SparkWorkerPublishDir=$(ArtifactPath)\Microsoft.Spark.Worker

      - task: PublishBuildArtifacts@1
        inputs:
          pathtoPublish: '$(ArtifactPath)'
          artifactName:  Microsoft.Spark.Binaries

  # The "Publish" stage is separated out from the "Sign" stage because we need to install powershell module
  # to zip files correctly for macOS; installing the module is not allowed in NetCoreInternal-Pool.
  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
    - job: Publish
      dependsOn:
        - Sign
      displayName: Publish Artifacts
      pool: 
        vmImage: 'windows-2022'

      variables:
        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
          _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)

      steps:
      # The following module needs to be installed to zip files correctly for macOS.
      - powershell: Install-Module -Name Microsoft.PowerShell.Archive -Scope CurrentUser -Force -AllowClobber -Verbose -MinimumVersion 1.2.5
        displayName: Install Microsoft.PowerShell.Archive

      - task: DownloadBuildArtifacts@0
        displayName: Download Signed Artifacts
        inputs:
          artifactName: Microsoft.Spark.Binaries
          downloadPath: $(Build.ArtifactStagingDirectory)

      - task: PowerShell@2
        displayName: Package Microsoft.Spark.Worker
        inputs:
          filePath: eng\common\build.ps1
          arguments: -restore -publish
                     -c $(buildConfiguration)
                     -ci
                     $(_OfficialBuildIdArgs)
                     /p:SparkWorkerPublishDir=$(ArtifactPath)\Microsoft.Spark.Worker
                     /p:SparkWorkerPackageOutputDir=$(ArtifactPath)

      - task: PublishBuildArtifacts@1
        inputs:
          pathtoPublish: '$(ArtifactPath)'
          artifactName:  Microsoft.Spark.Binaries

- template: azure-pipelines-e2e-tests-template.yml
  parameters:
    backwardCompatibleRelease: $(backwardCompatibleRelease)
    forwardCompatibleRelease: $(forwardCompatibleRelease)
    tests:
    - version: '2.4.0'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.1'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.3'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.4'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.5'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.6'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.7'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '2.4.8'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
    - version: '3.0.0'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
    - version: '3.0.1'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
    - version: '3.0.2'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_0)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
    - version: '3.1.1'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_1)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
    - version: '3.1.2'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_1)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
    - version: '3.2.0'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
    - version: '3.2.1'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
    - version: '3.2.2'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
    - version: '3.2.3'
      enableForwardCompatibleTests: false
      enableBackwardCompatibleTests: false
      jobOptions:
      - pool: 'Windows'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
      - pool: 'Linux'
        testOptions: ""
        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)