xamarin-macios/jenkins/Jenkinsfile

#!/bin/groovy

// global variables
repository = "xamarin/xamarin-macios"
commentFile = null
isPr = false
branchName = null
gitHash = null
packagePrefix = null
virtualPath = null
xiPackageUrl = null
xmPackageUrl = null
xiNotarizedPackageUrl = null
xmNotarizedPackageUrl = null
utils = null
errorMessage = null
currentStage = null
failedStages = []
workspace = null
manualException = false

xiPackageFilename = null
xmPackageFilename = null
xiNotarizedPkgFilename = null
xmNotarizedPkgFilename = null
msbuildZipFilename = null
bundleZipFilename = null
manifestFilename = null
artifactsFilename = null
reportPrefix = null
createFinalStatus = true
skipLocalTestRunReason = ""

@NonCPS
def getRedirect () {
    def connection = "${env.XAMARIN_BUILD_TASKS}".toURL().openConnection()
    connection.instanceFollowRedirects = false
    // should be in scope at the time of this call (called within a withCredentials block)
    connection.setRequestProperty("Authorization", "token ${env.GITHUB_AUTH_TOKEN}")
    def response = connection.responseCode
    connection.disconnect()
    if (response == 302) {
        return connection.getHeaderField("Location")
    } else {
        throw new Exception("DL link failed ${response}: ${content}")
    }
}

github_pull_request_info = null
def githubGetPullRequestInfo ()
{
    if (github_pull_request_info == null && isPr) {
        withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_PAT_TOKEN')]) {
            def url = "https://api.github.com/repos/${repository}/pulls/${env.CHANGE_ID}"
            def outputFile = ".github-pull-request-info.json"
            try {
                sh ("curl -vf -H 'Authorization: token ${GITHUB_PAT_TOKEN}' --output '${outputFile}' '${url}'")
                github_pull_request_info = readJSON (file: outputFile)
                echo ("Got pull request info: ${github_pull_request_info}")
            } finally {
                sh ("rm -f ${outputFile}")
            }
        }

    }
    return github_pull_request_info
}

github_pull_request_labels = null
def githubGetPullRequestLabels ()
{
    if (github_pull_request_labels == null) {
        github_pull_request_labels = []
        if (isPr) {
            def pinfo = githubGetPullRequestInfo ()
            def labels = pinfo ["labels"]
            if (labels != null) {
                for (int i = 0; i < labels.size (); i++) {
                    def label = labels [i]
                    github_pull_request_labels.add (label ["name"])
                }
            }
            echo ("Found labels ${github_pull_request_labels} for the pull request.")
        }
    }
    return github_pull_request_labels
}

def githubAddComment (url, markdown)
{
    def json = groovy.json.JsonOutput.toJson ([body: markdown])
    def jsonFile = "${workspace}/xamarin-macios/jenkins/commit-comments.json"
    try {
        writeFile (file: "${jsonFile}", text: "${json}")
        sh ("cat '${jsonFile}'")
        withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_COMMENT_TOKEN')]) {
            sh ("curl -i -H 'Authorization: token ${GITHUB_COMMENT_TOKEN}' ${url} --data '@${jsonFile}'")
        }
    } finally {
        sh ("rm -f ${jsonFile}")
    }
}

def commentOnCommit (commitHash, markdown)
{
    githubAddComment ("https://api.github.com/repos/${repository}/commits/${commitHash}/comments", markdown)
}

def commentOnPullRequest (pullRequest, markdown)
{
    githubAddComment ("https://api.github.com/repos/${repository}/issues/${pullRequest}/comments", markdown)
}

def addComment (markdown)
{
    if (isPr) {
        commentOnPullRequest ("${env.CHANGE_ID}", markdown)
    } else {
        commentOnCommit ("${gitHash}", markdown)
    }
}

def appendFileComment (comment)
{
    if (fileExists (commentFile))
        comment = readFile (commentFile) + comment

    writeFile (file: commentFile, text: comment)
}

def markdownToSlack (value)
{
    def tmpfile = "/tmp/slacker.md"
    writeFile (file: tmpfile, text: value)
    try {
        // this monstruousity converts markdown's [text](url) to slack's <url|text>
        sh ("sed -i '' 's/[[]\\(.*\\)[]][\\(]\\(.*\\)[\\)]/<\\2|\\1>/' '${tmpfile}'")
        value = readFile (tmpfile)
    } finally {
        sh ("rm -f '${tmpfile}'")
    }
    return value
}

def reportFinalStatusToSlack (err, gitHash, currentStage, fileContents)
{
    def status = currentBuild.currentResult
    if ("${status}" == "SUCCESS" && err == "")
        return // not reporting success to slack

    try {
        def authorName = null
        def authorEmail = null
        if (isPr) {
            authorName = env.CHANGE_AUTHOR_DISPLAY_NAME
            authorEmail = env.CHANGE_AUTHOR_EMAIL
            slackMessage = "Pull Request #<${env.CHANGE_URL}|${env.CHANGE_ID}> failed to build."
        } else {
            authorName = sh (script: "cd ${workspace} && git log -1 --pretty=%an", returnStdout: true).trim ()
            authorEmail = sh (script: "cd ${workspace} && git log -1 --pretty=%ae", returnStdout: true).trim ()
            slackMessage = "Commit <https://github.com/${repository}/commit/${gitHash}|${gitHash}> failed to build."
        }

        def title = null
        if (err != null) {
            title = "Internal jenkins failed in stage '${currentStage}': ${err}"
        } else {
            title = "Internal jenkins failed in stage '${currentStage}'"
        }
        def text = ""
        if (fileContents != null)
            text = "\"text\": ${groovy.json.JsonOutput.toJson (markdownToSlack (fileContents))},"
        // The attachments string must not start with a newline, it will produce a very helpful 'Invalid JSON String' exception with no additional info.
        def attachments = """[
            {
                \"author_name\": \"${authorName} (${authorEmail})\",
                \"title\": \"${title}\",
                \"title_link\": \"${env.RUN_DISPLAY_URL}\",
                \"color\": \"danger\",
                ${text}
                \"fallback\": \"Build failed\"
            }
        ]
        """
        echo (attachments)
        slackSend (botUser: true, channel: "#ios-notifications", color: "danger", message: slackMessage, attachments: attachments)
    } catch (e) {
        echo ("Failed to report to Slack: ${e}")
    }
}

def reportFinalStatus (err, gitHash, currentStage)
{
    if (!createFinalStatus)
        return

    def comment = ""
    def status = currentBuild.currentResult

    if ("${status}" == "SUCCESS" && err == "" && failedStages.size () == 0) {
        comment = "✅ [Jenkins job](${env.RUN_DISPLAY_URL}) (on internal Jenkins) succeeded"
    } else {
        // Aborted builds throw either a FlowInterruptedException or an AbortException (depending on what's executing),
        // so treat either as an abortion (both can be thrown in other circumstances as well, so it's just a best guess).
        if (err.contains ("FlowInterruptedException") || err.contains ("AbortException"))
            comment += "❌ [Build was (probably) aborted](${env.RUN_DISPLAY_URL})\n\n"
        comment += "🔥 [Jenkins job](${env.RUN_DISPLAY_URL}) (on internal Jenkins) failed"
        if (currentStage != "" && !failedStages.contains (currentStage))
            failedStages.add (currentStage)
        if (failedStages.size () > 0)
            comment += " in stage(s) '${failedStages.join (', ')}'"
        comment += " 🔥"
        if (!manualException && err != "")
            comment += " : ${err}"
        manager.addErrorBadge (comment)
        manager.buildFailure ()
    }

    def fileContents = null
    if (fileExists (commentFile)) {
        fileContents = readFile ("${commentFile}")
        comment += "\n\n" + fileContents
    }

    addComment ("${comment}")

    reportFinalStatusToSlack (err, gitHash, currentStage, fileContents)
}

def processAtMonkeyWrench (outputFile)
{
    def tmpfile = "atmonkeywrench.tmp"
    try {
        sh (script: "grep '^@MonkeyWrench: ...Summary: ' '${outputFile}' > ${tmpfile}", returnStatus: true /* don't throw exceptions if something goes wrong */)
        def lines = readFile ("${tmpfile}").split ("\n")
        for (int i = 0; i < lines.length; i++) {
            def summary = lines [i].substring (27 /*"@MonkeyWrench: AddSummary: ".length*/).trim ()
            summary = summary.replace ("<br/>", "")
            summary = summary.replace ("<a href='", "")
            def href_end = summary.indexOf ("'>")
            if (href_end > 0)
                summary = summary.substring (0, href_end)
            echo (summary)
        }
    } finally {
        sh ("rm -f '${tmpfile}'")
    }
}

def uploadFiles (glob, containerName, virtualPath)
{
    step ([
        $class: 'WAStoragePublisher',
        allowAnonymousAccess: true,
        cleanUpContainer: false,
        cntPubAccess: true,
        containerName: containerName,
        doNotFailIfArchivingReturnsNothing: false,
        doNotUploadIndividualFiles: false,
        doNotWaitForPreviousBuild: true,
        excludeFilesPath: '',
        filesPath: glob,
        storageAccName: 'bosstoragemirror',
        storageCredentialId: 'bc6a99d18d7d9ca3f6bf6b19e364d564',
        uploadArtifactsOnlyIfSuccessful: false,
        uploadZips: false,
        virtualPath: virtualPath,
        storageType: 'blobstorage'
    ])
}

@NonCPS
def getUpdateInfoAndVersion (file)
{
    def result = [:]
    def match = file =~ /xamarin.(mac|ios)-(\d+(\.\d+)+).*.pkg$/
    if (match.matches()) {
        result['platform'] = match[0][1]
        result['version'] = match[0][2]
    }
    return result
}

// There must be a better way than this hack to show a
// message in red in the Jenkins Blue Ocean UI...
def echoError (message)
{
    try {
        error (message)
    } catch (e) {
        // Ignore
    }
}

def indexOfElement (list, element)
{
    for (def i = 0; i < list.size (); i++) {
        if (list [i] == element)
            return i
    }
    return -1
}

def runXamarinMacTests (url, macOS, maccore_hash, xamarin_macios_hash)
{
    def failed = false
    def workspace = "${env.HOME}/jenkins/workspace/xamarin-macios"
    def failedTests = []
    try {
        echo ("Executing on ${env.NODE_NAME}")
        echo ("URL: ${url}")
        sh ("mkdir -p '${workspace}'")
        dir ("${workspace}") {
            // Download the script we need, and then execute it, so that we
            // don't clone all of xamarin-macios to get a single file.
            // Due to how github has implemented pull requests, this works
            // even for pull requests from forks, since each commit in the pull request
            // is also available from the main repository.
            sh ("""
                curl -fLO https://raw.githubusercontent.com/xamarin/xamarin-macios/${xamarin_macios_hash}/jenkins/prepare-packaged-macos-tests.sh
                chmod +x prepare-packaged-macos-tests.sh
                ./prepare-packaged-macos-tests.sh '${url}' '${maccore_hash}'
                """)
            def tests = [ "dontlink", "apitest", "introspection", "linksdk", "linkall", "xammac_tests" ];
            tests.each { test ->
                def t = "${test}"
                try {
                    timeout (time: 10, unit: 'MINUTES') {
                        sh ("make -C mac-test-package/tests exec-mac-${t} MONO_DEBUG=no-gdb-backtrace")
                        echo ("${t} succeeded")
                    }
                } catch (error) {
                    echoError ("${t} failed with error: ${error}")
                    failed = true
                    failedTests.add (t)
                }
            }
            // Run dontlink using the oldest system mono we support
            def t = "dontlink (system)"
            try {
                // install oldest supported mono
                sh ("./prepare-packaged-macos-tests.sh --install-old-mono")
                // run dontlink tests using the system mono
                sh ("make -C mac-test-package/tests exec-mac-system-dontlink")
            } catch (error) {
                echoError ("${t} failed with error: ${error}")
                failed = true
                failedTests.add (t)
            }
        }
    } finally {
        sh ("rm -rf ${workspace}/mac-test-package ${workspace}/*.zip ${workspace}/*.7z")
        // Find and upload any crash reports
        sh (script: """
            rm -Rf ${workspace}/crash-reports
            mkdir ${workspace}/crash-reports
            find ~/Library/Logs/DiagnosticReports/ -mtime -10m -exec cp {} ${workspace}/crash-reports \\;
            ls -la ${workspace}/crash-reports
            """,
            returnStatus: true /* Don't fail if something goes wrong */)
        try {
            if (findFiles (glob: "crash-reports/*").length > 0) {
                archiveArtifacts ("${workspace}/crash-reports/*")
            } else {
                echo ("No crash reports found")
            }
        } catch (e) {
            // Ignore any archiving errors.
        }
    }
    if (failed) {
        def failureMessage = "Xamarin.Mac tests on ${macOS} failed (${failedTests.join (', ')})"
        manager.addErrorBadge (failureMessage)
        error (failureMessage)
    }
}

def abortExecutingBuilds ()
{
    def job = Jenkins.instance.getItemByFullName (env.JOB_NAME)
    for (build in job.builds) {
        if (!build.isBuilding ())
            continue
        echo ("Current build: ${currentBuild.number} Checking build: ${build.number}");
        if (build.number > currentBuild.number) {
            error ("There is already a newer build in progress (#${build.number})")
        } else if (build.number < currentBuild.number) {
            def exec = build.getExecutor ()
            if (exec == null) {
                echo ("No executor for build ${build.number}")
            } else {
                exec.interrupt (Result.ABORTED, new CauseOfInterruption.UserInterruption ("Aborted by build #${currentBuild.number}"))
                echo ("Aborted previous build: #${build.number}")
            }
        }
    }
}

timestamps {
    def mainMacOSVersion = 14
    node ("xamarin-macios && macos-10.${mainMacOSVersion}") {
        try {
            timeout (time: 15, unit: 'HOURS') {
                // Hard-code a workspace, since branch-based and PR-based
                // builds would otherwise use different workspaces, which
                // wastes a lot of disk space.
                workspace = "${env.HOME}/jenkins/workspace/xamarin-macios"
                commentFile = "${workspace}/xamarin-macios/jenkins/pr-comments.md"
                withEnv ([
                    "BUILD_REVISION=jenkins",
                    "PATH=/Library/Frameworks/Mono.framework/Versions/Current/Commands:${env.PATH}",
                    "WORKSPACE=${workspace}"
                    ]) {
                    sh ("mkdir -p '${workspace}/xamarin-macios'")
                    dir ("${workspace}/xamarin-macios") {
                        stage ('Checkout') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            sh ("env | sort") // Print out environment for debug purposes
                            scmVars = checkout scm
                            isPr = (env.CHANGE_ID && !env.CHANGE_ID.empty ? true : false)
                            branchName = env.BRANCH_NAME
                            if (isPr) {
                                gitHash = sh (script: "git log -1 --pretty=%H refs/remotes/origin/${env.BRANCH_NAME}", returnStdout: true).trim ()
                            } else {
                                gitHash = scmVars.GIT_COMMIT
                            }
                            // Make sure we start from scratch
                            sh (script: 'make git-clean-all', returnStatus: true /* don't throw exceptions if something goes wrong */)
                            // Make really, really sure
                            sh ('git clean -xffd')
                            sh ('git submodule foreach --recursive git clean -xffd')
                        }
                    }

                    if (isPr) {
                        def hasBuildPackage = githubGetPullRequestLabels ().contains ("build-package")
                        def hasRunInternalTests = githubGetPullRequestLabels ().contains ("run-internal-tests")

                        if (!hasBuildPackage && !hasRunInternalTests) {
                            // don't add a comment to the pull request, since the public jenkins will also add comments, which ends up being too much.
                            createFinalStatus = false
                            echo ("Build skipped because the pull request doesn't have either of the labels 'build-package' or 'run-internal-tests'.")
                            return
                        }


                        if (!hasRunInternalTests)
                            skipLocalTestRunReason = "Not running tests here because they're run on public Jenkins."

                        // only aborting PR builds, since for normal branches we want to build as much as possible by default.
                        abortExecutingBuilds ()
                    }

                    dir ("${workspace}") {
                        stage ('Provisioning') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            sh ("cd ${workspace}/xamarin-macios && ./configure --enable-xamarin")
                            sh ("${workspace}/xamarin-macios/jenkins/provision-deps.sh")
                        }

                        stage ('Build') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            timeout (time: 1, unit: 'HOURS') {
                                withEnv ([
                                    "CURRENT_BRANCH=${branchName}",
                                    "PACKAGE_HEAD_BRANCH=${branchName}"
                                    ]) {
                                    sh ("${workspace}/xamarin-macios/jenkins/build.sh --configure-flags --enable-xamarin")
                                }
                            }
                        }

                        stage ('Packaging') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            sh ("${workspace}/xamarin-macios/jenkins/build-package.sh")
                            sh (script: "ls -la ${workspace}/package", returnStatus: true /* don't throw exceptions if something goes wrong */)
                        }

                        stage ('Signing') {
                            def entitlements = "${workspace}/xamarin-macios/mac-entitlements.plist"
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            def xiPackages = findFiles (glob: "package/xamarin.ios-*.pkg")
                            if (xiPackages.length > 0) {
                                xiPackageFilename = xiPackages [0].name
                                echo ("Created Xamarin.iOS package: ${xiPackageFilename}")
                            }
                            def xmPackages = findFiles (glob: "package/xamarin.mac-*.pkg")
                            if (xmPackages.length > 0) {
                                xmPackageFilename = xmPackages [0].name
                                echo ("Created Xamarin.Mac package: ${xmPackageFilename}")
                            }
                            def msbuildZip = findFiles (glob: "package/msbuild.zip")
                            if (msbuildZip.length > 0)
                                msbuildZipFilename = msbuildZip [0].name
                            def bundleZip = findFiles (glob: "package/bundle.zip")
                            if (bundleZip.length > 0)
                                bundleZipFilename = bundleZip [0].name

                            if (isPr) {
                              withCredentials ([string (credentialsId: 'codesign_keychain_pw', variable: 'PRODUCTSIGN_KEYCHAIN_PASSWORD')]) {
                                sh ("${workspace}/xamarin-macios/jenkins/productsign.sh")
                              }
                            } else {
                                try {
                                    pkgs = xiPackages + xmPackages
                                    if (fileExists('release-scripts')) {
                                        dir('release-scripts') {
                                            sh ('git checkout sign-and-notarized && git pull')
                                        }
                                    } else {
                                        sh ('git clone git@github.com:xamarin/release-scripts -b sign-and-notarized')
                                    }
                                    withCredentials([string(credentialsId: 'codesign_keychain_pw', variable: 'KEYCHAIN_PASS'), string(credentialsId: 'team_id', variable: 'TEAM_ID'), string(credentialsId: 'application_id', variable: 'APP_ID'), string(credentialsId: 'installer_id', variable: 'INSTALL_ID'), usernamePassword(credentialsId: 'apple_account', passwordVariable: 'APPLE_PASS', usernameVariable: 'APPLE_ACCOUNT')]) {
                                        sh (returnStatus: true, script: "security create-keychain -p ${env.KEYCHAIN_PASS} login.keychain") // needed to repopulate the keychain
                                        sh ("security unlock-keychain -p ${env.KEYCHAIN_PASS} login.keychain")
                                        timeout (time: 90, unit: 'MINUTES') {
                                          sh ("python release-scripts/sign_and_notarize.py -a ${env.APP_ID} -i ${env.INSTALL_ID} -u ${env.APPLE_ACCOUNT} -p ${env.APPLE_PASS} -t ${env.TEAM_ID} -d package/notarized -e ${entitlements} -k login.keychain " + pkgs.flatten ().join (" "))
                                        }
                                    }

                                    def xiNotarizedPackages = findFiles (glob: "package/notarized/xamarin.ios-*.pkg")
                                    if (xiNotarizedPackages.length > 0) {
                                        xiNotarizedPkgFilename = xiNotarizedPackages [0].name
                                        echo ("Created notarized Xamarin.iOS package: ${xiNotarizedPkgFilename}")
                                    }
                                    def xmNotarizedPackages = findFiles (glob: "package/notarized/xamarin.mac-*.pkg")
                                    if (xmNotarizedPackages.length > 0) {
                                        xmNotarizedPkgFilename = xmNotarizedPackages [0].name
                                        echo ("Created notarized Xamarin.Mac package: ${xmNotarizedPkgFilename}")
                                    }
                                } catch (ex) {
                                    echo "Notarization failed:\n${ex.getMessage()}"
                                    for (def stack : ex.getStackTrace()) {
                                        echo "\t${stack}"
                                    }
                                    manager.addWarningBadge("PKGs are not notarized")
                                }
                            }
                        }

                        stage ('Generate Workspace Info') {
                            withCredentials([string (credentialsId: '92cf81e5-6db5-408c-8a0d-192b36200a16', variable: 'GITHUB_AUTH_TOKEN'), string (credentialsId: 'xamarin-build-tasks', variable: 'XAMARIN_BUILD_TASKS')]) {
                                def redirect = getRedirect ()
                                sh "curl -o Xamarin.Build.Tasks.nupkg \"${redirect}\""
                                dir("BuildTasks") {
                                    deleteDir ()
                                    sh "unzip ../Xamarin.Build.Tasks.nupkg"
                                    sh "mono tools/BuildTasks/build-tasks.exe workspaceinfo -p ${workspace} -o ${workspace}/package-internal/ci-checkout.json"
                                    sh "mono tools/BuildTasks/build-tasks.exe dependencyinfo -p ${workspace} -o ${workspace}/package-internal/dependency-info.json"
                                }
                            }
                        }

                        stage ('Upload to Azure') {
                            currentStage = "${STAGE_NAME}"
                            virtualPath = "jenkins/${branchName}/${gitHash}/${env.BUILD_NUMBER}"
                            packagePrefix = "https://bosstoragemirror.blob.core.windows.net/wrench/${virtualPath}/package"

                            // Create manifest
                            def uploadingFiles = findFiles (glob: "package/*")
                            def manifest = ""
                            for (int i = 0; i < uploadingFiles.length; i++) {
                                def file = uploadingFiles [i]
                                manifest += "${packagePrefix}/${file.name}\n"
                            }
                            manifest += "${packagePrefix}/artifacts.json\n"
                            manifest += "${packagePrefix}/manifest\n"
                            writeFile (file: "package/manifest", text: manifest)

                            sh ("ls -la package")
                            uploadFiles ("package/*", "wrench", virtualPath)

                            def notarized_files = findFiles (glob: "package/notarized/*")
                            if (notarized_files.size () > 0)
                              uploadFiles ("package/notarized/*", "wrench", virtualPath)
                            uploadFiles ("package-internal/*", "jenkins-internal", virtualPath)

                            // Also upload manifest to a predictable url (without the build number)
                            // This manifest will be overwritten in subsequent builds (for this [PR/branch]+hash combination)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${branchName}/${gitHash}")
                            // And also create a 'latest' version (which really means 'latest built', not 'latest hash', but it will hopefully be good enough)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${branchName}/latest")
                            // And finally create a per-hash manifest (which will be overwritten in rebuilds of the same hash)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${gitHash}")

                            manifestFilename = "manifest"
                        }

                        stage ('Generate artifacts.json') {
                            withCredentials ([string (credentialsId: '92cf81e5-6db5-408c-8a0d-192b36200a16', variable: 'GITHUB_AUTH_TOKEN'), usernamePassword (credentialsId: 'd146d4aa-a437-4633-908f-b455876c44d0', passwordVariable: 'STORAGE_PASSWORD', usernameVariable: 'STORAGE_ACCOUNT')]) {
                                dir ("BuildTasks") {
                                    sh "mono tools/BuildTasks/build-tasks.exe artifacts -s ${workspace}/xamarin-macios -a ${env.STORAGE_ACCOUNT} -c ${env.STORAGE_PASSWORD} -u wrench/${virtualPath}/package"
                                    artifactsFilename = "artifacts.json"
                                }
                            }
                        }

                        stage ('Publish builds to GitHub') {
                            currentStage = "${STAGE_NAME}"
                            utils = load ("${workspace}/xamarin-macios/jenkins/utils.groovy")
                            def packagesMessage = ""
                            if (xiPackageFilename != null) {
                                xiPackageUrl = "${packagePrefix}/${xiPackageFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.iOS', "${xiPackageUrl}", 'SUCCESS', "${xiPackageFilename}")
                                packagesMessage += "* [${xiPackageFilename} (Not notarized)](${xiPackageUrl})\n"
                            }
                            if (xmPackageFilename != null) {
                                xmPackageUrl = "${packagePrefix}/${xmPackageFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.Mac', "${xmPackageUrl}", 'SUCCESS', "${xmPackageFilename}")
                                packagesMessage += "* [${xmPackageFilename} (Not notarized)](${xmPackageUrl})\n"
                            }
                            if (xiNotarizedPkgFilename != null) {
                                xiNotarizedPackageUrl = "${packagePrefix}/notarized/${xiNotarizedPkgFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.iOS-notarized', "${xiNotarizedPackageUrl}", 'SUCCESS', "${xiNotarizedPkgFilename}")
                                packagesMessage += "* [${xiNotarizedPkgFilename} (Notarized)](${xiNotarizedPackageUrl})\n"
                            }
                            if (xmNotarizedPkgFilename != null) {
                                xmNotarizedPackageUrl = "${packagePrefix}/notarized/${xmNotarizedPkgFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.Mac-notarized', "${xmNotarizedPackageUrl}", 'SUCCESS', "${xmNotarizedPkgFilename}")
                                packagesMessage += "* [${xmNotarizedPkgFilename} (Notarized)](${xmNotarizedPackageUrl})\n"
                            }
                            if (manifestFilename != null) {
                                def manifestUrl = "${packagePrefix}/${manifestFilename}"
                                utils.reportGitHubStatus (gitHash, "${manifestFilename}", "${manifestUrl}", 'SUCCESS', "${manifestFilename}")
                            }
                            if (artifactsFilename != null) {
                                def artifactUrl = "${packagePrefix}/${artifactsFilename}"
                                utils.reportGitHubStatus (gitHash, "Jenkins: Artifacts", "${artifactUrl}", 'SUCCESS', "${artifactsFilename}")
                            }
                            if (bundleZipFilename != null) {
                                def bundleZipUrl = "${packagePrefix}/${bundleZipFilename}"
                                utils.reportGitHubStatus (gitHash, "bundle.zip", "${bundleZipUrl}", 'SUCCESS', "${bundleZipFilename}")
                            }
                            if (msbuildZipFilename != null) {
                                def msbuildZipUrl = "${packagePrefix}/${msbuildZipFilename}"
                                utils.reportGitHubStatus (gitHash, "msbuild.zip", "${msbuildZipUrl}", 'SUCCESS', "${msbuildZipFilename}")
                            }

                            if (packagesMessage != "")
                                appendFileComment ("✅ Packages: \n${packagesMessage}\n")
                        }

                        dir ('xamarin-macios') {
                            stage ('Launch external tests') {
                                currentStage = "${STAGE_NAME}"
                                // VSTS does not allow any branch name anymore, it has to be an existing branch.
                                // Since pull requests don't create branches in the xamarin org (that VSTS sees at least),
                                // I've created a 'pull-request' branch which we'll use for pull requests.
                                def vsts_branch = isPr ? "pull-request" : branchName;
                                if (isPr && !githubGetPullRequestLabels ().contains ("trigger-device-tests")) {
                                    echo "Currently not launching external tests for pull requests"
                                } else {
                                    def outputFile = "${workspace}/xamarin-macios/wrench-launch-external.output.tmp"
                                    try {
                                        withCredentials ([string (credentialsId: 'macios_provisionator_pat', variable: 'PROVISIONATOR_VSTS_PAT')]) {
                                            sh ("make -C ${workspace}/xamarin-macios/tests wrench-launch-external MAC_PACKAGE_URL=${xmPackageUrl} IOS_PACKAGE_URL=${xiPackageUrl} WRENCH_URL=${env.RUN_DISPLAY_URL} BUILD_REVISION=${gitHash} BUILD_LANE=${vsts_branch} BUILD_WORK_HOST=${env.NODE_NAME} 2>&1 | tee ${outputFile}")
                                        }
                                        processAtMonkeyWrench (outputFile)
                                    } catch (error) {
                                        echo ("🚫 Launching external tests failed: ${error} 🚫")
                                        manager.addWarningBadge ("Failed to launch external tests")
                                    } finally {
                                        sh ("rm -f '${outputFile}'")
                                    }
                                }

                                if (isPr && githubGetPullRequestLabels ().contains ("run-sample-tests")) {
                                    def outputFile = "${workspace}/xamarin-macios/wrench-launch-external.output.tmp"
                                    try {
                                        withCredentials ([string (credentialsId: 'macios_provisionator_pat', variable: 'PROVISIONATOR_VSTS_PAT')]) {
                                            sh ("make -C ${workspace}/maccore/tests/external wrench-launch-sample-builds 'BUILD_REVISION=${gitHash}' 'BUILD_BRANCH=${vsts_branch}' V=1 2>&1 | tee ${outputFile}")
                                        }
                                        processAtMonkeyWrench (outputFile)
                                    } catch (error) {
                                        echo ("🚫 Launching external sample tests failed: ${error} 🚫")
                                        manager.addWarningBadge ("Failed to launch external sample tests")
                                    } finally {
                                        sh ("rm -f '${outputFile}'")
                                    }
                                }
                            }

                            stage ('Install Provisioning Profiles') {
                                currentStage = "${STAGE_NAME}"
                                sh ("${workspace}/maccore/tools/install-qa-provisioning-profiles.sh")
                            }

                            stage ('Publish reports') {
                                currentStage = "${STAGE_NAME}"
                                reportPrefix = sh (script: "${workspace}/xamarin-macios/jenkins/publish-results.sh | grep '^Url Prefix: ' | sed 's/^Url Prefix: //'", returnStdout: true).trim ()
                                if (skipLocalTestRunReason == "") {
                                    echo ("Html report: ${reportPrefix}/tests/index.html")
                                } else {
                                    echo ("Html report: ${skipLocalTestRunReason}")
                                }
                                echo ("API diff (from stable): ${reportPrefix}/api-diff/index.html")
                                echo ("API diff (from previous commit / before pull request): ${reportPrefix}/apicomparison/api-diff.html")
                                echo ("Generator diff: ${reportPrefix}/generator-diff/index.html")
                            }

                            stage ('API diff') {
                                currentStage = "${STAGE_NAME}"
                                def apidiffResult = sh (script: "${workspace}/xamarin-macios/jenkins/build-api-diff.sh --publish", returnStatus: true)
                                if (apidiffResult != 0)
                                    manager.addWarningBadge ("Failed to generate API diff")
                                echo ("API diff (from stable): ${reportPrefix}/api-diff/index.html")
                            }

                            stage ('API & Generator comparison') {
                                currentStage = "${STAGE_NAME}"
                                def compareResult = sh (script: "${workspace}/xamarin-macios/jenkins/compare.sh --publish", returnStatus: true)
                                if (compareResult != 0)
                                    manager.addWarningBadge ("Failed to generate API / Generator diff")
                                echo ("API diff (from previous commit / before pull request): ${reportPrefix}/apicomparison/api-diff.html")
                                echo ("Generator diff: ${reportPrefix}/generator-diff/index.html")
                            }

                            def hasXamarinMacTests = true
                            stage ("Package XM tests") {
                                currentStage = "${STAGE_NAME}"
                                def exitCode = sh (script: "make -C ${workspace}/xamarin-macios/tests package-tests", returnStatus: true)
                                if (exitCode != 0) {
                                    hasXamarinMacTests = false
                                    echoError ("Failed to package Xamarin.Mac tests (exit code: ${exitCode}")
                                    failedStages.add (currentStage)
                                }
                                uploadFiles ("tests/*.7z", "wrench", virtualPath)
                            }

                            timeout (time: 13, unit: 'HOURS') {
                                // We run tests locally and on older macOS bots in parallel.
                                // The older macOS tests run quickly (and the bots should usually be idle),
                                // which means that the much longer normal (local) test run should take
                                // longer to complete (which is important since this will block until all tests
                                // have been run, even if any older macOS bots are busy doing other things, preventing
                                // our tests from running there), giving the older macOS bots plenty of
                                // time to finish their test runs.
                                stage ('Run tests parallelized') {
                                    def builders = [:]

                                    // Add test runs on older macOS versions
                                    if (hasXamarinMacTests) {
                                        def url = "https://bosstoragemirror.blob.core.windows.net/wrench/${virtualPath}/tests/mac-test-package.7z"
                                        def maccore_hash = sh (script: "grep NEEDED_MACCORE_VERSION ${workspace}/xamarin-macios/mk/xamarin.mk | sed 's/NEEDED_MACCORE_VERSION := //'", returnStdout: true).trim ()
                                        // Get the min and current macOS version from Make.config,
                                        // and calculate all the macOS versions in between (including both end points).
                                        // The trims/splits/toIntegers at the end are to select the minor part of the macOS version number,
                                        // then we just loop from the minor part of the min version to the minor part of the current version.
                                        def firstOS = sh (returnStdout: true, script: "grep ^MIN_OSX_SDK_VERSION= '${workspace}/xamarin-macios/Make.config' | sed 's/.*=//'").trim ().split ("\\.")[1].toInteger ()
                                        def lastOS = sh (returnStdout: true, script: "grep ^OSX_SDK_VERSION= '${workspace}/xamarin-macios/Make.config' | sed 's/.*=//'").trim ().split ("\\.")[1].toInteger ()
                                        def macOSes = []
                                        def excludedOSes = []
                                        for (os = firstOS; os <= lastOS; os++)
                                            macOSes.add (os)
                                        // If any macOS version needs to be excluded manually, it can be done like this (in this case to remove macOS 10.14):
                                        // Any macOS versions excluded like this still get a entry in the Jenkins UI, making it explicit that the OS version was skipped.
                                        //     excludedOSes.add (14)
                                        // Have in mind that the value in the list is only the minor part of the macOS version number.
                                        for (i = 0; i < macOSes.size (); i++) {
                                            def os = macOSes [i];
                                            def macOS = "${os}" // Need to bind the label variable before the closure
                                            def excluded = false
                                            def nodeText = "XM tests on 10.${macOS}"
                                            if (indexOfElement (excludedOSes, os) >= 0) {
                                                excluded = true
                                                nodeText = "ℹ️ XM tests not executed on 10.${macOS} ℹ️"
                                            } else if (os == mainMacOSVersion) {
                                                excluded = true
                                                nodeText = "ℹ️ XM tests not executed on a separate 10.${macOS} bot because they're already executed as a part of the main test run ℹ️"
                                            }
                                            builders [nodeText] = {
                                                try {
                                                    if (excluded) {
                                                        echo (nodeText)
                                                    } else {
                                                        node ("xamarin-macios && macos-10.${macOS}") {
                                                            stage ("Running XM tests on '10.${macOS}'") {
                                                                runXamarinMacTests (url, "macOS 10.${macOS}", maccore_hash, gitHash)
                                                            }
                                                        }
                                                    }
                                                } catch (err) {
                                                    currentStage = "Running XM tests on '10.${macOS}'"
                                                    def msg = "Xamarin.Mac tests on 10.${macOS} failed: " + err.getMessage ();
                                                    appendFileComment ("🔥 [${msg}](${env.RUN_DISPLAY_URL}) 🔥\n")
                                                    failedStages.add (currentStage)
                                                    throw err
                                                }
                                            }
                                        }
                                    }

                                    // Add standard test run
                                    builders ["All tests"] = {
                                        stage ('Run tests') {
                                            currentStage = "Test run"
                                            echo ("Building on ${env.NODE_NAME}")
                                            if (skipLocalTestRunReason != "") {
                                                echo (skipLocalTestRunReason)
                                                appendFileComment ("ℹ️ Test run skipped: ${skipLocalTestRunReason}\n")
                                            } else {
                                                def XHARNESS_GITHUB_TOKEN_FILE = "${workspace}/xamarin-macios/jenkins/.github-pat-token"
                                                withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_PAT_TOKEN')]) {
                                                    writeFile (file: "${XHARNESS_GITHUB_TOKEN_FILE}", text: "${GITHUB_PAT_TOKEN}")
                                                }
                                                echo ("Html report: ${reportPrefix}/tests/index.html")
                                                def runTestResult = null
                                                withEnv (["XHARNESS_GITHUB_TOKEN_FILE=${XHARNESS_GITHUB_TOKEN_FILE}"]) {
                                                    runTestResult = sh (script: "${workspace}/xamarin-macios/jenkins/run-tests.sh --target=wrench-jenkins --publish --keychain=xamarin-macios", returnStatus: true)
                                                }
                                                if (runTestResult != 0) {
                                                    failedStages.add (currentStage)
                                                    manualException = true
                                                    error ("Test run failed: ${reportPrefix}/tests/index.html")
                                                } else {
                                                    echo ("${currentStage} succeeded")
                                                }
                                            }
                                        }
                                    }

                                    // Run it all parallelized
                                    parallel builders
                                }
                            }

                            currentStage = ""
                        } // dir ("xamarin-macios")
                    } // dir ("${workspace}")
                }
                reportFinalStatus ("", "${gitHash}", "${currentStage}")
            } // timeout
        } catch (err) {
            reportFinalStatus ("${err}", "${gitHash}", "${currentStage}")
        } finally {
            stage ('Final tasks') {
                sh (script: "${workspace}/xamarin-macios/jenkins/publish-results.sh", returnStatus: true /* don't throw exceptions if something goes wrong */)
                sh (script: "make git-clean-all -C ${workspace}/xamarin-macios", returnStatus: true /* don't throw exceptions if something goes wrong */)
            }
        }
    } // node
} // timestamps