xamarin-macios/jenkins/Jenkinsfile

#!/bin/groovy

// global variables
repository = "xamarin/xamarin-macios"
commentFile = null
isPr = false
branchName = null
gitHash = null
packagePrefix = null
virtualPath = null
xiPackageUrl = null
xmPackageUrl = null
xiNotarizedPackageUrl = null
xmNotarizedPackageUrl = null
utils = null
errorMessage = null
currentStage = null
failedStages = []
workspace = null
manualException = false

xiPackageFilename = null
xmPackageFilename = null
xiNotarizedPkgFilename = null
xmNotarizedPkgFilename = null
msbuildZipFilename = null
bundleZipFilename = null
manifestFilename = null
artifactsFilename = null
reportPrefix = null
createFinalStatus = true
skipLocalTestRunReason = ""

@NonCPS
def getRedirect () {
    def connection = "${env.XAMARIN_BUILD_TASKS}".toURL().openConnection()
    connection.instanceFollowRedirects = false
    // should be in scope at the time of this call (called within a withCredentials block)
    connection.setRequestProperty("Authorization", "token ${env.GITHUB_AUTH_TOKEN}")
    def response = connection.responseCode
    connection.disconnect()
    if (response == 302) {
        return connection.getHeaderField("Location")
    } else {
        throw new Exception("Failed to get redirect link for Xamarin.Build.Tasks: ${response}")
    }
}

github_pull_request_info = null
def githubGetPullRequestInfo ()
{
    if (github_pull_request_info == null && isPr) {
        withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_PAT_TOKEN')]) {
            def url = "https://api.github.com/repos/${repository}/pulls/${env.CHANGE_ID}"
            def outputFile = ".github-pull-request-info.json"
            try {
                sh ("curl -vf -H 'Authorization: token ${GITHUB_PAT_TOKEN}' --output '${outputFile}' '${url}'")
                github_pull_request_info = readJSON (file: outputFile)
                echo ("Got pull request info: ${github_pull_request_info}")
            } finally {
                sh ("rm -f ${outputFile}")
            }
        }

    }
    return github_pull_request_info
}

github_pull_request_labels = null
def githubGetPullRequestLabels ()
{
    if (github_pull_request_labels == null) {
        github_pull_request_labels = []
        if (isPr) {
            def pinfo = githubGetPullRequestInfo ()
            def labels = pinfo ["labels"]
            if (labels != null) {
                for (int i = 0; i < labels.size (); i++) {
                    def label = labels [i]
                    github_pull_request_labels.add (label ["name"])
                }
            }
            echo ("Found labels ${github_pull_request_labels} for the pull request.")
        }
    }
    return github_pull_request_labels
}

custom_labels = null
def getCustomLabels ()
{
    if (custom_labels == null) {
        custom_labels = []
        def custom_labels_file = "${workspace}/xamarin-macios/jenkins/custom-labels.txt"
        if (fileExists (custom_labels_file)) {
            def contents = sh (script: "grep -v '^[[:space:]]*#' ${custom_labels_file} | grep -v '^[[:space:]]*\$' | tr \$'\n' ' ' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*\$//'", returnStdout: true)
            custom_labels += contents.tokenize (' ')
            echo ("Found labels ${custom_labels} in ${custom_labels_file}")
        } else {
            echo ("The custom labels file ${custom_labels_file} does not exist")
        }
    }
    return custom_labels
}

labels = null
def getLabels ()
{
    if (labels == null) {
        labels = []
        labels += githubGetPullRequestLabels ()
        labels += getCustomLabels ()
        echo ("Found labels: ${labels}")
    }
    return labels
}

def githubAddComment (url, markdown)
{
    if (markdown.length () > 32768) {
        // GitHub only allows adding comments < 65536 characters log, so lets cap at half that, which should be more than enough to say that something went horribly wrong
        markdown = markdown.substring (0, 32768)
    }

    def json = groovy.json.JsonOutput.toJson ([body: markdown])
    def jsonFile = "${workspace}/xamarin-macios/jenkins/commit-comments.json"
    try {
        writeFile (file: "${jsonFile}", text: "${json}")
        sh ("cat '${jsonFile}'")
        withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_COMMENT_TOKEN')]) {
            sh ("curl -i -H 'Authorization: token ${GITHUB_COMMENT_TOKEN}' ${url} --data '@${jsonFile}'")
        }
    } finally {
        sh ("rm -f ${jsonFile}")
    }
}

def commentOnCommit (commitHash, markdown)
{
    githubAddComment ("https://api.github.com/repos/${repository}/commits/${commitHash}/comments", markdown)
}

def commentOnPullRequest (pullRequest, markdown)
{
    githubAddComment ("https://api.github.com/repos/${repository}/issues/${pullRequest}/comments", markdown)
}

def addComment (markdown)
{
    if (isPr) {
        commentOnPullRequest ("${env.CHANGE_ID}", markdown)
    } else {
        commentOnCommit ("${gitHash}", markdown)
    }
}

def appendFileComment (comment)
{
    if (fileExists (commentFile))
        comment = readFile (commentFile) + comment

    writeFile (file: commentFile, text: comment)
}

def markdownToSlack (value)
{
    def tmpfile = "/tmp/slacker.md"
    writeFile (file: tmpfile, text: value)
    try {
        // this monstruousity converts markdown's [text](url) to slack's <url|text>
        sh ("sed -i '' 's/[[]\\(.*\\)[]][\\(]\\(.*\\)[\\)]/<\\2|\\1>/' '${tmpfile}'")
        value = readFile (tmpfile)
    } finally {
        sh ("rm -f '${tmpfile}'")
    }
    return value
}

def reportFinalStatusToSlack (err, gitHash, currentStage, fileContents)
{
    def status = currentBuild.currentResult
    if ("${status}" == "SUCCESS" && err == "")
        return // not reporting success to slack

    try {
        def authorName = null
        def authorEmail = null
        if (isPr) {
            authorName = env.CHANGE_AUTHOR_DISPLAY_NAME
            authorEmail = env.CHANGE_AUTHOR_EMAIL
            slackMessage = "Pull Request #<${env.CHANGE_URL}|${env.CHANGE_ID}> failed to build."
        } else {
            authorName = sh (script: "cd ${workspace}/xamarin-macios && git log -1 --pretty=%an", returnStdout: true).trim ()
            authorEmail = sh (script: "cd ${workspace}/xamarin-macios && git log -1 --pretty=%ae", returnStdout: true).trim ()
            slackMessage = "Commit <https://github.com/${repository}/commit/${gitHash}|${gitHash}> failed to build."
        }

        def title = null
        if (err != null) {
            title = "Internal jenkins failed in stage '${currentStage}': ${err}"
        } else {
            title = "Internal jenkins failed in stage '${currentStage}'"
        }
        def text = ""
        if (fileContents != null)
            text = "\"text\": ${groovy.json.JsonOutput.toJson (markdownToSlack (fileContents))},"
        // The attachments string must not start with a newline, it will produce a very helpful 'Invalid JSON String' exception with no additional info.
        def attachments = """[
            {
                \"author_name\": \"${authorName} (${authorEmail})\",
                \"title\": \"${title}\",
                \"title_link\": \"${env.RUN_DISPLAY_URL}\",
                \"color\": \"danger\",
                ${text}
                \"fallback\": \"Build failed\"
            }
        ]
        """
        echo (attachments)
        slackSend (botUser: true, channel: "#ios-notifications", color: "danger", message: slackMessage, attachments: attachments)
    } catch (e) {
        echo ("Failed to report to Slack: ${e}")
    }
}

def reportFinalStatus (err, gitHash, currentStage)
{
    if (!createFinalStatus)
        return

    def comment = ""
    def status = currentBuild.currentResult

    if ("${status}" == "SUCCESS" && err == "" && failedStages.size () == 0) {
        comment = "✅ [Jenkins job](${env.RUN_DISPLAY_URL}) (on internal Jenkins) succeeded"
    } else {
        // Aborted builds throw either a FlowInterruptedException or an AbortException (depending on what's executing),
        // so treat either as an abortion (both can be thrown in other circumstances as well, so it's just a best guess).
        if (err.contains ("FlowInterruptedException") || err.contains ("AbortException"))
            comment += "❌ [Build was (probably) aborted](${env.RUN_DISPLAY_URL})\n\n"
        comment += "🔥 [Jenkins job](${env.RUN_DISPLAY_URL}) (on internal Jenkins) failed"
        if (currentStage != "" && !failedStages.contains (currentStage))
            failedStages.add (currentStage)
        if (failedStages.size () > 0) {
            // Somehow the same stage can end up in the list of failed stages multiple times. Avoid showing that to the user.
            failedStages.unique ()
            comment += " in stage(s) '${failedStages.join (', ')}'"
        }
        comment += " 🔥"
        if (!manualException && err != "")
            comment += " : ${err}"
        manager.addErrorBadge (comment)
        manager.buildFailure ()
    }

    def fileContents = null
    if (fileExists (commentFile)) {
        fileContents = readFile ("${commentFile}")
        comment += "\n\n" + fileContents
    }

    addComment ("${comment}")

    reportFinalStatusToSlack (err, gitHash, currentStage, fileContents)
}

def processAtMonkeyWrench (outputFile)
{
    def tmpfile = "atmonkeywrench.tmp"
    try {
        sh (script: "grep '^@MonkeyWrench: ...Summary: ' '${outputFile}' > ${tmpfile}", returnStatus: true /* don't throw exceptions if something goes wrong */)
        def lines = readFile ("${tmpfile}").split ("\n")
        for (int i = 0; i < lines.length; i++) {
            def summary = lines [i].substring (27 /*"@MonkeyWrench: AddSummary: ".length*/).trim ()
            summary = summary.replace ("<br/>", "")
            summary = summary.replace ("<a href='", "")
            def href_end = summary.indexOf ("'>")
            if (href_end > 0)
                summary = summary.substring (0, href_end)
            echo (summary)
        }
    } finally {
        sh ("rm -f '${tmpfile}'")
    }
}

def uploadFiles (glob, containerName, virtualPath)
{
    step ([
        $class: 'WAStoragePublisher',
        allowAnonymousAccess: true,
        cleanUpContainer: false,
        cntPubAccess: true,
        containerName: containerName,
        doNotFailIfArchivingReturnsNothing: false,
        doNotUploadIndividualFiles: false,
        doNotWaitForPreviousBuild: true,
        excludeFilesPath: '',
        filesPath: glob,
        storageAccName: 'bosstoragemirror',
        storageCredentialId: 'bc6a99d18d7d9ca3f6bf6b19e364d564',
        uploadArtifactsOnlyIfSuccessful: false,
        uploadZips: false,
        virtualPath: virtualPath,
        storageType: 'blobstorage'
    ])
}

@NonCPS
def getUpdateInfoAndVersion (file)
{
    def result = [:]
    def match = file =~ /xamarin.(mac|ios)-(\d+(\.\d+)+).*.pkg$/
    if (match.matches()) {
        result['platform'] = match[0][1]
        result['version'] = match[0][2]
    }
    return result
}

// There must be a better way than this hack to show a
// message in red in the Jenkins Blue Ocean UI...
def echoError (message)
{
    try {
        error (message)
    } catch (e) {
        // Ignore
    }
}

def indexOfElement (list, element)
{
    for (def i = 0; i < list.size (); i++) {
        if (list [i] == element)
            return i
    }
    return -1
}

def runXamarinMacTests (url, macOS, maccore_hash, xamarin_macios_hash)
{
    def failed = false
    def workspace = "${env.HOME}/jenkins/workspace/xamarin-macios"
    def failedTests = []
    try {
        echo ("Executing on ${env.NODE_NAME}")
        echo ("URL: ${url}")
        sh ("mkdir -p '${workspace}'")
        dir ("${workspace}") {
            // Download the script we need, and then execute it, so that we
            // don't clone all of xamarin-macios to get a single file.
            // Due to how github has implemented pull requests, this works
            // even for pull requests from forks, since each commit in the pull request
            // is also available from the main repository.
            sh ("""
                curl -fLO https://raw.githubusercontent.com/xamarin/xamarin-macios/${xamarin_macios_hash}/jenkins/prepare-packaged-macos-tests.sh
                chmod +x prepare-packaged-macos-tests.sh
                ./prepare-packaged-macos-tests.sh '${url}' '${maccore_hash}'
                """)
            def tests = [ "dontlink", "apitest", "introspection", "linksdk", "linkall", "xammac_tests" ];
            tests.each { test ->
                def t = "${test}"
                try {
                    timeout (time: 10, unit: 'MINUTES') {
                        sh ("make -C mac-test-package/tests exec-mac-${t} MONO_DEBUG=no-gdb-backtrace")
                        echo ("${t} succeeded")
                    }
                } catch (error) {
                    echoError ("${t} failed with error: ${error}")
                    failed = true
                    failedTests.add (t)
                }
            }
            // Run dontlink using the oldest system mono we support
            def t = "dontlink (system)"
            try {
                // install oldest supported mono
                sh ("./prepare-packaged-macos-tests.sh --install-old-mono")
                // run dontlink tests using the system mono
                sh ("make -C mac-test-package/tests exec-mac-system-dontlink")
            } catch (error) {
                echoError ("${t} failed with error: ${error}")
                failed = true
                failedTests.add (t)
            }
        }
    } finally {
        sh ("rm -rf ${workspace}/mac-test-package ${workspace}/*.zip ${workspace}/*.7z")
        // Find and upload any crash reports
        sh (script: """
            rm -Rf ${workspace}/crash-reports
            mkdir ${workspace}/crash-reports
            find ~/Library/Logs/DiagnosticReports/ -mtime -10m -exec cp {} ${workspace}/crash-reports \\;
            ls -la ${workspace}/crash-reports
            """,
            returnStatus: true /* Don't fail if something goes wrong */)
        try {
            if (findFiles (glob: "crash-reports/*").length > 0) {
                archiveArtifacts ("${workspace}/crash-reports/*")
            } else {
                echo ("No crash reports found")
            }
        } catch (e) {
            // Ignore any archiving errors.
        }
    }
    if (failed) {
        def failureMessage = "Xamarin.Mac tests on ${macOS} failed (${failedTests.join (', ')})"
        manager.addErrorBadge (failureMessage)
        error (failureMessage)
    }
}

def abortExecutingBuilds ()
{
    def job = Jenkins.instance.getItemByFullName (env.JOB_NAME)
    for (build in job.builds) {
        if (!build.isBuilding ())
            continue
        echo ("Current build: ${currentBuild.number} Checking build: ${build.number}");
        if (build.number > currentBuild.number) {
            error ("There is already a newer build in progress (#${build.number})")
        } else if (build.number < currentBuild.number) {
            def exec = build.getExecutor ()
            if (exec == null) {
                echo ("No executor for build ${build.number}")
            } else {
                exec.interrupt (Result.ABORTED, new CauseOfInterruption.UserInterruption ("Aborted by build #${currentBuild.number}"))
                echo ("Aborted previous build: #${build.number}")
            }
        }
    }
}

timestamps {
    def mainMacOSVersion = "10.15"
    node ("xamarin-macios && macos-${mainMacOSVersion}") {
        try {
            timeout (time: 15, unit: 'HOURS') {
                // Hard-code a workspace, since branch-based and PR-based
                // builds would otherwise use different workspaces, which
                // wastes a lot of disk space.
                workspace = "${env.HOME}/jenkins/workspace/xamarin-macios"
                commentFile = "${workspace}/xamarin-macios/jenkins/pr-comments.md"
                withEnv ([
                    "BUILD_REVISION=jenkins",
                    "PATH=/Library/Frameworks/Mono.framework/Versions/Current/Commands:${env.PATH}",
                    "WORKSPACE=${workspace}"
                    ]) {
                    sh ("mkdir -p '${workspace}/xamarin-macios'")
                    dir ("${workspace}/xamarin-macios") {
                        stage ('Checkout') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            sh ("env | sort") // Print out environment for debug purposes

                            branchName = env.BRANCH_NAME
                            echo ("Branch name: ${branchName}")

                            scmVars = checkout scm
                            isPr = (env.CHANGE_ID && !env.CHANGE_ID.empty ? true : false)
                            if (isPr) {
                                gitHash = sh (script: "git log -1 --pretty=%H refs/remotes/origin/${env.BRANCH_NAME}", returnStdout: true).trim ()
                            } else {
                                gitHash = scmVars.GIT_COMMIT
                            }
                            // Make sure we start from scratch
                            sh (script: 'make git-clean-all', returnStatus: true /* don't throw exceptions if something goes wrong */)
                            // Make really, really sure
                            sh ('git clean -xffd')
                            sh ('git submodule foreach --recursive git clean -xffd')
                        }
                    }

                    if (isPr) {
                        def hasBuildPackage = getLabels ().contains ("build-package")
                        def hasRunInternalTests = getLabels ().contains ("run-internal-tests")

                        if (!hasBuildPackage && !hasRunInternalTests) {
                            // don't add a comment to the pull request, since the public jenkins will also add comments, which ends up being too much.
                            createFinalStatus = false
                            echo ("Build skipped because the pull request doesn't have either of the labels 'build-package' or 'run-internal-tests'.")
                            return
                        }

                        if (!hasRunInternalTests)
                            skipLocalTestRunReason = "Not running tests here because they're run on public Jenkins."

                        // only aborting PR builds, since for normal branches we want to build as much as possible by default.
                        abortExecutingBuilds ()
                    }

                    dir ("${workspace}") {
                        stage ('Cleanup') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Cleaning up on ${env.NODE_NAME}")
                            sh ("${workspace}/xamarin-macios/jenkins/clean-jenkins-bots.sh")
                        }

                        stage ('Provisioning') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            sh ("cd ${workspace}/xamarin-macios && ./configure --enable-xamarin")
                            sh ("${workspace}/xamarin-macios/jenkins/provision-deps.sh")
                        }

                        stage ('Build') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            timeout (time: 1, unit: 'HOURS') {
                                withEnv ([
                                    "CURRENT_BRANCH=${branchName}",
                                    "PACKAGE_HEAD_BRANCH=${branchName}"
                                    ]) {
                                    sh ("${workspace}/xamarin-macios/jenkins/build.sh --configure-flags --enable-xamarin")
                                }
                            }
                        }

                        // Delete any packages we might have - the package directory might have packages from previous builds.
                        sh (script: "rm -Rf ${workspace}/package", returnStatus: true /* don't throw exceptions if something goes wrong */)

                        stage ('Packaging') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            def skipPackages = getLabels ().contains ("skip-packages")
                            if (!skipPackages) {
                                sh ("${workspace}/xamarin-macios/jenkins/build-package.sh")
                                sh (script: "ls -la ${workspace}/package", returnStatus: true /* don't throw exceptions if something goes wrong */)
                            } else {
                                echo ("Packaging skipped because the label 'skip-packages' was found")
                            }
                        }

                        stage ('Nugetizing') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            def skipNugets = getLabels ().contains ("skip-nugets")
                            if (!skipNugets) {
                                sh ("${workspace}/xamarin-macios/jenkins/build-nugets.sh")
                                sh (script: "ls -la ${workspace}/package", returnStatus: true /* don't throw exceptions if something goes wrong */)
                            } else {
                                echo ("Nugetizing skipped because the label 'skip-nugets' was found")
                            }
                        }

                        stage ('Signing') {
                            def entitlements = "${workspace}/xamarin-macios/mac-entitlements.plist"
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            def skipSigning = getLabels ().contains ("skip-signing")
                            if (!skipSigning) {
                                def xiPackages = findFiles (glob: "package/xamarin.ios-*.pkg")
                                if (xiPackages.length > 0) {
                                    xiPackageFilename = xiPackages [0].name
                                    echo ("Created Xamarin.iOS package: ${xiPackageFilename}")
                                }
                                def xmPackages = findFiles (glob: "package/xamarin.mac-*.pkg")
                                if (xmPackages.length > 0) {
                                    xmPackageFilename = xmPackages [0].name
                                    echo ("Created Xamarin.Mac package: ${xmPackageFilename}")
                                }
                                def msbuildZip = findFiles (glob: "package/msbuild.zip")
                                if (msbuildZip.length > 0)
                                    msbuildZipFilename = msbuildZip [0].name
                                def bundleZip = findFiles (glob: "package/bundle.zip")
                                if (bundleZip.length > 0)
                                    bundleZipFilename = bundleZip [0].name

                                if (isPr) {
                                  withCredentials ([string (credentialsId: 'codesign_keychain_pw', variable: 'PRODUCTSIGN_KEYCHAIN_PASSWORD')]) {
                                    sh ("${workspace}/xamarin-macios/jenkins/productsign.sh")
                                  }
                                } else {
                                    try {
                                        pkgs = xiPackages + xmPackages
                                        if (fileExists('release-scripts')) {
                                            dir('release-scripts') {
                                                sh ('git checkout sign-and-notarized && git pull')
                                            }
                                        } else {
                                            sh ('git clone git@github.com:xamarin/release-scripts -b sign-and-notarized')
                                        }
                                        withCredentials([string(credentialsId: 'codesign_keychain_pw', variable: 'KEYCHAIN_PASS'), string(credentialsId: 'team_id', variable: 'TEAM_ID'), string(credentialsId: 'application_id', variable: 'APP_ID'), string(credentialsId: 'installer_id', variable: 'INSTALL_ID'), usernamePassword(credentialsId: 'apple_account', passwordVariable: 'APPLE_PASS', usernameVariable: 'APPLE_ACCOUNT')]) {
                                            sh (returnStatus: true, script: "security create-keychain -p ${env.KEYCHAIN_PASS} login.keychain") // needed to repopulate the keychain
                                            sh ("security unlock-keychain -p ${env.KEYCHAIN_PASS} login.keychain")
                                            timeout (time: 90, unit: 'MINUTES') {
                                              sh ("python release-scripts/sign_and_notarize.py -a ${env.APP_ID} -i ${env.INSTALL_ID} -u ${env.APPLE_ACCOUNT} -p ${env.APPLE_PASS} -t ${env.TEAM_ID} -d package/notarized -e ${entitlements} -k login.keychain " + pkgs.flatten ().join (" "))
                                            }
                                        }

                                        def xiNotarizedPackages = findFiles (glob: "package/notarized/xamarin.ios-*.pkg")
                                        if (xiNotarizedPackages.length > 0) {
                                            xiNotarizedPkgFilename = xiNotarizedPackages [0].name
                                            echo ("Created notarized Xamarin.iOS package: ${xiNotarizedPkgFilename}")
                                        }
                                        def xmNotarizedPackages = findFiles (glob: "package/notarized/xamarin.mac-*.pkg")
                                        if (xmNotarizedPackages.length > 0) {
                                            xmNotarizedPkgFilename = xmNotarizedPackages [0].name
                                            echo ("Created notarized Xamarin.Mac package: ${xmNotarizedPkgFilename}")
                                        }
                                    } catch (ex) {
                                        echo "Notarization failed:\n${ex.getMessage()}"
                                        for (def stack : ex.getStackTrace()) {
                                            echo "\t${stack}"
                                        }
                                        manager.addWarningBadge("PKGs are not notarized")
                                    }
                                }
                            } else {
                                echo ("Signing skipped because the label 'skip-signing' was found")
                            }
                        }

                        stage ('Generate Workspace Info') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            withCredentials([string (credentialsId: '92cf81e5-6db5-408c-8a0d-192b36200a16', variable: 'GITHUB_AUTH_TOKEN'), string (credentialsId: 'xamarin-build-tasks', variable: 'XAMARIN_BUILD_TASKS')]) {
                                def redirect = getRedirect ()
                                sh "curl -o Xamarin.Build.Tasks.nupkg \"${redirect}\""
                                dir("BuildTasks") {
                                    deleteDir ()
                                    sh "unzip ../Xamarin.Build.Tasks.nupkg"
                                    sh "mono tools/BuildTasks/build-tasks.exe workspaceinfo -p ${workspace} -o ${workspace}/package-internal/ci-checkout.json"
                                    sh "mono tools/BuildTasks/build-tasks.exe dependencyinfo -p ${workspace} -o ${workspace}/package-internal/dependency-info.json"
                                }
                            }
                        }

                        stage ('Upload to Azure') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            virtualPath = "jenkins/${branchName}/${gitHash}/${env.BUILD_NUMBER}"
                            packagePrefix = "https://bosstoragemirror.blob.core.windows.net/wrench/${virtualPath}/package"

                            // Create manifest
                            def uploadingFiles = findFiles (glob: "package/*")
                            def manifest = ""
                            for (int i = 0; i < uploadingFiles.length; i++) {
                                def file = uploadingFiles [i]
                                manifest += "${packagePrefix}/${file.name}\n"
                            }
                            manifest += "${packagePrefix}/artifacts.json\n"
                            manifest += "${packagePrefix}/manifest\n"
                            writeFile (file: "package/manifest", text: manifest)

                            sh ("ls -la package")
                            uploadFiles ("package/*", "wrench", virtualPath)

                            def notarized_files = findFiles (glob: "package/notarized/*")
                            if (notarized_files.size () > 0)
                              uploadFiles ("package/notarized/*", "wrench", virtualPath)
                            uploadFiles ("package-internal/*", "jenkins-internal", virtualPath)

                            // Also upload manifest to a predictable url (without the build number)
                            // This manifest will be overwritten in subsequent builds (for this [PR/branch]+hash combination)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${branchName}/${gitHash}")
                            // And also create a 'latest' version (which really means 'latest built', not 'latest hash', but it will hopefully be good enough)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${branchName}/latest")
                            // And finally create a per-hash manifest (which will be overwritten in rebuilds of the same hash)
                            uploadFiles ("package/manifest", "wrench", "jenkins/${gitHash}")

                            manifestFilename = "manifest"
                        }

                        stage ('Generate artifacts.json') {
                            withCredentials ([string (credentialsId: '92cf81e5-6db5-408c-8a0d-192b36200a16', variable: 'GITHUB_AUTH_TOKEN'), usernamePassword (credentialsId: 'd146d4aa-a437-4633-908f-b455876c44d0', passwordVariable: 'STORAGE_PASSWORD', usernameVariable: 'STORAGE_ACCOUNT')]) {
                                dir ("BuildTasks") {
                                    try {
                                        sh "mono tools/BuildTasks/build-tasks.exe artifacts -s ${workspace}/xamarin-macios -a ${env.STORAGE_ACCOUNT} -c ${env.STORAGE_PASSWORD} -u wrench/${virtualPath}/package"
                                        artifactsFilename = "artifacts.json"
                                    } catch (err) {
                                        def msg = "Failed to generate/upload artifacts.json: " + err.getMessage ();
                                        echo ("⚠️ ${msg} ⚠️")
                                        manager.addWarningBadge (msg)
                                        appendFileComment ("⚠️ [${msg}](${env.RUN_DISPLAY_URL}) 🔥\n")
                                    }
                                }
                            }
                        }

                        stage ('Publish builds to GitHub') {
                            currentStage = "${STAGE_NAME}"
                            echo ("Building on ${env.NODE_NAME}")
                            utils = load ("${workspace}/xamarin-macios/jenkins/utils.groovy")
                            def packagesMessage = ""
                            if (xiPackageFilename != null) {
                                xiPackageUrl = "${packagePrefix}/${xiPackageFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.iOS', "${xiPackageUrl}", 'SUCCESS', "${xiPackageFilename}")
                                packagesMessage += "* [${xiPackageFilename} (Not notarized)](${xiPackageUrl})\n"
                            }
                            if (xmPackageFilename != null) {
                                xmPackageUrl = "${packagePrefix}/${xmPackageFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.Mac', "${xmPackageUrl}", 'SUCCESS', "${xmPackageFilename}")
                                packagesMessage += "* [${xmPackageFilename} (Not notarized)](${xmPackageUrl})\n"
                            }
                            if (xiNotarizedPkgFilename != null) {
                                xiNotarizedPackageUrl = "${packagePrefix}/notarized/${xiNotarizedPkgFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.iOS-notarized', "${xiNotarizedPackageUrl}", 'SUCCESS', "${xiNotarizedPkgFilename}")
                                packagesMessage += "* [${xiNotarizedPkgFilename} (Notarized)](${xiNotarizedPackageUrl})\n"
                            }
                            if (xmNotarizedPkgFilename != null) {
                                xmNotarizedPackageUrl = "${packagePrefix}/notarized/${xmNotarizedPkgFilename}"
                                utils.reportGitHubStatus (gitHash, 'PKG-Xamarin.Mac-notarized', "${xmNotarizedPackageUrl}", 'SUCCESS', "${xmNotarizedPkgFilename}")
                                packagesMessage += "* [${xmNotarizedPkgFilename} (Notarized)](${xmNotarizedPackageUrl})\n"
                            }
                            if (manifestFilename != null) {
                                def manifestUrl = "${packagePrefix}/${manifestFilename}"
                                utils.reportGitHubStatus (gitHash, "${manifestFilename}", "${manifestUrl}", 'SUCCESS', "${manifestFilename}")
                            }
                            if (artifactsFilename != null) {
                                def artifactUrl = "${packagePrefix}/${artifactsFilename}"
                                utils.reportGitHubStatus (gitHash, "Jenkins: Artifacts", "${artifactUrl}", 'SUCCESS', "${artifactsFilename}")
                            }
                            if (bundleZipFilename != null) {
                                def bundleZipUrl = "${packagePrefix}/${bundleZipFilename}"
                                utils.reportGitHubStatus (gitHash, "bundle.zip", "${bundleZipUrl}", 'SUCCESS', "${bundleZipFilename}")
                            }
                            if (msbuildZipFilename != null) {
                                def msbuildZipUrl = "${packagePrefix}/${msbuildZipFilename}"
                                utils.reportGitHubStatus (gitHash, "msbuild.zip", "${msbuildZipUrl}", 'SUCCESS', "${msbuildZipFilename}")
                            }

                            def nupkgs = findFiles (glob: "package/*.nupkg")
                            for (def i = 0; i < nupkgs.size (); i++) {
                                def nupkg = nupkgs [i]
                                def nupkg_name = nupkg.name
                                def nupkgUrl = "${packagePrefix}/${nupkg_name}"
                                utils.reportGitHubStatus (gitHash, nupkg_name, nupkgUrl, "SUCCESS", nupkg_name)
                                packagesMessage += "* [${nupkg_name}](${nupkgUrl})\n"
                            }

                            // Publish any other *.pkg files we've created
                            def pkgs = findFiles (glob: "package/*.pkg")
                            pkgs += findFiles (glob: "package/*.msi")
                            for (def i = 0; i < pkgs.size (); i++) {
                                def pkg = pkgs [i];
                                def pkg_name = pkg.name
                                if (pkg_name == xiPackageFilename || pkg_name == xmPackageFilename)
                                    continue // we've already handled these files
                                def pkgUrl = "${packagePrefix}/${pkg_name}"
                                utils.reportGitHubStatus (gitHash, pkg_name, pkgUrl, "SUCCESS", pkg_name)
                                packagesMessage += "* [${pkg_name}](${pkgUrl})\n"
                            }

                            if (packagesMessage != "")
                                appendFileComment ("✅ Packages built successfully\n<details><summary>View packages</summary>\n\n${packagesMessage}\n</details>\n\n")
                        }

                        stage ('Publish Nugets') {
                            currentStage = "${STAGE_NAME}"
                            withCredentials ([string (credentialsId: 'azdo-package-feed-token', variable: 'AZDO_PACKAGE_FEED_TOKEN')]) {
                                def nupkgs = findFiles (glob: "package/notarized/*")
                                if (nupkgs.size () == 0) {
                                    echo ("No NuGet packages to publish.")
                                } else {
                                    def nugetResult = sh (script: "${workspace}/xamarin-macios/jenkins/publish-to-nuget.sh --apikey=${AZDO_PACKAGE_FEED_TOKEN} --source=https://pkgs.dev.azure.com/azure-public/vside/_packaging/xamarin-impl/nuget/v3/index.json package/*.nupkg", returnStatus: true)
                                    if (nugetResult != 0) {
                                        failedStages.add (currentStage)
                                        manualException = true
                                        def msg = "Failed to publish NuGet packages. Please review log for details."
                                        echo ("⚠️ ${msg} ⚠️")
                                        manager.addWarningBadge (msg)
                                        appendFileComment ("⚠️ [${msg}](${env.RUN_DISPLAY_URL}) 🔥\n")
                                    } else {
                                        echo ("${currentStage} succeeded")
                                    }
                                }
                            }
                        }

                        dir ('xamarin-macios') {
                            stage ('Launch external tests') {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                // VSTS does not allow any branch name anymore, it has to be an existing branch.
                                // Since pull requests don't create branches in the xamarin org (that VSTS sees at least),
                                // I've created a 'pull-request' branch which we'll use for pull requests.
                                def vsts_branch = isPr ? "pull-request" : branchName;
                                def skipExternalTests = getLabels ().contains ("skip-external-tests")
                                if (skipExternalTests) {
                                    echo ("Skipping external tests because the label 'skip-external-tests' was found")
                                } else if (isPr && !getLabels ().contains ("trigger-device-tests")) {
                                    echo ("Skipping external tests because the label 'trigger-device-tests' was not found and it's required for external tests to run for pull requests")
                                } else {
                                    def outputFile = "${workspace}/xamarin-macios/wrench-launch-external.output.tmp"
                                    try {
                                        withCredentials ([string (credentialsId: 'macios_provisionator_pat', variable: 'PROVISIONATOR_VSTS_PAT'), string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_PAT_TOKEN')]) {
                                            sh ("make -C ${workspace}/xamarin-macios/tests wrench-launch-external MAC_PACKAGE_URL=${xmPackageUrl} IOS_PACKAGE_URL=${xiPackageUrl} WRENCH_URL=${env.RUN_DISPLAY_URL} BUILD_REVISION=${gitHash} BUILD_LANE=${vsts_branch} BUILD_WORK_HOST=${env.NODE_NAME} GITHUB_TOKEN=${GITHUB_PAT_TOKEN} 2>&1 | tee ${outputFile}")
                                        }
                                        processAtMonkeyWrench (outputFile)
                                    } catch (error) {
                                        echo ("🚫 Launching external tests failed: ${error} 🚫")
                                        manager.addWarningBadge ("Failed to launch external tests")
                                    } finally {
                                        sh ("rm -f '${outputFile}'")
                                    }
                                }

                                if (getLabels ().contains ("run-sample-tests")) {
                                    def outputFile = "${workspace}/xamarin-macios/wrench-launch-external.output.tmp"
                                    try {
                                        withCredentials ([string (credentialsId: 'macios_provisionator_pat', variable: 'PROVISIONATOR_VSTS_PAT')]) {
                                            sh ("make -C ${workspace}/maccore/tests/external wrench-launch-sample-builds 'BUILD_REVISION=${gitHash}' 'BUILD_BRANCH=${vsts_branch}' V=1 2>&1 | tee ${outputFile}")
                                        }
                                        processAtMonkeyWrench (outputFile)
                                    } catch (error) {
                                        echo ("🚫 Launching external sample tests failed: ${error} 🚫")
                                        manager.addWarningBadge ("Failed to launch external sample tests")
                                    } finally {
                                        sh ("rm -f '${outputFile}'")
                                    }
                                }
                            }

                            stage ('Install Provisioning Profiles') {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                withCredentials ([string (credentialsId: 'login-keychain-password', variable: "LOGIN_KEYCHAIN_PASSWORD")]) {
                                    sh ("${workspace}/maccore/tools/install-qa-provisioning-profiles.sh")
                                }
                            }

                            stage ('Publish reports') {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                reportPrefix = sh (script: "${workspace}/xamarin-macios/jenkins/publish-results.sh | grep '^Url Prefix: ' | sed 's/^Url Prefix: //'", returnStdout: true).trim ()
                                if (skipLocalTestRunReason == "") {
                                    echo ("Html report: ${reportPrefix}/tests/index.html")
                                } else {
                                    echo ("Html report: ${skipLocalTestRunReason}")
                                }
                                echo ("API diff (from stable): ${reportPrefix}/api-diff/index.html")
                                echo ("API diff (from previous commit / before pull request): ${reportPrefix}/apicomparison/api-diff.html")
                                echo ("Generator diff: ${reportPrefix}/generator-diff/index.html")
                            }

                            stage ('API diff') {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                def apidiffResult = sh (script: "${workspace}/xamarin-macios/jenkins/build-api-diff.sh --publish", returnStatus: true)
                                if (apidiffResult != 0)
                                    manager.addWarningBadge ("Failed to generate API diff")
                                echo ("API diff (from stable): ${reportPrefix}/api-diff/index.html")
                            }

                            stage ('API & Generator comparison') {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                def compareResult = sh (script: "${workspace}/xamarin-macios/jenkins/compare.sh --publish", returnStatus: true)
                                if (compareResult != 0)
                                    manager.addWarningBadge ("Failed to generate API / Generator diff")
                                echo ("API diff (from previous commit / before pull request): ${reportPrefix}/apicomparison/api-diff.html")
                                echo ("Generator diff: ${reportPrefix}/generator-diff/index.html")
                            }

                            def hasXamarinMacTests = true
                            stage ("Package XM tests") {
                                currentStage = "${STAGE_NAME}"
                                echo ("Building on ${env.NODE_NAME}")
                                def skipPackagedXamarinMacTests = getLabels ().contains ("skip-packaged-xamarin-mac-tests")
                                if (skipPackagedXamarinMacTests) {
                                    echo ("Skipping packaged Xamarin.Mac tests because the label 'skip-packaged-xamarin-mac-tests' was found")
                                    hasXamarinMacTests = false
                                } else {
                                    def exitCode = sh (script: "make -C ${workspace}/xamarin-macios/tests package-tests", returnStatus: true)
                                    if (exitCode != 0) {
                                        hasXamarinMacTests = false
                                        echoError ("Failed to package Xamarin.Mac tests (exit code: ${exitCode})")
                                        failedStages.add (currentStage)
                                    } else {
                                        def packaged_xm_tests = findFiles (glob: "tests/*.7z")
                                        if (packaged_xm_tests.size () > 0) {
                                            uploadFiles ("tests/*.7z", "wrench", virtualPath)
                                        } else {
                                            // This may happen if the Xamarin.Mac build has been disabled
                                            manager.addWarningBadge("Could not find any packaged Xamarin.Mac tests to upload")
                                            hasXamarinMacTests = false
                                        }
                                    }
                                }
                            }

                            timeout (time: 13, unit: 'HOURS') {
                                // We run tests locally and on older macOS bots in parallel.
                                // The older macOS tests run quickly (and the bots should usually be idle),
                                // which means that the much longer normal (local) test run should take
                                // longer to complete (which is important since this will block until all tests
                                // have been run, even if any older macOS bots are busy doing other things, preventing
                                // our tests from running there), giving the older macOS bots plenty of
                                // time to finish their test runs.
                                stage ('Run tests parallelized') {
                                    def builders = [:]

                                    // Add test runs on older macOS versions
                                    if (hasXamarinMacTests) {
                                        def url = "https://bosstoragemirror.blob.core.windows.net/wrench/${virtualPath}/tests/mac-test-package.7z"
                                        def maccore_hash = sh (script: "grep NEEDED_MACCORE_VERSION ${workspace}/xamarin-macios/mk/xamarin.mk | sed 's/NEEDED_MACCORE_VERSION := //'", returnStdout: true).trim ()
                                        // Get the min and current macOS version from Make.config,
                                        // and calculate all the macOS versions in between (including both end points).
                                        // The trims/splits/toIntegers at the end are to select the minor part of the macOS version number,
                                        // then we just loop from the minor part of the min version to the minor part of the current version.
                                        def firstOS = sh (returnStdout: true, script: "grep ^MIN_MACOS_SDK_VERSION= '${workspace}/xamarin-macios/Make.config' | sed 's/.*=//'").trim ().split ("\\.")[1].toInteger ()
                                        def lastOS = sh (returnStdout: true, script: "grep ^MACOS_NUGET_VERSION= '${workspace}/xamarin-macios/Make.versions' | sed -e 's/.*=//' -e 's/.[0-9]*\$//'").trim ().split ("\\.")[1].toInteger ()
                                        def macOSes = []
                                        def excludedOSes = []
                                        // If any macOS version needs to be excluded manually, it can be done like this (in this case to remove macOS 10.14):
                                        // Any macOS versions excluded like this still get a entry in the Jenkins UI, making it explicit that the OS version was skipped.
                                        //     excludedOSes.add ("10.14")
                                        excludedOSes.add ("11.0") // We have a bot labelled as 10.16 (and we're running there)
                                        // Have in mind that the value in the list is only the minor part of the macOS version number.
                                        for (i = 0; i < macOSes.size (); i++) {
                                            def os = macOSes [i];
                                            def macOS = "${os}" // Need to bind the label variable before the closure
                                            def excluded = false
                                            def nodeText = "XM tests on ${macOS}"
                                            if (indexOfElement (excludedOSes, os) >= 0) {
                                                excluded = true
                                                nodeText = "ℹ️ XM tests not executed on ${macOS} ℹ️"
                                            } else if (os == mainMacOSVersion) {
                                                excluded = true
                                                nodeText = "ℹ️ XM tests not executed on a separate ${macOS} bot because they're already executed as a part of the main test run ℹ️"
                                            }
                                            builders [nodeText] = {
                                                try {
                                                    timeout (time: 15, unit: 'MINUTES') {
                                                        if (excluded) {
                                                            echo (nodeText)
                                                        } else {
                                                            node ("xamarin-macios && macos-10.${macOS}") {
                                                                stage ("Running XM tests on '10.${macOS}'") {
                                                                    runXamarinMacTests (url, "macOS 10.${macOS}", maccore_hash, gitHash)
                                                                }
                                                            }
                                                        }
                                                    }
                                                } catch (err) {
                                                    currentStage = "Running XM tests on '${macOS}'"
                                                    def msg = "Xamarin.Mac tests on ${macOS} failed: " + err.getMessage ();
                                                    appendFileComment ("🔥 [${msg}](${env.RUN_DISPLAY_URL}) 🔥\n")
                                                    failedStages.add (currentStage)
                                                    throw err
                                                }
                                            }
                                        }
                                    }

                                    // Add standard test run
                                    builders ["All tests"] = {
                                        stage ('Run tests') {
                                            currentStage = "Test run"
                                            echo ("Building on ${env.NODE_NAME}")
                                            if (skipLocalTestRunReason != "") {
                                                echo (skipLocalTestRunReason)
                                                appendFileComment ("ℹ️ Test run skipped: ${skipLocalTestRunReason}\n")
                                            } else {
                                                def XHARNESS_GITHUB_TOKEN_FILE = "${workspace}/xamarin-macios/jenkins/.github-pat-token"
                                                withCredentials ([string (credentialsId: 'macios_github_comment_token', variable: 'GITHUB_PAT_TOKEN')]) {
                                                    writeFile (file: "${XHARNESS_GITHUB_TOKEN_FILE}", text: "${GITHUB_PAT_TOKEN}")
                                                }
                                                echo ("Html report: ${reportPrefix}/tests/index.html")
                                                def runTestResult = null
                                                withEnv (["XHARNESS_GITHUB_TOKEN_FILE=${XHARNESS_GITHUB_TOKEN_FILE}"]) {
                                                    runTestResult = sh (script: "${workspace}/xamarin-macios/jenkins/run-tests.sh --target=wrench-jenkins --publish --keychain=xamarin-macios", returnStatus: true)
                                                }
                                                if (runTestResult != 0) {
                                                    failedStages.add (currentStage)
                                                    manualException = true
                                                    error ("Test run failed: ${reportPrefix}/tests/index.html")
                                                } else {
                                                    echo ("${currentStage} succeeded")
                                                }
                                            }
                                        }
                                    }

                                    // Run it all parallelized
                                    parallel builders
                                }
                            }

                            currentStage = ""
                        } // dir ("xamarin-macios")
                    } // dir ("${workspace}")
                }
                reportFinalStatus ("", "${gitHash}", "${currentStage}")
            } // timeout
        } catch (err) {
            timeout (time: 5, unit: 'MINUTES') { // Give 5 minutes to publish results to GitHub/Slack
                reportFinalStatus ("${err}", "${gitHash}", "${currentStage}")
            } // timeout
        } finally {
            timeout (time: 60, unit: 'MINUTES') { // Give an hour to publish and upload stuff
                stage ('Final tasks') {
                    sh (script: "${workspace}/xamarin-macios/jenkins/publish-results.sh", returnStatus: true /* don't throw exceptions if something goes wrong */)
                    sh (script: "make git-clean-all -C ${workspace}/xamarin-macios", returnStatus: true /* don't throw exceptions if something goes wrong */)
                }
            } // timeout
        } // try
    } // node
} // timestamps