Added delta toolchain build to automated PR check (#5687)

2023-06-21 14:50:47 -07:00 · 2023-06-21 14:50:47 -07:00 · 7503d73320
--- a/.pipelines/PackageBuildPRCheck.yml
+++ b/.pipelines/PackageBuildPRCheck.yml
@ -1,62 +0,0 @@
-trigger: none
-pr:
-  - fasttrack/*
-
-parameters:
-  - name: buildConfiguration
-    type: object
-    default:
-    - name: "AMD64"
-      agentPool: "mariner-core-x64-1es-mariner2-gpt-test"
-      agentVMName: "Mariner-Build-Mariner2-1ES-Managed-Image2-GPT-Test"
-    - name: "ARM64"
-      agentPool: "mariner-core-arm64-1es-mariner2"
-      agentVMName: "Mariner-Build-Mariner2-1ES-ARM64-Managed-Image"
-
-resources:
-  repositories:
-    - repository: templates
-      type: git
-      name: OneBranch.Pipelines/GovernedTemplates
-      ref: refs/heads/main
-
-extends:
-  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
-  parameters:
-    featureFlags:
-      runOnHost: true
-    globalSdl:
-      credscan:
-        suppressionsFile: .config/CredScanSuppressions.json
-    stages:
-      - ${{ each configuration in parameters.buildConfiguration }}:
-        - stage: Toolchain_${{ configuration.name }}
-          jobs:
-            - job: Build
-              pool:
-                type: linux
-                isCustom: true
-                name: ${{ configuration.agentPool }}
-                vmImage: ${{ configuration.agentVMName }}
-              steps:
-                - template: .pipelines/templates/ToolchainBuild.yml@self
-
-        - stage: RPMs_${{ configuration.name }}
-          dependsOn: Toolchain_${{ configuration.name }}
-          jobs:
-            - job: Build
-              pool:
-                type: linux
-                isCustom: true
-                name: ${{ configuration.agentPool }}
-                vmImage: ${{ configuration.agentVMName }}
-              strategy:
-                matrix:
-                  regular:
-                    runCheck: 'false'
-                  ptest:
-                    runCheck: 'true'
-              steps:
-                - template: .pipelines/templates/PackageBuild.yml@self
-                  parameters:
-                    isCheckBuild: '$(runCheck)'
--- a/.pipelines/livepatching/BuildLivepatch.sh
+++ b/.pipelines/livepatching/BuildLivepatch.sh
@ -1,5 +1,8 @@
 #!/bin/bash

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 set -e

 ROOT_DIR="$(git rev-parse --show-toplevel)"
--- a/.pipelines/livepatching/BuildLivepatchSigned.sh
+++ b/.pipelines/livepatching/BuildLivepatchSigned.sh
@ -1,5 +1,8 @@
 #!/bin/bash

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
 set -e

 ROOT_DIR="$(git rev-parse --show-toplevel)"
--- a/.pipelines/prchecks/PackageBuildPRCheck.yml
+++ b/.pipelines/prchecks/PackageBuildPRCheck.yml
@ -0,0 +1,93 @@
+# Since we're boosting our builds by using a private, pre-compiled raw toolchain
+# the pipeline requires defining the following variables outside of the YAML:
+# - rawToolchainCacheURL_AMD64
+# - rawToolchainCacheURL_ARM64
+# - agentPool_AMD64
+# - agentPool_ARM64
+# - agentVMName_AMD64
+# - agentVMName_ARM64
+
+trigger: none
+
+parameters:
+  - name: buildConfiguration
+    type: object
+    default:
+    - name: "AMD64"
+      agentPool: "$(agentPool_AMD64)"
+      agentVMName: "$(agentVMName_AMD64)"
+      rawToolchainCacheURL: "$(rawToolchainCacheURL_AMD64)"
+      rawToolchainExpectedHash: "f56df34b90915c93f772d3961bf5e9eeb8c1233db43dd92070214e4ce6b72894"
+    - name: "ARM64"
+      agentPool: "$(agentPool_ARM64)"
+      agentVMName: "$(agentVMName_ARM64)"
+      rawToolchainCacheURL: "$(rawToolchainCacheURL_ARM64)"
+      rawToolchainExpectedHash: "65de43b3bdcfdaac71df1f11fd1f830a8109b1eb9d7cb6cbc2e2d0e929d0ef76"
+
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
+  parameters:
+    featureFlags:
+      runOnHost: true
+    globalSdl:
+      credscan:
+        suppressionsFile: .config/CredScanSuppressions.json
+    stages:
+      - ${{ each configuration in parameters.buildConfiguration }}:
+        - stage: Toolchain_${{ configuration.name }}
+          jobs:
+            - job: Build
+              pool:
+                type: linux
+                isCustom: true
+                name: ${{ configuration.agentPool }}
+                vmImage: ${{ configuration.agentVMName }}
+              variables:
+                ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+              steps:
+                - template: .pipelines/templates/RawToolchainDownload.yml@self
+                  parameters:
+                    rawToolchainCacheURL: ${{ configuration.rawToolchainCacheURL }}
+                    rawToolchainExpectedHash: ${{ configuration.rawToolchainExpectedHash }}
+
+                - template: .pipelines/templates/ToolchainBuild.yml@self
+                  parameters:
+                    buildArtifactsFolder: $(ob_outputDirectory)
+
+                # 1. Automatic publishing seems to be broken in the OneBranch templates when the build runs directly on the agent host.
+                #    Once that's fixed, we should remove this step.
+                # 2. The value for 'artifact' cannot be changed, as this is the only value OneBranch accepts.
+                #    We cannot also use pre-defined variables like "drop_$(System.StageName)_$(Agent.JobName)", to automatically track name changes.
+                - task: PublishPipelineArtifact@1
+                  inputs:
+                    artifact: drop_Toolchain_${{ configuration.name }}_Build
+                    targetPath: $(ob_outputDirectory)
+                  condition: always()
+                  displayName: 'Publish toolchain artifacts'
+
+        - stage: RPMs_${{ configuration.name }}
+          dependsOn: Toolchain_${{ configuration.name }}
+          jobs:
+            - job: Build
+              pool:
+                type: linux
+                isCustom: true
+                name: ${{ configuration.agentPool }}
+                vmImage: ${{ configuration.agentVMName }}
+              strategy:
+                matrix:
+                  regular:
+                    runCheck: 'false'
+                  ptest:
+                    runCheck: 'true'
+              steps:
+                - template: .pipelines/templates/PackageBuild.yml@self
+                  parameters:
+                    isCheckBuild: '$(runCheck)'
--- a/.pipelines/templates/RawToolchainDownload.yml
+++ b/.pipelines/templates/RawToolchainDownload.yml
@ -0,0 +1,37 @@
+parameters:
+  - name: coreRepoRoot
+    type: string
+    default: "$(Build.SourcesDirectory)"
+
+  - name: rawToolchainCacheURL
+    type: string
+
+  - name: rawToolchainExpectedHash
+    type: string
+
+steps:
+  - bash: |
+      set -e
+
+      raw_toolchain_file_path="${{ parameters.coreRepoRoot }}/build/toolchain/toolchain_from_container.tar.gz"
+
+      echo "-- Downloading cached raw toolchain."
+
+      mkdir -p "$(dirname "$raw_toolchain_file_path")"
+      if ! wget --quiet --timeout=30 --continue "$RAW_TOOLCHAIN_URL" -O "$raw_toolchain_file_path"; then
+          echo "-- ERROR: failed to download raw toolchain cache." >&2
+          exit 1
+      fi
+
+      # Verifying toolchain's SHA-256 hash.
+      cache_sha256=$(sha256sum "$raw_toolchain_file_path" | cut -d' ' -f1)
+      if [[ "$cache_sha256" != "${{ parameters.rawToolchainExpectedHash }}" ]]; then
+          echo "-- ERROR: raw toolchain hash verification failed. Expected (${{ parameters.rawToolchainExpectedHash }}). Got ($cache_sha256)." >&2
+          exit 1
+      fi
+      echo "-- Raw toolchain hash OK."
+
+      touch "$raw_toolchain_file_path"
+    env:
+      RAW_TOOLCHAIN_URL: ${{ parameters.rawToolchainCacheURL }}
+    displayName: "Populate raw toolchain"
--- a/.pipelines/templates/ToolchainBuild.yml
+++ b/.pipelines/templates/ToolchainBuild.yml
@ -1,4 +1,38 @@
+parameters:
+  - name: buildArtifactsFolder
+    type: string
+    default: "$(Build.ArtifactStagingDirectory)"
+
+  - name: coreRepoRoot
+    type: string
+    default: "$(Build.SourcesDirectory)"
+
 steps:
-  - script: |
-      echo "The used architecture is: $(uname -m)"
-    displayName: 'Use parameter'
+  - bash: sudo make -C "${{ parameters.coreRepoRoot }}/toolkit" "-j$(nproc)" toolchain QUICK_REBUILD=y
+    displayName: "Build toolchain"
+
+  - bash: |
+      failed_rpms_log="${{ parameters.coreRepoRoot }}/build/logs/toolchain/failures.txt"
+
+      if [[ -f "$failed_rpms_log" ]]; then
+          echo "List of RPMs that failed to build:" >&2
+          cat "$failed_rpms_log" >&2
+      else
+          echo "Build failed - no specific RPM" >&2
+      fi
+    condition: failed()
+    displayName: "Print failed RPMs"
+
+  - bash: |
+      published_artifacts_dir="${{ parameters.buildArtifactsFolder }}/ARTIFACTS"
+      mkdir -p "$published_artifacts_dir"
+      cp "${{ parameters.coreRepoRoot }}"/build/toolchain/toolchain_built_{,s}rpms_all.tar.gz "$published_artifacts_dir"
+    condition: succeeded()
+    displayName: "Copy artifacts for publishing"
+
+  - bash: |
+      published_logs_dir="${{ parameters.buildArtifactsFolder }}/LOGS"
+      mkdir -p "$published_logs_dir"
+      tar -C "${{ parameters.coreRepoRoot }}/build/logs/toolchain" -czf "$published_logs_dir/toolchain.logs.tar.gz" .
+    condition: always()
+    displayName: "Copy logs for publishing"