Add Microsoft.MSBuildCache (#17393)

Add Microsoft.MSBuildCache This change adds a new pipeline which enables caching in the build. This is added as a separate pipeline for now with the eventual goal of enabling for PR and/or CI builds. Documentation for Microsoft.MSBuildCache can be found in the GitHub repo: https://github.com/microsoft/MSBuildCache Preliminary numbers below. * [Baseline](https://dev.azure.com/ms/terminal/_build/results?buildId=579399&view=results): 12 min * [0% Cache hits](https://dev.azure.com/ms/terminal/_build/results?buildId=579419&view=results): 16 mins * [100% cache hits](https://dev.azure.com/ms/terminal/_build/results?buildId=579427&view=results): 3 mins
2024-06-20 14:02:26 -07:00 · 2024-06-20 14:02:26 -07:00 · 613a539613
--- a/.github/actions/spelling/expect/expect.txt
+++ b/.github/actions/spelling/expect/expect.txt
@ -738,6 +738,7 @@ HABCDEF
 Hackathon
 HALTCOND
 HANGEUL
+hardlinks
 hashalg
 HASSTRINGS
 hbitmap
@ -1080,6 +1081,7 @@ MOUSEFIRST
 MOUSEHWHEEL
 MOVESTART
 msb
+msbuildcache
 msctf
 msctls
 msdata
@ -1489,6 +1491,7 @@ reparented
 reparenting
 replatformed
 Replymessage
+reportfileaccesses
 repositorypath
 Requiresx
 rerasterize
@ -2005,6 +2008,7 @@ wincontypes
 WINCORE
 windbg
 WINDEF
+windir
 windll
 WINDOWALPHA
 windowdpiapi
--- a/.gitignore
+++ b/.gitignore
@ -283,3 +283,6 @@ MSG*.bin
 profiles.json
 *.metaproj
 *.swp
+
+# MSBuildCache
+/MSBuildCacheLogs/
--- a/Directory.Build.props
+++ b/Directory.Build.props
@ -0,0 +1,60 @@
+<Project>
+  <!--
+    NOTE! This file gets written-over entirely by the release builds.
+    Any build logic in this file must be optional and only apply to non-release builds.
+  -->
+
+  <!-- MsBuildCache -->
+  <PropertyGroup>
+    <!-- Off by default -->
+    <MsBuildCacheEnabled Condition="'$(MsBuildCacheEnabled)' == ''">false</MsBuildCacheEnabled>
+
+    <!-- Always off during package restore -->
+    <MsBuildCacheEnabled Condition=" '$(ExcludeRestorePackageImports)' == 'true' ">false</MsBuildCacheEnabled>
+
+    <!-- In Azure pipelines, use Pipeline Caching as the cache storage backend. Otherwise, use the local cache. -->
+    <MSBuildCachePackageName Condition="'$(TF_BUILD)' != ''">Microsoft.MSBuildCache.AzurePipelines</MSBuildCachePackageName>
+    <MSBuildCachePackageName Condition="'$(MSBuildCachePackageName)' == ''">Microsoft.MSBuildCache.Local</MSBuildCachePackageName>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
+    <!-- Change this to bust the cache -->
+    <MSBuildCacheCacheUniverse Condition="'$(MSBuildCacheCacheUniverse)' == ''">202310210737</MSBuildCacheCacheUniverse>
+
+    <!--
+      Visual Studio telemetry reads various ApplicationInsights.config files and other files after the project is finished, likely in a detached process.
+      This is acceptable and should not impact cache correctness.
+    -->
+    <MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
+      $(MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns);
+      \**\ApplicationInsights.config;
+      $(LocalAppData)\Microsoft\VSApplicationInsights\**;
+      $(LocalAppData)\Microsoft\Windows\INetCache\**;
+      A:\;
+      E:\;
+      $(windir)\**;
+    </MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
+
+    <!-- 
+      This repo uses a common output directory with many projects writing duplicate outputs. Allow everything, but note this costs some performance in the form of requiring
+      the cache to use copies instead of hardlinks when pulling from cache.
+    -->
+    <MSBuildCacheIdenticalDuplicateOutputPatterns>$(MSBuildCacheIdenticalDuplicateOutputPatterns);bin\**</MSBuildCacheIdenticalDuplicateOutputPatterns>
+
+    <!-- version of MSBuildCache is not part of the cache key -->
+    <PackagesConfigFile>$(MSBuildThisFileDirectory)\dep\nuget\packages.config</PackagesConfigFile>
+    <MSBuildCacheIgnoredInputPatterns>$(MSBuildCacheIgnoredInputPatterns);$(PackagesConfigFile)</MSBuildCacheIgnoredInputPatterns>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(MSBuildCacheEnabled)' == 'true' and '$(MSBuildCachePackageRoot)' == ''">
+    <PackagesConfigContents>$([System.IO.File]::ReadAllText("$(PackagesConfigFile)"))</PackagesConfigContents>
+    <MSBuildCachePackageVersion>$([System.Text.RegularExpressions.Regex]::Match($(PackagesConfigContents), 'Microsoft.MSBuildCache.*?version="(.*?)"').Groups[1].Value)</MSBuildCachePackageVersion>
+    <MSBuildCachePackageRoot>$(MSBuildThisFileDirectory)packages\$(MSBuildCachePackageName).$(MSBuildCachePackageVersion)</MSBuildCachePackageRoot>
+    <MSBuildCacheSharedCompilationPackageRoot>$(MSBuildThisFileDirectory)packages\Microsoft.MSBuildCache.SharedCompilation.$(MSBuildCachePackageVersion)</MSBuildCacheSharedCompilationPackageRoot>
+  </PropertyGroup>
+
+  <ImportGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
+    <Import Project="$(MSBuildCachePackageRoot)\build\$(MSBuildCachePackageName).props" />
+    <Import Project="$(MSBuildCacheSharedCompilationPackageRoot)\build\Microsoft.MSBuildCache.SharedCompilation.props" />
+  </ImportGroup>
+</Project>
--- a/Directory.Build.targets
+++ b/Directory.Build.targets
@ -0,0 +1,11 @@
+<Project>
+  <!--
+    NOTE! This file gets written-over entirely by the release builds.
+    Any build logic in this file must be optional and only apply to non-release builds.
+  -->
+
+  <ImportGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
+    <Import Project="$(MSBuildCachePackageRoot)\build\$(MSBuildCachePackageName).targets" />
+    <Import Project="$(MSBuildCacheSharedCompilationPackageRoot)\build\Microsoft.MSBuildCache.SharedCompilation.targets" />
+  </ImportGroup>
+</Project>
--- a/build/pipelines/ci-caching.yml
+++ b/build/pipelines/ci-caching.yml
@ -1,30 +1,32 @@
 trigger:
  batch: true
-#  branches:
-#    include:
-#      - main
-#      - feature/*
-#      - gh-readonly-queue/*
-#  paths:
-#    exclude:
-#      - doc/*
-#      - samples/*
-#      - tools/*
+  branches:
+    include:
+      - main
+      - feature/*
+      - gh-readonly-queue/*
+  paths:
+    exclude:
+      - doc/*
+      - samples/*
+      - tools/*

-#pr:
-#  branches:
-#    include:
-#      - main
-#      - feature/*
-#  paths:
-#    exclude:
-#      - doc/*
-#      - samples/*
-#      - tools/*
+pr:
+  branches:
+    include:
+      - main
+      - feature/*
+  paths:
+    exclude:
+      - doc/*
+      - samples/*
+      - tools/*

 variables:
  - name: runCodesignValidationInjectionBG
    value: false
+  - name: EnablePipelineCache 
+    value: true

 #     0.0.yyMM.dd##
 #     0.0.1904.0900
@ -81,6 +83,8 @@ stages:
            buildConfigurations: [Release]
            buildEverything: true
            keepAllExpensiveBuildOutputs: false
+            ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}:
+              enableCaching: true

    - ${{ if eq(parameters.runTests, true) }}:
      - stage: Test_${{ platform }}
--- a/build/pipelines/templates-v2/job-build-project.yml
+++ b/build/pipelines/templates-v2/job-build-project.yml
@ -68,6 +68,9 @@ parameters:
  - name: signingIdentity
    type: object
    default: {}
+  - name: enableCaching
+    type: boolean
+    default: false

 jobs:
 - job: ${{ parameters.jobName }}
@ -95,6 +98,7 @@ jobs:
    # Yup.
    BuildTargetParameter: ' '
    SelectedSigningFragments: ' '
+    MSBuildCacheParameters: ' '
    # When building the unpackaged distribution, build it in portable mode if it's Canary-branded
    ${{ if eq(parameters.branding, 'Canary') }}:
      UnpackagedBuildArguments: -PortableMode
@ -111,6 +115,7 @@ jobs:
    clean: true
    submodules: true
    persistCredentials: True
+
  # This generates either nothing for BuildTargetParameter, or /t:X;Y;Z, to control targets later.
  - pwsh: |-
      If (-Not [bool]::Parse("${{ parameters.buildEverything }}")) {
@ -139,6 +144,17 @@ jobs:
      }
    displayName: Prepare Build and Sign Targets

+  - ${{ if eq(parameters.enableCaching, true) }}:
+    - pwsh: |-
+        $MSBuildCacheParameters = ""
+        $MSBuildCacheParameters += " -graph"
+        $MSBuildCacheParameters += " -reportfileaccesses"
+        $MSBuildCacheParameters += " -p:MSBuildCacheEnabled=true"
+        $MSBuildCacheParameters += " -p:MSBuildCacheLogDirectory=$(Build.SourcesDirectory)\MSBuildCacheLogs"
+        Write-Host "MSBuildCacheParameters: $MSBuildCacheParameters"
+        Write-Host "##vso[task.setvariable variable=MSBuildCacheParameters]$MSBuildCacheParameters"
+      displayName: Prepare MSBuildCache variables
+
  - pwsh: |-
      .\build\scripts\Generate-ThirdPartyNotices.ps1 -MarkdownNoticePath .\NOTICE.md -OutputPath .\src\cascadia\CascadiaPackage\NOTICE.html
    displayName: Generate NOTICE.html from NOTICE.md
@ -160,21 +176,37 @@ jobs:
        ${{ parameters.additionalBuildOptions }}
        /bl:$(Build.SourcesDirectory)\msbuild.binlog
        $(BuildTargetParameter)
+        $(MSBuildCacheParameters)
      platform: $(BuildPlatform)
      configuration: $(BuildConfiguration)
+      msbuildArchitecture: x64
      maximumCpuCount: true
+    ${{ if eq(parameters.enableCaching, true) }}:
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

  - ${{ if eq(parameters.publishArtifacts, true) }}:
    - publish: $(Build.SourcesDirectory)/msbuild.binlog
      artifact: logs-$(BuildPlatform)-$(BuildConfiguration)${{ parameters.artifactStem }}
      condition: always()
      displayName: Publish Build Log
+    - ${{ if eq(parameters.enableCaching, true) }}:
+      - publish: $(Build.SourcesDirectory)\MSBuildCacheLogs
+        artifact: logs-msbuildcache-$(BuildPlatform)-$(BuildConfiguration)${{ parameters.artifactStem }}
+        condition: always()
+        displayName: Publish MSBuildCache Logs
  - ${{ else }}:
    - task: CopyFiles@2
      displayName: Copy Build Log
      inputs:
        contents: $(Build.SourcesDirectory)/msbuild.binlog
        TargetFolder: $(Terminal.BinDir)
+    - ${{ if eq(parameters.enableCaching, true) }}:
+      - task: CopyFiles@2
+        displayName: Copy MSBuildCache Logs
+        inputs:
+          contents: $(Build.SourcesDirectory)/MSBuildCacheLogs/**
+          TargetFolder: $(Terminal.BinDir)/MSBuildCacheLogs

  # This saves ~2GiB per architecture. We won't need these later.
  # Removes:
--- a/dep/nuget/packages.config
+++ b/dep/nuget/packages.config
@ -17,4 +17,9 @@
  <package id="Newtonsoft.Json" version="13.0.1" targetFramework="net45" />
  <package id="Selenium.Support" version="3.5.0" targetFramework="net45" />
  <package id="Selenium.WebDriver" version="3.5.0" targetFramework="net45" />
+
+  <!-- MSBuildCache -->
+  <package id="Microsoft.MSBuildCache.AzurePipelines" version="0.1.273-preview" />
+  <package id="Microsoft.MSBuildCache.Local" version="0.1.273-preview" />
+  <package id="Microsoft.MSBuildCache.SharedCompilation" version="0.1.273-preview" />
 </packages>