Get publication working (#3)

- add missing pipeline configuration e.g. make Tests tab readable - enable post-build signing, source-build, source-index - use `build.sh` instead of `cibuild.sh` to avoid Linux and macOS publishing - disable Linux and macOS builds in official builds and manual builds (when `skipTests`) nits: - add default VS Code settings - future-proof `vmImage` values - remove a few unnecessary settings - do not need MicroBuild or (outdated) Helix settings - eng/common/ settings handle most artifact / test result publication - remove whitespace
2021-11-22 11:25:37 -08:00 · 2021-11-22 11:25:37 -08:00 · 67f54bc819
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -0,0 +1,8 @@
+{
+    "markdownlint.config": {
+        "MD028": false,
+        "MD025": {
+            "front_matter_title": ""
+        }
+    }
+}
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -1,36 +1,95 @@
-variables:
-  - name: Build.Repository.Clean
-    value: true
-  - name: _TeamName
-    value: AspNetCore
-  - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
-    value: true
-  - name: _HelixType
-    value: build/product
-  - name: _DotNetArtifactsCategory
-    value: .NETCore
-
-  # Variables for public PR builds
-  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-    - name: _HelixSource
-      value: pr/aspnet/AspLabs/$(Build.SourceBranch)
-
-  # Variables for internal Official builds
-  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - name: _HelixSource
-      value: official/aspnet/AspLabs/$(Build.SourceBranch)
-
-resources:
-  containers:
-  - container: LinuxContainer
-    image: mcr.microsoft.com/dotnet-buildtools-prereqs:ubuntu-18.04-c103199-20180628134610
-    options: --init # This ensures all the stray defunct processes are reaped.
+#
+# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details on this file.
+#

 trigger:
- main
+  batch: true
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*

 pr:
- "*"
+  autoCancel: true
+  branches:
+    include:
+    - "*"
+
+parameters:
+# Choose whether to skip tests when running pipeline manually.
+- name: skipTests
+  default: false
+  displayName: Skip tests?
+  type: boolean
+# Choose whether to test source indexing. Ignored in public builds.
+# Will cause inaccessible links on https://source.dot.net/ unless commits are also available in GitHub.
+- name: testSourceIndexing
+  default: false
+  displayName: Test source indexing? !Danger! see comments in YAML.
+  type: boolean
+
+variables:
+- name: Build.Repository.Clean
+  value: true
+- name: _TeamName
+  value: AspNetCore
+- name: DOTNET_CLI_TELEMETRY_PROFILE
+  value: '$(Build.Repository.Uri)'
+- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
+  value: true
+- name: _DotNetArtifactsCategory
+  value: .NETCore
+- name: _DotNetPublishToBlobFeed
+  value: true
+- name: _PublishUsingPipelines
+  value: true
+- name: PostBuildSign
+  value: true
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - name: _SignType
+    value: ''
+  - name: _BuildArgs
+    value: /p:PostBuildSign=$(PostBuildSign)
+  - name: _PublishArgs
+    value: ''
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+    # DotNet-MSRC-Storage provides: dotnetclimsrc-read-sas-token-base64
+  - group: DotNet-MSRC-Storage
+  - name: _InternalBuildArgs
+    value: -RuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet
+           -RuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64)
+           /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
+           /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet
+           /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
+           /p:PostBuildSign=$(PostBuildSign)
+  - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
+    # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
+    - group: DotNet-Blob-Feed
+    # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
+    - group: Publish-Build-Assets
+    - name: _SignType
+      value: real
+    - name: _BuildArgs
+      value: $(_InternalBuildArgs)
+             /p:TeamName=$(_TeamName)
+             /p:OfficialBuildId=$(Build.BuildNumber)
+    - name: _PublishArgs
+      value: /p:Publish=true
+             /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
+             /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+             /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+             /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
+             /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+             /p:DotNetSignType=$(_SignType)
+  - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
+    - name: _SignType
+      value: test
+    - name: _BuildArgs
+      value: $(_InternalBuildArgs)
+             /p:PostBuildSign=$(PostBuildSign)
+    - name: _PublishArgs
+      value: ''

 stages:
 - stage: build
@ -39,13 +98,26 @@ stages:
  - template: /eng/common/templates/jobs/jobs.yml
    parameters:
      enablePublishBuildArtifacts: true
-      enablePublishTestResults: true
      enablePublishBuildAssets: true
+      enablePublishTestResults: true
      enablePublishUsingPipelines: true
-      helixRepo: aspnet/AspLabs
-      # enableMicrobuild can't be read from a user-defined variable (Azure DevOps limitation)
-      ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-        enableMicrobuild: true
+      enableSourceBuild: true
+      enableTelemetry: true
+      mergeTestResults: true
+      testResultsFormat: xUnit # Have no vsTest results in any job.
+      workspace:
+        clean: all
+      ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(parameters.testSourceIndexing, 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+        enableSourceIndex: true
+        sourceIndexParams:
+          dependsOn:
+          - Windows
+          - Source_Build_Complete
+          - ${{ if ne(parameters.skipTests, 'true') }}:
+            - macOS
+            - Linux
+          pool:
+            vmImage: windows-latest
      jobs:
      - job: Windows
        pool:
@ -55,29 +127,18 @@ stages:
          ${{ if ne(variables['System.TeamProject'], 'public') }}:
            name: NetCore1ESPool-Internal
            demands: ImageOverride -equals Build.Server.Amd64.VS2019
-        variables:
-        - name: _HelixBuildConfig
-          value: $(_BuildConfig)
        strategy:
          matrix:
            ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
              Debug:
                _BuildConfig: Debug
-                _SignType: test
-                _BuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
            Release:
              _BuildConfig: Release
-              # PRs and external builds are not signed.
-              ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-                _SignType: test
-                _BuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
-              ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-                _SignType: real
-                _BuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:OfficialBuildId=$(Build.BuildNumber)
-                  /p:DotNetPublishUsingPipelines=true
-                  /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
-                  /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
-                  /p:DotNetPublishToBlobFeed=true
+        testRunTitle: $(Agent.Os)-$(_BuildConfig)
+        variables:
+        - LC_ALL: 'en_US.UTF-8'
+        - LANG: 'en_US.UTF-8'
+        - LANGUAGE: 'en_US.UTF-8'
        steps:
        - checkout: self
          clean: true
@ -85,16 +146,12 @@ stages:
            -configuration $(_BuildConfig)
            -prepareMachine
            $(_BuildArgs)
+            $(_PublishArgs)
          name: Build
          displayName: Build
          condition: succeeded()
-        - task: PublishTestResults@2
-          displayName: Publish xUnit Test Results
-          condition: always()
-          continueOnError: true
-          inputs:
-            testRunner: xunit
-            testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
+        # Following somewhat duplicates BlobArtifacts and PackageArtifacts uploaded during Build step.
+        # But, the layout makes shipping packages (and symbol packages) immediately obvious.
        - task: PublishBuildArtifacts@1
          displayName: Publish Packages
          condition: and(eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release'))
@ -104,94 +161,64 @@ stages:
            parallel: true
            pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)'
            publishLocation: Container
-        - task: PublishBuildArtifacts@1
-          displayName: Publish Logs
-          condition: always()
-          continueOnError: true
-          inputs:
-            artifactName: Logs_$(Agent.Os)_$(Agent.JobName)
-            parallel: true
-            pathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
-            publishLocation: Container

-      - job: macOS
-        pool:
-          vmImage: macOS-10.14
-        strategy:
-          matrix:
-            debug:
-              _BuildConfig: Debug
-            release:
-              _BuildConfig: Release
-        variables:
-        - name: _HelixBuildConfig
-          value: $(_BuildConfig)
-        steps:
-        - checkout: self
-          clean: true
-        - script: eng/common/cibuild.sh
-            --configuration $(_BuildConfig)
-            --prepareMachine
-          name: Build
-          displayName: Build
-          condition: succeeded()
-        - task: PublishTestResults@2
-          displayName: Publish xUnit Test Results
-          condition: always()
-          continueOnError: true
-          inputs:
-            testRunner: xunit
-            testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
-        - task: PublishBuildArtifacts@1
-          displayName: Publish Logs
-          condition: always()
-          continueOnError: true
-          inputs:
-            artifactName: Logs_$(Agent.Os)_$(Agent.JobName)
-            parallel: true
-            pathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
-            publishLocation: Container
+      - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}:
+        - job: macOS
+          pool:
+            vmImage: macOS-latest
+          strategy:
+            matrix:
+              Debug:
+                _BuildConfig: Debug
+              Release:
+                _BuildConfig: Release
+          testRunTitle: $(Agent.Os)-$(_BuildConfig)
+          variables:
+          - LC_ALL: 'en_US.UTF-8'
+          - LANG: 'en_US.UTF-8'
+          - LANGUAGE: 'en_US.UTF-8'
+          steps:
+          - checkout: self
+            clean: true
+          - script: eng/common/build.sh
+              --configuration $(_BuildConfig)
+              --ci --prepareMachine --restore --build --test --pack
+              $(_BuildArgs)
+            name: Build
+            displayName: Build
+            condition: succeeded()

-      - job: Linux
-        pool:
-          vmImage: ubuntu-18.04
-          container: LinuxContainer
-        strategy:
-          matrix:
-            debug:
-              _BuildConfig: Debug
-            release:
-              _BuildConfig: Release
-        variables:
-        - name: _HelixBuildConfig
-          value: $(_BuildConfig)
-        steps:
-        - checkout: self
-          clean: true
-        - script: eng/common/cibuild.sh
-            --configuration $(_BuildConfig)
-            --prepareMachine
-          name: Build
-          displayName: Build
-          condition: succeeded()
-        - task: PublishTestResults@2
-          displayName: Publish xUnit Test Results
-          condition: always()
-          continueOnError: true
-          inputs:
-            testRunner: xunit
-            testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
-        - task: PublishBuildArtifacts@1
-          displayName: Publish Logs
-          condition: always()
-          continueOnError: true
-          inputs:
-            artifactName: Logs_$(Agent.Os)_$(Agent.JobName)
-            parallel: true
-            pathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
-            publishLocation: Container
+        - job: Linux
+          pool:
+            vmImage: ubuntu-latest
+          strategy:
+            matrix:
+              Debug:
+                _BuildConfig: Debug
+              Release:
+                _BuildConfig: Release
+          testRunTitle: $(Agent.Os)-$(_BuildConfig)
+          variables:
+          - LC_ALL: 'en_US.UTF-8'
+          - LANG: 'en_US.UTF-8'
+          - LANGUAGE: 'en_US.UTF-8'
+          steps:
+          - checkout: self
+            clean: true
+          - script: eng/common/build.sh
+              --configuration $(_BuildConfig)
+              --ci --prepareMachine --restore --build --test --pack
+              $(_BuildArgs)
+            name: Build
+            displayName: Build
+            condition: succeeded()

 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
  - template: eng\common\templates\post-build\post-build.yml
    parameters:
      publishingInfraVersion: 3
+      enableSymbolValidation: false
+      enableSigningValidation: false
+      enableNugetValidation: false
+      pool:
+        vmImage: windows-latest
--- a/eng/SourceBuild.props
+++ b/eng/SourceBuild.props
@ -0,0 +1,25 @@
+<Project>
+  <PropertyGroup>
+    <GitHubRepositoryName>razor-compiler</GitHubRepositoryName>
+    <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+    <CloneSubmodulesToInnerSourceBuildRepo>false</CloneSubmodulesToInnerSourceBuildRepo>
+  </PropertyGroup>
+
+  <!--
+    Remove inner source .globalconfig file as both the inner and outer config files get loaded and cause a conflict.
+    Leaving the inner will cause all conflicting settings to be ignored.
+    https://docs.microsoft.com/en-us/dotnet/fundamentals/code-analysis/configuration-files#general-options
+  -->
+  <Target Name="RemoveInnerGlobalConfig"
+          DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
+          BeforeTargets="RunInnerSourceBuildCommand">
+    <Delete Files="$(InnerSourceBuildRepoRoot).globalconfig" />
+  </Target>
+
+  <Target Name="CustomizeInnerBuildArgs"
+          BeforeTargets="GetSourceBuildCommandConfiguration">
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:SourceBuildRuntimeIdentifier=$(TargetRuntimeIdentifier)</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+</Project>
--- a/eng/SourceBuildPrebuiltBaseline.xml
+++ b/eng/SourceBuildPrebuiltBaseline.xml
@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>