[main] Update common Docker engineering infrastructure with latest (#5843)

Co-authored-by: Logan Bussell <loganbussell@microsoft.com>
2024-09-05 15:43:09 -07:00 · 2024-09-05 15:43:09 -07:00 · 487bd3d77c
--- a/eng/common/templates/1es-official.yml
+++ b/eng/common/templates/1es-official.yml
@ -20,14 +20,14 @@ parameters:
 - name: pool
  type: object
  default:
-    name: NetCore1ESPool-Internal
-    image: 1es-ubuntu-2204
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
    os: linux
 - name: sourceAnalysisPool
  type: object
  default:
-    name: NetCore1ESPool-Internal
-    image: 1es-windows-2022
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
    os: windows

 resources:
--- a/eng/common/templates/1es-unofficial.yml
+++ b/eng/common/templates/1es-unofficial.yml
@ -23,8 +23,14 @@ parameters:
 - name: pool
  type: object
  default:
-    name: NetCore1ESPool-Internal
-    image: 1es-windows-2022
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
+    os: linux
+- name: sourceAnalysisPool
+  type: object
+  default:
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
    os: windows

 resources:
@ -50,4 +56,5 @@ extends:
        exclude:
        - repository: InternalVersionsRepo
        - repository: PublicVersionsRepo
+      sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
    stages: ${{ parameters.stages }}
--- a/eng/common/templates/jobs/cg-build-projects.yml
+++ b/eng/common/templates/jobs/cg-build-projects.yml
@ -12,8 +12,8 @@ jobs:
 - job: BuildProjects
  displayName: Build Projects
  pool:
-    name: NetCore1ESPool-Internal
-    image: 1es-ubuntu-2204
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
    os: linux
  steps:
  - powershell: >
--- a/eng/common/templates/jobs/publish.yml
+++ b/eng/common/templates/jobs/publish.yml
@ -7,6 +7,7 @@ parameters:
 jobs:
 - job: Publish
  pool: ${{ parameters.pool }}
+  timeoutInMinutes: 90
  variables:
  - name: imageBuilder.commonCmdArgs
    value: >
@ -27,6 +28,8 @@ jobs:
    value: $(artifactsPath)/imageInfo
  - name: sourceBuildIdOutputDir
    value: $(Build.ArtifactStagingDirectory)/sourceBuildId
+  - name: imageInfoRawContentUrl
+    value: https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)
  - ${{ parameters.customPublishVariables }}
  steps:
  - template: /eng/common/templates/steps/retain-build.yml@self
@ -103,6 +106,10 @@ jobs:
    parameters:
      dryRunArg: $(dryRunArg)
      condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
+  - script: |
+      mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
+      curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-old.json $(imageInfoRawContentUrl)
+    displayName: Download Original Image Info File
  - script: >
      $(runImageBuilderCmd) publishImageInfo
      '$(imageInfoContainerDir)/image-info.json'
@ -117,6 +124,8 @@ jobs:
      $(imageBuilder.commonCmdArgs)
    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
    displayName: Publish Image Info
+  - script: curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-new.json $(imageInfoRawContentUrl)
+    displayName: Download Updated Image Info File
  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
    parameters:
      displayName: Ingest Kusto Image Info
@ -134,6 +143,33 @@ jobs:
        --architecture '*'
        $(dryRunArg)
        $(imageBuilder.commonCmdArgs)
+  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: Generate EOL Annotation Data
+      serviceConnection: $(publish.serviceConnectionName)
+      internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
+      args: >
+        generateEolAnnotationData
+        '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
+        '$(artifactsPath)/eol-annotation-data/image-info-old.json'
+        '$(artifactsPath)/eol-annotation-data/image-info-new.json'
+        '$(publishRepoPrefix)'
+        '$(acr.server)'
+        $(generateEolAnnotationDataExtraOptions)
+        $(dryRunArg)
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/eol-annotation-data
+      artifactName: eol-annotation-data-$(System.JobAttempt)
+      displayName: Publish EOL Annotation Data Artifact
+      internalProjectName: internal
+      publicProjectName: public
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
+  - template: /eng/common/templates/steps/annotate-eol-digests.yml@self
+    parameters:
+      internalProjectName: ${{ parameters.internalProjectName }}
+      dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
  - script: >
      $(runImageBuilderCmd) postPublishNotification
      '$(publishNotificationRepoName)'
@ -154,6 +190,9 @@ jobs:
      --task "Wait for MCR Doc Ingestion (Authenticated)"
      --task "Publish Image Info"
      --task "Ingest Kusto Image Info (Authenticated)"
+      --task "Generate EOL Annotation Data (Authenticated)"
+      --task "Annotate EOL Images (Authenticated)"
+      --task "Wait for Annotation Ingestion (Authenticated)"
      $(dryRunArg)
      $(imageBuilder.commonCmdArgs)
    displayName: Post Publish Notification
--- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml
+++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml
@ -70,47 +70,53 @@ stages:
      ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
        vmImage: $(defaultLinuxAmd64PoolImage)
      ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: NetCore1ESPool-Internal
-        image: 1es-ubuntu-2204
+        name: $(linuxAmd64InternalPoolName)
+        image: $(linuxAmd64InternalPoolImage)
        os: linux

    # Linux Arm64
    linuxArm64Pool:
      os: linux
      hostArchitecture: Arm64
-      image: Mariner-2-Docker-ARM64
+      image: $(linuxArm64PoolImage)
      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
-        name: Docker-Linux-Arm-Public
+        name: $(linuxArm64PublicPoolName)
      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: Docker-Linux-Arm-Internal
+        name: $(linuxArm64InternalPoolName)

    # Linux Arm32
    linuxArm32Pool:
      os: linux
      hostArchitecture: Arm64
-      image: Mariner-2-Docker-ARM64
+      image: $(linuxArm32PoolImage)
      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
-        name: Docker-Linux-Arm-Public
+        name: $(linuxArm32PublicPoolName)
      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: Docker-Linux-Arm-Internal
+        name: $(linuxArm32InternalPoolName)

    # Windows Server 2016
    windows2016Pool:
      os: windows
-      name: Docker-2016-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2016PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2016PublicPoolImage)
      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2016-NESDockerBuilds-PT
+        image: $(windowsServer2016InternalPoolImage)

    # Windows Server 2019 (1809)
    windows1809Pool:
      os: windows
-      name: Docker-1809-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2019PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2019PublicPoolImage)
      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2019-1809-NESDockerBuilds-1ESPT
+        image: $(windowsServer2019InternalPoolImage)

    # Windows Server 2022
    windows2022Pool:
      os: windows
-      name: Docker-2022-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2022PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2022PublicPoolImage)
      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2022-NESDockerBuilds-1ESPT
+        image: $(windowsServer2022InternalPoolImage)
--- a/eng/common/templates/steps/annotate-eol-digests.yml
+++ b/eng/common/templates/steps/annotate-eol-digests.yml
@ -9,12 +9,14 @@ steps:
      displayName: Annotate EOL Images
      serviceConnection: $(publish.serviceConnectionName)
      internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
      args: >
        annotateEolDigests
-        /repo/${{ parameters.dataFile }}
+        ${{ parameters.dataFile }}
        $(acr.server)
        $(publishRepoPrefix)
        $(artifactsPath)/annotation-digests/annotation-digests.txt
+        $(dryRunArg)
  - template: /eng/common/templates/steps/publish-artifact.yml@self
    parameters:
      path: $(Build.ArtifactStagingDirectory)/annotation-digests
@ -22,11 +24,13 @@ steps:
      displayName: Publish Annotation Digests List
      internalProjectName: internal
      publicProjectName: public
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
    parameters:
      displayName: Wait for Annotation Ingestion
      serviceConnection: $(marStatus.serviceConnectionName)
      internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
      args: >
        waitForMarAnnotationIngestion
        $(artifactsPath)/annotation-digests/annotation-digests.txt
--- a/eng/common/templates/variables/common.yml
+++ b/eng/common/templates/variables/common.yml
@ -17,6 +17,8 @@ variables:
  value: ""
 - name: imageBuilderDockerRunExtraOptions
  value: ""
+- name: generateEolAnnotationDataExtraOptions
+  value: ""
 - name: productVersionComponents
  value: 2
 - name: imageInfoVariant
@ -56,6 +58,16 @@ variables:
 - name: defaultWindows2022PoolImage
  value: windows-2022

+- name: default1ESInternalPoolName
+  value: NetCore1ESPool-Internal
+- name: default1ESInternalPoolImage
+  value: 1es-ubuntu-2204
+
+- name: defaultSourceAnalysisPoolName
+  value: NetCore1ESPool-Internal
+- name: defaultSourceAnalysisPoolImage
+  value: 1es-windows-2022
+
 # Define these as placeholder values to allow string validation to succeed since we don't have the
 # variable group with the actual values in public builds. For internal builds, the variable group
 # will cause these values to be overridden with the real values.
--- a/eng/common/templates/variables/docker-images.yml
+++ b/eng/common/templates/variables/docker-images.yml
@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2516503
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2530171
  imageNames.imageBuilder: $(imageNames.imageBuilderName)
  imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
  imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner
--- a/eng/common/templates/variables/dotnet/common.yml
+++ b/eng/common/templates/variables/dotnet/common.yml
@ -7,6 +7,46 @@ variables:
 - name: dockerHubRegistryCreds
  value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)'

+- name: linuxAmd64InternalPoolImage
+  value: 1es-ubuntu-2204
+- name: linuxAmd64InternalPoolName
+  value: NetCore1ESPool-Internal
+
+- name: linuxArm64PoolImage
+  value: Mariner-2-Docker-ARM64
+- name: linuxArm64PublicPoolName
+  value: Docker-Linux-Arm-Public
+- name: linuxArm64InternalPoolName
+  value: Docker-Linux-Arm-Internal
+
+- name: linuxArm32PoolImage
+  value: Mariner-2-Docker-ARM64
+- name: linuxArm32PublicPoolName
+  value: Docker-Linux-Arm-Public
+- name: linuxArm32InternalPoolName
+  value: Docker-Linux-Arm-Internal
+
+- name: windowsServer2016PublicPoolImage
+  value: Server2016-NESDockerBuilds
+- name: windowsServer2016InternalPoolImage
+  value: Server2016-NESDockerBuilds-PT
+- name: windowsServer2016PoolName
+  value: Docker-2016-${{ variables['System.TeamProject'] }}
+
+- name: windowsServer2019PublicPoolImage
+  value: Server2019-1809-NESDockerBuilds
+- name: windowsServer2019InternalPoolImage
+  value: Server2019-1809-NESDockerBuilds-1ESPT
+- name: windowsServer2019PoolName
+  value: Docker-1809-${{ variables['System.TeamProject'] }}
+
+- name: windowsServer2022PublicPoolImage
+  value: Server2022-NESDockerBuilds
+- name: windowsServer2022InternalPoolImage
+  value: Server2022-NESDockerBuilds-1ESPT
+- name: windowsServer2022PoolName
+  value: Docker-2022-${{ variables['System.TeamProject'] }}
+
 - group: DotNet-Docker-Common
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
  - group: DotNet-Docker-Secrets-WIF
--- a/eng/pipelines/update-readmes.yml
+++ b/eng/pipelines/update-readmes.yml
@ -13,10 +13,6 @@ extends:
      jobs:
      - job: UpdateReadmes
        displayName: Update Readmes
-        pool:
-          name: NetCore1ESPool-Internal
-          image: 1es-ubuntu-2204
-          os: linux
        steps:
        - template: /eng/common/templates/steps/init-docker-linux.yml@self
        - template: /eng/pipelines/steps/set-public-source-branch-var.yml@self