Update Android base image to 2023-12-05. (#35526)

Also tells Docker to invalidate caches so that our apt-get update attempt actually does something.
2023-12-06 12:17:01 -08:00 · 2023-12-06 12:17:01 -08:00 · 24d884e1b8
--- a/scripts/azure-pipelines/android/Dockerfile
+++ b/scripts/azure-pipelines/android/Dockerfile
@ -1,55 +1,65 @@
-FROM ubuntu:20.04
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN \
-  apt-get update && \
-  apt-get -y upgrade && \
-  apt-get -y --no-install-recommends install git g++ wget curl zip vim \
-  pkg-config tar cmake unzip ca-certificates
-
-# Download Android NDK
-RUN \
-  wget https://dl.google.com/android/repository/android-ndk-r25c-linux.zip && \
-  unzip android-ndk-r25c-linux.zip && \
-  rm -rf android-ndk-r25c-linux.zip
-
-ENV ANDROID_NDK_HOME /android-ndk-r25c
-
-# Download Powershell
-RUN \
-  wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb && \
-  dpkg -i packages-microsoft-prod.deb && \
-  rm -f packages-microsoft-prod.deb && \
-  apt-get -y install software-properties-common && \
-  apt-get update && \
-  add-apt-repository universe && \
-  apt-get -y install powershell
-
-RUN \
-  apt-get -y --no-install-recommends install autoconf nasm bison python2 \
-  flex build-essential libtool libtool-bin gettext automake autoconf-archive
-
-# python related
-RUN \
-  apt-get -y --no-install-recommends install python3-setuptools python3-pip python3-venv
-
-# freeglut
-RUN \
-  apt-get -y --no-install-recommends install libxi-dev libgl1-mesa-dev \
-  libglu1-mesa-dev mesa-common-dev libxrandr-dev libxxf86vm-dev
-
-# glfw3
-RUN \
-  apt-get -y --no-install-recommends install libxinerama-dev libxcursor-dev
-
-# required by qt5-base
-RUN \
-  apt-get -y --no-install-recommends install libxext-dev libxfixes-dev libxrender-dev \
-  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev \
-  libxkbcommon-dev libxcb-keysyms1-dev \
-  libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev \
-  libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \
-  libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev \
-  libxcb-cursor-dev libxkbcommon-x11-dev
-
-WORKDIR /vcpkg
+# syntax=docker/dockerfile:1.4
+FROM ubuntu:20.04
+
+ADD https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb /packages-microsoft-prod.deb
+ADD https://dl.google.com/android/repository/android-ndk-r25c-linux.zip /android-ndk-r25c-linux.zip
+
+# Add apt packages
+
+## vcpkg prerequisites
+ENV APT_PACKAGES="git curl zip unzip tar"
+
+## Common build prereqs
+ENV APT_PACKAGES="$APT_PACKAGES g++ vim pkg-config cmake ca-certificates"
+
+ENV APT_PACKAGES="$APT_PACKAGES autoconf nasm bison python2 flex build-essential libtool libtool-bin gettext automake autoconf-archive"
+
+## Python related
+ENV APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-pip python3-venv"
+
+## freeglut
+ENV APT_PACKAGES="$APT_PACKAGES libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxrandr-dev libxxf86vm-dev"
+
+# glfw3
+ENV APT_PACKAGES="$APT_PACKAGES libxinerama-dev libxcursor-dev"
+
+# qt5-base
+ENV APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \
+  libxcb1-dev libx11-xcb-dev libxcb-glx0-dev libxcb-util0-dev \
+  libxkbcommon-dev libxcb-keysyms1-dev \
+  libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev \
+  libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev \
+  libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev \
+  libxcb-cursor-dev libxkbcommon-x11-dev"
+
+## PowerShell
+ENV APT_PACKAGES="$APT_PACKAGES powershell"
+
+RUN <<END_OF_SCRIPT bash
+export DEBIAN_FRONTEND=noninteractive
+
+# Apt prereqs itself
+apt-get -y update
+apt-get -y --no-install-recommends install ca-certificates
+
+# Add apt repos
+
+## PowerShell
+dpkg -i packages-microsoft-prod.deb
+rm -f packages-microsoft-prod.deb
+add-apt-repository universe
+
+# Run apt things
+apt-get -y update
+apt-get -y dist-upgrade
+
+apt-get -y --no-install-recommends install $APT_PACKAGES
+
+# Android NDK
+unzip /android-ndk-r25c-linux.zip
+rm -f android-ndk-r25c-linux.zip
+END_OF_SCRIPT
+
+ENV ANDROID_NDK_HOME /android-ndk-r25c
+
+WORKDIR /vcpkg
--- a/scripts/azure-pipelines/android/azure-pipelines.yml
+++ b/scripts/azure-pipelines/android/azure-pipelines.yml
@ -1,136 +1,136 @@
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: MIT
-#
-
-parameters:
-  - name: vcpkgToolSha
-    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
-    type: string
-    default: 'use default'
-  - name: triplet
-    type: string
-  - name: jobName
-    type: string
-  - name: poolName
-    type: string
-  - name: dockerImage
-    type: string
-  - name: tripletPattern
-    displayName: 'Enable the triplets which contain this substring'
-    type: string
-    default: ''
-
-jobs:
- job: ${{ parameters.jobName }}
-  condition: and(succeeded(), contains('^${{ parameters.triplet }}$', '${{ parameters.tripletPattern }}'))
-  pool:
-    name: ${{ parameters.poolName }}
-  workspace:
-    clean: resources
-  timeoutInMinutes: 1440 # 1 day
-  cancelTimeoutInMinutes: 1
-  variables:
-  - name: WORKING_ROOT
-    value: /mnt/vcpkg-ci
-  - name: VCPKG_DOWNLOADS
-    value: /mnt/vcpkg-ci/downloads
-  - group: vcpkg-asset-caching-credentials
-  - name: X_VCPKG_ASSET_SOURCES
-    value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"
-  - group: vcpkg-binary-caching-credentials
-  - name: X_VCPKG_BINARY_SOURCE_STUB
-    value: "x-azblob,$(root-bin-url-wus3),$(sas-bin-wus3)"
-
-  steps:
-    # Note: /mnt is the Azure machines' temporary disk.
-  - bash: |
-      curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-      az login --identity
-      az acr login --name AndContainerRegistry
-    displayName: 'Set up managed identity'
-  - bash: |
-      sudo mkdir /home/agent -m=777
-      sudo chown `id -u` /home/agent
-      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
-      sudo rm -rf ${{ variables.WORKING_ROOT }}/failure-logs
-      sudo mkdir ${{ variables.WORKING_ROOT }}/failure-logs -m=777
-      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
-      exit 0
-    displayName: 'Create working directories'
-  - bash: |
-      docker run --rm \
-      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
-      ${{ parameters.dockerImage }} \
-      ./bootstrap-vcpkg.sh
-    displayName: 'Bootstrap vcpkg'
-    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
-  - bash: |
-      docker run --rm \
-      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
-      ${{ parameters.dockerImage }} \
-      ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
-    displayName: "Build vcpkg with CMake"
-    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
-  - bash: |
-      CONTAINERS=$(docker ps -a -q)
-      if [ -n "$CONTAINERS" ]; then
-        docker rm -f $CONTAINERS
-      fi
-    displayName: 'Stop and remove all docker containers'
-  - bash: |
-      docker pull ${{ parameters.dockerImage }}
-    displayName: 'Pull Docker Container'
-  - task: Bash@3
-    inputs:
-      targetType: inline
-      script: |
-        USER=$(id --user)
-        docker run --init -i \
-        -a stderr \
-        -a stdout \
-        --user $USER \
-        --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
-        --mount type=bind,source=$(WORKING_ROOT)/failure-logs,target=/vcpkg/failure-logs \
-        --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \
-        --env X_VCPKG_ASSET_SOURCES="x-azurl,$(root-url-ea),$(sas-ea),readwrite" \
-        ${{ parameters.dockerImage }} \
-        pwsh \
-         -File /vcpkg/scripts/azure-pipelines/test-modified-ports.ps1 \
-         -Triplet ${{ parameters.triplet }} \
-         -BuildReason $(Build.Reason) \
-         -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" \
-         -WorkingRoot ${{ variables.WORKING_ROOT }} \
-         -IsLinuxHost
-      failOnStderr: true
-    displayName: '*** Test Modified Ports'
-  - bash: |
-      CONTAINERS=$(docker ps -a -q)
-      if [ -n "$CONTAINERS" ]; then
-        docker rm -f $CONTAINERS
-      fi
-    displayName: 'Stop and remove all docker containers'
-    condition: always()
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
-    inputs:
-      PathtoPublish: '$(WORKING_ROOT)/failure-logs'
-      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
-    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
-  - bash: |
-      python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
-    displayName: 'Build a file list for all packages'
-    condition: always()
-  - task: PublishBuildArtifacts@1
-    displayName: 'Publish Artifact: file lists for ${{ parameters.triplet }}'
-    condition: always()
-    inputs:
-      PathtoPublish: scripts/list_files
-      ArtifactName: 'file lists for ${{ parameters.triplet }}'
-  - task: PublishTestResults@2
-    displayName: 'Publish Test Results'
-    condition: ne(variables['XML_RESULTS_FILE'], '')
-    inputs:
-      testRunTitle: ${{ parameters.triplet }}
-      testResultsFormat: xUnit
-      testResultsFiles: $(XML_RESULTS_FILE)
-      platform: ${{ parameters.triplet }}
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: triplet
+    type: string
+  - name: jobName
+    type: string
+  - name: poolName
+    type: string
+  - name: dockerImage
+    type: string
+  - name: tripletPattern
+    displayName: 'Enable the triplets which contain this substring'
+    type: string
+    default: ''
+
+jobs:
+- job: ${{ parameters.jobName }}
+  condition: and(succeeded(), contains('^${{ parameters.triplet }}$', '${{ parameters.tripletPattern }}'))
+  pool:
+    name: ${{ parameters.poolName }}
+  workspace:
+    clean: resources
+  timeoutInMinutes: 1440 # 1 day
+  cancelTimeoutInMinutes: 1
+  variables:
+  - name: WORKING_ROOT
+    value: /mnt/vcpkg-ci
+  - name: VCPKG_DOWNLOADS
+    value: /mnt/vcpkg-ci/downloads
+  - group: vcpkg-asset-caching-credentials
+  - name: X_VCPKG_ASSET_SOURCES
+    value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"
+  - group: vcpkg-binary-caching-credentials
+  - name: X_VCPKG_BINARY_SOURCE_STUB
+    value: "x-azblob,$(root-bin-url-wus3),$(sas-bin-wus3)"
+
+  steps:
+    # Note: /mnt is the Azure machines' temporary disk.
+  - bash: |
+      curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      az login --identity
+      az acr login --name AndContainerRegistry
+    displayName: 'Set up managed identity'
+  - bash: |
+      sudo mkdir /home/agent -m=777
+      sudo chown `id -u` /home/agent
+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
+      sudo rm -rf ${{ variables.WORKING_ROOT }}/failure-logs
+      sudo mkdir ${{ variables.WORKING_ROOT }}/failure-logs -m=777
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
+      exit 0
+    displayName: 'Create working directories'
+  - bash: |
+      docker run --rm \
+      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+      ${{ parameters.dockerImage }} \
+      ./bootstrap-vcpkg.sh
+    displayName: 'Bootstrap vcpkg'
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: |
+      docker run --rm \
+      --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+      ${{ parameters.dockerImage }} \
+      ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: |
+      CONTAINERS=$(docker ps -a -q)
+      if [ -n "$CONTAINERS" ]; then
+        docker rm -f $CONTAINERS
+      fi
+    displayName: 'Stop and remove all docker containers'
+  - bash: |
+      docker pull ${{ parameters.dockerImage }}
+    displayName: 'Pull Docker Container'
+  - task: Bash@3
+    inputs:
+      targetType: inline
+      script: |
+        USER=$(id --user)
+        docker run --init -i \
+        -a stderr \
+        -a stdout \
+        --user $USER \
+        --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \
+        --mount type=bind,source=$(WORKING_ROOT)/failure-logs,target=/vcpkg/failure-logs \
+        --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \
+        --env X_VCPKG_ASSET_SOURCES="x-azurl,$(root-url-ea),$(sas-ea),readwrite" \
+        ${{ parameters.dockerImage }} \
+        pwsh \
+         -File /vcpkg/scripts/azure-pipelines/test-modified-ports.ps1 \
+         -Triplet ${{ parameters.triplet }} \
+         -BuildReason $(Build.Reason) \
+         -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" \
+         -WorkingRoot ${{ variables.WORKING_ROOT }} \
+         -IsLinuxHost
+      failOnStderr: true
+    displayName: '*** Test Modified Ports'
+  - bash: |
+      CONTAINERS=$(docker ps -a -q)
+      if [ -n "$CONTAINERS" ]; then
+        docker rm -f $CONTAINERS
+      fi
+    displayName: 'Stop and remove all docker containers'
+    condition: always()
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
+    inputs:
+      PathtoPublish: '$(WORKING_ROOT)/failure-logs'
+      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
+    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
+  - bash: |
+      python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/
+    displayName: 'Build a file list for all packages'
+    condition: always()
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: file lists for ${{ parameters.triplet }}'
+    condition: always()
+    inputs:
+      PathtoPublish: scripts/list_files
+      ArtifactName: 'file lists for ${{ parameters.triplet }}'
+  - task: PublishTestResults@2
+    displayName: 'Publish Test Results'
+    condition: ne(variables['XML_RESULTS_FILE'], '')
+    inputs:
+      testRunTitle: ${{ parameters.triplet }}
+      testResultsFormat: xUnit
+      testResultsFiles: $(XML_RESULTS_FILE)
+      platform: ${{ parameters.triplet }}
--- a/scripts/azure-pipelines/android/create-docker-image.ps1
+++ b/scripts/azure-pipelines/android/create-docker-image.ps1
@ -1,37 +1,39 @@
-# Create Docker image for Android
-
-$Location = 'westus3'
-$Date = (Get-Date -Format 'yyyy-MM-dd')
-$ResourceGroupName = "And-Registry"
-$ContainerRegistryName = "AndContainerRegistry"
-$ErrorActionPreference = 'Stop'
-
-Get-AzResourceGroup -Name $ResourceGroupName -ErrorVariable error -ErrorAction SilentlyContinue
-if ($error) {
-    New-AzResourceGroup -Name $ResourceGroupName -Location $Location
-    New-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName -EnableAdminUser -Sku Basic
-}
-
-$registry = Get-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName
-Connect-AzContainerRegistry -Name $registry.Name
-
-$imageName = "vcpkg-android"
-Push-Location $PSScriptRoot
-try {
-    docker build . -t $imageName
-    
-    $remote = [string]::Format('andcontainerregistry.azurecr.io/{0}:{1}', $imageName, $Date)
-    docker tag $imageName $remote
-    
-    docker push $remote
-    
-    #removes from local environment
-    docker rmi --force $remote $imageName
-    
-    # pulls and runs ...
-    docker logout
-} finally {
-    Pop-Location
-}
-
-Write-Host "Remote: $remote"
+# Create Docker image for Android
+
+$Location = 'westus3'
+$Date = (Get-Date -Format 'yyyy-MM-dd')
+$ResourceGroupName = "And-Registry"
+$ContainerRegistryName = "AndContainerRegistry"
+$ErrorActionPreference = 'Stop'
+
+Get-AzResourceGroup -Name $ResourceGroupName -ErrorVariable error -ErrorAction SilentlyContinue
+if ($error) {
+    New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+    New-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName -EnableAdminUser -Sku Basic
+}
+
+$registry = Get-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName
+Connect-AzContainerRegistry -Name $registry.Name
+
+$imageName = "vcpkg-android"
+Push-Location $PSScriptRoot
+try {
+    docker builder prune -f --filter "until=24h"
+
+    docker build . -t $imageName
+    
+    $remote = [string]::Format('andcontainerregistry.azurecr.io/{0}:{1}', $imageName, $Date)
+    docker tag $imageName $remote
+    
+    docker push $remote
+    
+    #removes from local environment
+    docker rmi --force $remote $imageName
+    
+    # pulls and runs ...
+    docker logout
+} finally {
+    Pop-Location
+}
+
+Write-Host "Remote: $remote"
--- a/scripts/azure-pipelines/android/provision-image.sh
+++ b/scripts/azure-pipelines/android/provision-image.sh
@ -13,4 +13,4 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/
 echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
 apt-get update
-apt-get -y --no-install-recommends install docker-ce docker-ce-cli
+apt-get -y --no-install-recommends install docker-ce docker-ce-cli
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@ -6,7 +6,7 @@ variables:
  linux-pool: 'PrLin-1ES-Pool'
  windows-pool: 'PrWin-2023-10-11'
  osx-pool: 'PrOsx-2023-09-11'
-  linux-docker-image: 'andcontainerregistry.azurecr.io/vcpkg-android:2023-10-11'
+  linux-docker-image: 'andcontainerregistry.azurecr.io/vcpkg-android:2023-12-05'

 parameters:
  - name: vcpkgToolSha