Migrate to PMC v4 for publishing (#3674)

2023-06-03 17:31:36 -07:00 · 2023-06-03 17:31:36 -07:00 · 0aa9bf0f52
--- a/.azure/OneBranch.Publish.yml
+++ b/.azure/OneBranch.Publish.yml
@ -18,39 +18,51 @@ parameters:
 - name: opensslrpmrepos
  type: object
  default:
-  - 582bd4c5ae062a5d0fec5b8b # microsoft-rhel7.3-prod
-  - 584a0f48d6a6e37205720776 # microsoft-sles12-prod
-  - 59d40cdcf3c7fa07032ce385 # microsoft-centos7-prod
-  - 5c38ea9dea0fc9f93bd67db4 # microsoft-opensuse15-pro
-  - 5c3d1796ea0fc9f93bd67def # microsoft-sles15-prod
-  - 5e5ed94a523a8019fe47607e # microsoft-centos8-prod
-  - 5e8526cde45fff4588da61f9 # microsoft-fedora32-prod
-  - 5f7e2cfb68e42e6e7085f4df # microsoft-fedora33-prod
-  - 6001dd94435efd1330acd076 # microsoft-rhel8.1-prod
-  - 606e1da573e50659b0803a7b # microsoft-fedora34-prod
-  - 6271bc683ac6d73aa84d6737 # microsoft-fedora36-prod
-  - 6400e6f92dd6874e6880b590 # microsoft-fedora37-prod
+  - microsoft-sles12-prod-yum
+  - microsoft-sles15-prod-yum
+  - microsoft-centos7-prod-yum
+  - microsoft-centos8-prod-yum
+  - microsoft-opensuse15-prod-yum
+  - microsoft-fedora32-prod-yum
+  - microsoft-fedora33-prod-yum
+  - microsoft-fedora34-prod-yum
+  - microsoft-rhel7.3-prod-yum
+  - microsoft-rhel8.0-prod-yum
+  - microsoft-rhel8.1-prod-yum
+  - cbl-mariner-1.0-prod-Microsoft-x86_64-rpms-yum
+  - cbl-mariner-2.0-prod-Microsoft-x86_64-yum
+  - cbl-mariner-2.0-prod-Microsoft-aarch64-yum
 - name: openssldebrepos
  type: object
  default:
-  - 582bd623ae062a5d0fec5b8c # microsoft-ubuntu-xenial-prod
-  - 599211761cc20bce4a8ab950 # microsoft-debian-stretch-prod
-  - 5a9dc3f2424a5c053cc3ff2e # microsoft-ubuntu-bionic-prod
-  - 5d23b16c9a6e3b375bbba42e # microsoft-debian-buster-prod
-  - 5e852952e45fffa1beda61fe # microsoft-ubuntu-focal-prod
-  - 5f7e2d6668e42e03f785f4e0 # microsoft-ubuntu-groovy-prod
-  - 606e057173e5060519803a74 # microsoft-ubuntu-hirsute-prod
-  - 611ab3a32acdcd0744c8c841 # microsoft-debian-bullseye-prod
+  - microsoft-ubuntu-xenial-prod-apt
+  - microsoft-debian-stretch-prod-apt
+  - microsoft-ubuntu-bionic-prod-apt
+  - microsoft-debian-buster-prod-apt
+  - microsoft-ubuntu-focal-prod-apt
+  - microsoft-ubuntu-groovy-prod-apt
+  - microsoft-ubuntu-hirsute-prod-apt
+  - microsoft-debian-bullseye-prod-apt
 - name: openssl3debrepos
  type: object
  default:
-  - 61faea6cea3a770ab120ac8a # microsoft-ubuntu-jammy-prod
+  - microsoft-ubuntu-jammy-prod-apt
+  - microsoft-ubuntu-kinetic-prod-apt
+  - microsoft-ubuntu-lunar-prod-apt
+- name: openssl3rpmrepos
+  type: object
+  default:
+  - microsoft-fedora36-prod-yum
+  - microsoft-fedora37-prod-yum
+  - microsoft-fedora38-prod-yum
+  - microsoft-rhel9.0-prod-yum

 stages:
 - stage: UploadPackage_stage
-  condition: eq(variables['Build.SourceBranch'], 'refs/tags/')
+  condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/')
  jobs:
-  - job: UploadPackage
+  - job: UploadPackage_openssl_debs
+    displayName: Upload openSSL based DEB packages to repos
    workspace:
      clean: all
    pool:
@ -68,6 +80,52 @@ stages:
          runId: $(resources.pipeline.onebranch.runID)
          artifact: drop_package_linux_distribution_openssl
          path: $(Build.SourcesDirectory)/artifacts/signed/openssl
+      - task: DownloadSecureFile@1
+        name:  pmcv4cert
+        displayName: 'Download cert for PMC v4'
+        inputs:
+          secureFile: 'auth.pem'
+      - ${{ each repo in parameters.openssldebrepos }}:
+        - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl -r ${{ repo }} -n "*.deb"
+          displayName: ${{ repo }}
+          continueOnError: true
+  - job: UploadPackage_openssl_rpms
+    displayName: Upload openSSL based RPM packages to repos
+    workspace:
+      clean: all
+    pool:
+      vmImage: 'ubuntu-latest'
+    variables:
+    - group: MsQuicAADApp
+    steps:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          source: specific
+          project: $(resources.pipeline.onebranch.projectID)
+          pipeline: $(resources.pipeline.onebranch.pipelineID)
+          preferTriggeringPipeline: true
+          runVersion: specific
+          runId: $(resources.pipeline.onebranch.runID)
+          artifact: drop_package_linux_distribution_openssl
+          path: $(Build.SourcesDirectory)/artifacts/signed/openssl
+      - task: DownloadSecureFile@1
+        name:  pmcv4cert
+        displayName: 'Download cert for PMC v4'
+        inputs:
+          secureFile: 'auth.pem'
+      - ${{ each repo in parameters.opensslrpmrepos }}:
+        - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl -r ${{ repo }} -n "*.rpm"
+          displayName: ${{ repo }}
+          continueOnError: true
+  - job: UploadPackage_openssl3_debs
+    displayName: Upload openSSL3 based DEB packages to repos
+    workspace:
+      clean: all
+    pool:
+      vmImage: 'ubuntu-latest'
+    variables:
+    - group: MsQuicAADApp
+    steps:
      - task: DownloadPipelineArtifact@2
        inputs:
          source: specific
@ -78,17 +136,40 @@ stages:
          runId: $(resources.pipeline.onebranch.runID)
          artifact: drop_package_linux_distribution_openssl3
          path: $(Build.SourcesDirectory)/artifacts/signed/openssl3
-      - task: Docker@2
-        displayName: Login to ACR
+      - task: DownloadSecureFile@1
+        name:  pmcv4cert
+        displayName: 'Download cert for PMC v4'
        inputs:
-          command: login
-          containerRegistry: msquicdockerregistry
-      - ${{ each repo in parameters.opensslrpmrepos }}:
-        - script: sh scripts/upload-linux-packages.sh -i $(ClientId) -s $(Secret) -f $(Build.SourcesDirectory)/artifacts/signed/openssl -r ${{ repo }} -n "*.rpm"
-          displayName: Upload openssl RPM packages to ${{ repo }}
-      - ${{ each repo in parameters.openssldebrepos }}:
-        - script: sh scripts/upload-linux-packages.sh -i $(ClientId) -s $(Secret) -f $(Build.SourcesDirectory)/artifacts/signed/openssl -r ${{ repo }} -n "*.deb"
-          displayName: Upload openssl DEB packages to ${{ repo }}
+          secureFile: 'auth.pem'
      - ${{ each repo in parameters.openssl3debrepos }}:
-        - script: sh scripts/upload-linux-packages.sh -i $(ClientId) -s $(Secret) -f $(Build.SourcesDirectory)/artifacts/signed/openssl3 -r ${{ repo }} -n "*.deb"
-          displayName: Upload openssl3 DEB packages to ${{ repo }}
+        - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl3 -r ${{ repo }} -n "*.deb"
+          displayName: ${{ repo }}
+          continueOnError: true
+  - job: UploadPackage_openssl3_rpms
+    displayName: Upload openSSL3 based RPM packages to repos
+    workspace:
+      clean: all
+    pool:
+      vmImage: 'ubuntu-latest'
+    variables:
+    - group: MsQuicAADApp
+    steps:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          source: specific
+          project: $(resources.pipeline.onebranch.projectID)
+          pipeline: $(resources.pipeline.onebranch.pipelineID)
+          preferTriggeringPipeline: true
+          runVersion: specific
+          runId: $(resources.pipeline.onebranch.runID)
+          artifact: drop_package_linux_distribution_openssl3
+          path: $(Build.SourcesDirectory)/artifacts/signed/openssl3
+      - task: DownloadSecureFile@1
+        name:  pmcv4cert
+        displayName: 'Download cert for PMC v4'
+        inputs:
+          secureFile: 'auth.pem'
+      - ${{ each repo in parameters.openssl3rpmrepos }}:
+        - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl3 -r ${{ repo }} -n "*.rpm"
+          displayName: ${{ repo }}
+          continueOnError: true
--- a/.azure/dockers/ob/pkg-publish/Dockerfile
+++ b/.azure/dockers/ob/pkg-publish/Dockerfile
@ -1,31 +0,0 @@
-FROM ubuntu:latest
-
-LABEL maintainer="huanyi"
-
-COPY publish-packages.sh /usr/src/
-
-RUN chmod 755 /usr/src/publish-packages.sh
-
-WORKDIR /usr/src/
-
-# Predefine tiemzone to workaround tzdata installation stuck
-ENV TZ=America/Los_Angeles
-
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-# Install prerequisites
-RUN apt-get update && apt-get install -y \
-    curl \
-    gnupg2
-
-RUN curl http://tux-devrepo.corp.microsoft.com/keys/tux-devrepo.asc > tux-devrepo.asc; apt-key add tux-devrepo.asc
-
-RUN echo "deb [arch=amd64] http://tux-devrepo.corp.microsoft.com/repos/tux-dev/ xenial main" | tee /etc/apt/sources.list.d/tuxdev.list
-
-RUN apt-get update
-
-RUN apt-get install azure-repoapi-client -y
-
-RUN mkdir ~/.repoclient/
-
-ENTRYPOINT ["./publish-packages.sh"]
--- a/.azure/dockers/ob/pkg-publish/publish-packages.sh
+++ b/.azure/dockers/ob/pkg-publish/publish-packages.sh
@ -1,32 +0,0 @@
-#!/bin/bash
-while getopts i:s:f:r: flag
-do
-    case "${flag}" in
-        i) AADClientId=${OPTARG};;
-        s) AADClientSecret=${OPTARG};;
-        f) FilePath=${OPTARG};;
-        r) Repo=${OPTARG};;
-    esac
-done
-echo "AADClientId: $AADClientId"
-echo "FilePath: $FilePath"
-ls -lsa $FilePath
-ConfigString="
-{
-    \"server\": \"azure-apt-cat.cloudapp.net\",
-    \"port\": \"443\",
-    \"AADClientId\": \"$AADClientId\",
-    \"AADClientSecret\": \"$AADClientSecret\",
-    \"AADResource\": \"https://microsoft.onmicrosoft.com/945999e9-da09-4b5b-878f-b66c414602c0\",
-    \"AADTenant\": \"72f988bf-86f1-41af-91ab-2d7cd011db47\",
-    \"AADAuthorityUrl\": \"https://login.microsoftonline.com\",
-    \"repositoryId\": \"5ca39edc03f790615107e1e1\"
-}
-"
-
-echo $ConfigString | jq > ~/.repoclient/prodconfig.json
-echo "publish-docker: Uploading to $FilePath to repo $Repo"
-res=`repoclient -s pmc -v v3 package add -k $FilePath -r $Repo`
-echo $res | jq
-echo "publish-docker: submissionId:"
-echo $res | jq -r '.message.submissionId?'
--- a/scripts/upload-linux-packages.sh
+++ b/scripts/upload-linux-packages.sh
@ -1,17 +1,67 @@
 #!/bin/bash
-while getopts i:s:f:r:n: flag
+
+while getopts i:c:f:r:n:l: flag
 do
    case "${flag}" in
-        i) AADClientId=${OPTARG};;
-        s) AADClientSecret=${OPTARG};;
+        i) ClientId=${OPTARG};;
+        c) Cert=${OPTARG};;
        f) Folder=${OPTARG};;
        r) Repo=${OPTARG};;
        n) NameFilter=${OPTARG};;
    esac
 done

+ConfigString="
+[prod]\n
+base_url = \"https://pmc-ingest.trafficmanager.net/api/v4\"\n
+msal_client_id = \"$ClientId\"\n
+msal_scope = \"api://d48bb382-20ec-41b9-a0ea-07758a21ccd0/.default\"\n
+msal_cert_path = \"~/.config/pmc/auth.pem\"\n
+msal_SNIAuth = true\n
+msal_authority = \"https://login.microsoftonline.com/MSAzureCloud.onmicrosoft.com\"\n
+"
+
+mkdir -p ~/.config/pmc/
+echo -e $ConfigString > ~/.config/pmc/settings.toml
+cp $Cert ~/.config/pmc/auth.pem
+pmc() {
+    docker run -t --volume ~/.config/pmc:/root/.config/pmc --volume "$Folder":/root/packages --rm mcr.microsoft.com/pmc/pmc-cli "$@"
+}
+IFS='-' read -ra parts <<< "$Repo"
+if [ "${parts[-1]}" = "apt" ]; then
+    echo "apt repo"
+    release=${parts[2]}
+    echo "Release name: $release"
+    archesString=`pmc repo release list "$Repo" | jq -r '.results[] | select(.name == "lunar") | .architectures[]'`
+    readarray -t arches <<< "$archesString"
+    echo "Supported arches in $Repo: ${arches[@]}"
+else
+    echo "yum repo"
+    release=""
+fi
+
 for filename in `find $Folder -maxdepth 1 -type f -name "$NameFilter"`; do
    basefilename=`basename $filename`
+    if [ ! -z "$release" ]; then
+        echo -n "Checking if $basefilename is supported by the repo: "
+        match_found=false
+        for arch in "${arches[@]}"; do
+            if [[ "$basefilename" == *"$arch"* ]]; then
+                match_found=true
+            fi
+        done
+
+        if [ "$match_found" = false ]; then
+            echo "❌"
+            continue
+        else
+            echo "✅"
+        fi
+    fi
    echo "Uploading $filename to $Repo"
-    docker run -v $Folder:/usr/src/hostpwd msquicdockerregistry.azurecr.io/private/msquic/publish-linux-packages:vnext -i $AADClientId -s $AADClientSecret -f /usr/src/hostpwd/$basefilename -r $Repo
-done
+    packageId=`pmc -q --id-only package upload /root/packages/$basefilename | tr -d '\r'`
+    echo "package ID: $packageId"
+    pmc repo package update --add-packages "$packageId" "$Repo" $release
+done
+
+pmc repo publish "$Repo"