From e58b38460b39381170eceac884c0b854f3fdd301 Mon Sep 17 00:00:00 2001 From: Daiki AMINAKA <1991.daiki@gmail.com> Date: Wed, 14 Aug 2024 13:56:37 -0700 Subject: [PATCH] Publish package to more distros and refactoring (#4455) * Publish package to more distros and refactoring * fix? * fix * fix * fix file dependencies * fix * fix * fix * fix --- .azure/OneBranch.Official.yml | 12 +- .azure/OneBranch.Publish.yml | 183 ++++++++++---------- .azure/OneBranch.PullRequest.yml | 12 +- .azure/obtemplates/build-distribution.yml | 12 +- .azure/obtemplates/build-linux-packages.yml | 16 +- .azure/obtemplates/build-linux.yml | 38 ++-- .azure/obtemplates/download-artifacts.yml | 4 +- scripts/make-packages.sh | 65 ++++--- scripts/package-distribution.ps1 | 37 ++-- 9 files changed, 195 insertions(+), 184 deletions(-) diff --git a/.azure/OneBranch.Official.yml b/.azure/OneBranch.Official.yml index 9f023aaa2..80de7ea1e 100644 --- a/.azure/OneBranch.Official.yml +++ b/.azure/OneBranch.Official.yml @@ -78,10 +78,10 @@ extends: - container: linux_build_container # Default container image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-cross' type: Linux - - container: ubuntu_2204_cross + - container: kernel5_15_cross image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-cross' type: Linux - - container: ubuntu_2404_cross + - container: kernel6_8_cross image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-24.04-cross' type: Linux @@ -179,23 +179,23 @@ extends: - template: .azure/obtemplates/build-linux.yml@self parameters: config: Release - os: ubuntu_2204 + kernel: "kernel5_15" tls: openssl3 - template: .azure/obtemplates/build-linux.yml@self parameters: config: Debug - os: ubuntu_2204 + kernel: "kernel5_15" tls: openssl3 - template: .azure/obtemplates/build-linux.yml@self parameters: config: Release - os: ubuntu_2404 + kernel: "kernel6_8" tls: openssl3 xdp: "-UseXdp" - template: .azure/obtemplates/build-linux.yml@self parameters: config: Debug - os: ubuntu_2404 + kernel: "kernel6_8" tls: openssl3 xdp: "-UseXdp" diff --git a/.azure/OneBranch.Publish.yml b/.azure/OneBranch.Publish.yml index 9df1a6cc8..0551f64ab 100644 --- a/.azure/OneBranch.Publish.yml +++ b/.azure/OneBranch.Publish.yml @@ -15,73 +15,78 @@ variables: DisableDockerDetector: true parameters: -- name: opensslrpmcblrepos + # build on ubuntu 20.04 openssl 1.1 +- name: kernel5_4rpmcblrepos type: object default: - cbl-mariner-1.0-prod-Microsoft-x86_64-rpms-yum - cbl-mariner-2.0-prod-Microsoft-x86_64-yum - cbl-mariner-2.0-prod-Microsoft-aarch64-yum -- name: opensslrpmrepos +- name: kernel5_4rpmrepos type: object default: - - microsoft-sles12-prod-yum - - microsoft-sles15-prod-yum - - microsoft-centos7-prod-yum - - microsoft-centos8-prod-yum - - microsoft-opensuse15-prod-yum - - microsoft-fedora32-prod-yum - - microsoft-fedora33-prod-yum - - microsoft-fedora34-prod-yum - - microsoft-fedora37-prod-yum - - microsoft-rhel7.3-prod-yum - - microsoft-rhel8.0-prod-yum - - microsoft-rhel8.1-prod-yum -- name: openssldebrepos + - microsoft-sles12-prod-yum # 12 3.12 + - microsoft-sles15-prod-yum # 15 4.12 + - microsoft-centos7-prod-yum # 7 3.10 + - microsoft-centos8-prod-yum # 8 4.18 + - microsoft-opensuse15-prod-yum # 15 4.12 + - microsoft-fedora32-prod-yum # 32 5.6 + - microsoft-fedora33-prod-yum # 33 5.8 + - microsoft-fedora34-prod-yum # 34 5.11 + - microsoft-fedora35-prod-yum # 35 5.14 + - microsoft-rhel7.3-prod-yum # 7.3 3.10 + - microsoft-rhel8.0-prod-yum # 8.0 4.18 + - microsoft-rhel8.1-prod-yum # 8.1 4.18 +- name: kernel5_4debrepos type: object default: - - microsoft-debian-stretch-prod-apt - - microsoft-debian-buster-prod-apt - - microsoft-debian-bullseye-prod-apt -- name: openssl3debrepos + - microsoft-ubuntu-xenial-prod-apt # 16.04 4.4 + - microsoft-ubuntu-bionic-prod-apt # 18.04 4.15 + - microsoft-ubuntu-focal-prod-apt # 20.04 5.4 + - microsoft-ubuntu-groovy-prod-apt # 20.10 5.8 + - microsoft-ubuntu-hirsute-prod-apt # 21.04 5.11 + - microsoft-debian-stretch-prod-apt # 9 4.9 + - microsoft-debian-buster-prod-apt # 10 4.19 + - microsoft-debian-bullseye-prod-apt # 11 5.10 + + # built on ubuntu 22.04, openssl3 +- name: kernel5_15rpmrepos type: object default: - - microsoft-debian-bookworm-prod-apt -- name: openssl3rpmrepos + - microsoft-fedora36-prod-yum # 36 5.17 + - microsoft-fedora37-prod-yum # 37 6.0 + - microsoft-fedora38-prod-yum # 38 6.2 + - microsoft-fedora39-prod-yum # 39 6.5 + - microsoft-rhel9.0-prod-yum # 9.0 5.14 +- name: kernel5_15debrepos type: object default: - - microsoft-fedora36-prod-yum - - microsoft-fedora38-prod-yum - - microsoft-fedora39-prod-yum - - microsoft-rhel9.0-prod-yum -- name: debug # debug mode will not actually upload and publish packages - type: boolean - default: false -- name: ubuntu20repos + - microsoft-ubuntu-jammy-prod-apt # 22.04 5.15 + - microsoft-ubuntu-kinetic-prod-apt # 22.10 5.19 + - microsoft-ubuntu-lunar-prod-apt # 23.04 6.2 + - microsoft-ubuntu-mantic-prod-apt # 23.10 6.5 + - microsoft-debian-bookworm-prod-apt # 12 6.1 + + # built on ubuntu 24.04 openssl3 XDP +- name: kernel6_8cblrpmrepos type: object default: - - microsoft-ubuntu-xenial-prod-apt - - microsoft-ubuntu-bionic-prod-apt - - microsoft-ubuntu-focal-prod-apt - - microsoft-ubuntu-groovy-prod-apt - - microsoft-ubuntu-hirsute-prod-apt -- name: ubuntu22repos + - azurelinux-3.0-prod-ms-oss-x86_64-yum # 3.0 6.6 +- name: kernel6_8rpmrepos type: object default: - - microsoft-ubuntu-jammy-prod-apt - - microsoft-ubuntu-kinetic-prod-apt - - microsoft-ubuntu-lunar-prod-apt - - microsoft-ubuntu-mantic-prod-apt -- name: ubuntu24repos + - microsoft-fedora40-prod-yum # 40 6.8 +- name: kernel6_8debrepos type: object default: - - microsoft-ubuntu-noble-prod-apt + - microsoft-ubuntu-noble-prod-apt # 24.04 6.8 stages: - stage: UploadPackage_stage condition: or(startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['Build.Reason'], 'Manual')) jobs: - - job: UploadPackage_openssl_debs - displayName: Upload openSSL based DEB packages to repos + - job: UploadPackage_kernel5_4_debs + displayName: Upload kernel5_4 based DEB packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -98,20 +103,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2004_openssl - path: $(Build.SourcesDirectory)/artifacts/signed/openssl + artifact: drop_package_linux_distribution_kernel5_4 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel5_4 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.openssldebrepos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl/gen -r ${{ repo }} -n "*.deb" + - ${{ each repo in parameters.kernel5_4debrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel5_4/gen -r ${{ repo }} -n "*.deb" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_openssl_rpms - displayName: Upload openSSL based RPM packages to repos + - job: UploadPackage_kernel5_4_rpms + displayName: Upload kernel5_4 based RPM packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -128,20 +133,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2004_openssl - path: $(Build.SourcesDirectory)/artifacts/signed/openssl + artifact: drop_package_linux_distribution_kernel5_4 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel5_4 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.opensslrpmrepos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl/gen -r ${{ repo }} -n "*.rpm" + - ${{ each repo in parameters.kernel5_4rpmrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel5_4/gen -r ${{ repo }} -n "*.rpm" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_openssl_rpms_cbl - displayName: Upload openSSL based RPM packages to CBL repos + - job: UploadPackage_kernel5_4_rpms_cbl + displayName: Upload kernel5_4 based RPM packages to CBL repos timeoutInMinutes: 120 workspace: clean: all @@ -158,20 +163,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2004_openssl - path: $(Build.SourcesDirectory)/artifacts/signed/openssl + artifact: drop_package_linux_distribution_kernel5_4 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel5_4 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.opensslrpmcblrepos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl/cbl -r ${{ repo }} -n "*.rpm" + - ${{ each repo in parameters.kernel5_4rpmcblrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel5_4/cbl -r ${{ repo }} -n "*.rpm" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_openssl3_debs - displayName: Upload openSSL3 based DEB packages to repos + - job: UploadPackage_kernel6_8_rpms_cbl + displayName: Upload kernel6_8 based RPM packages to CBL repos timeoutInMinutes: 120 workspace: clean: all @@ -188,20 +193,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2204_openssl3 - path: $(Build.SourcesDirectory)/artifacts/signed/openssl3 + artifact: drop_package_linux_distribution_kernel6_8 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel6_8 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.openssl3debrepos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl3/gen -r ${{ repo }} -n "*.deb" + - ${{ each repo in parameters.kernel6_8cblrpmrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel6_8/cbl -r ${{ repo }} -n "*.rpm" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_ubuntu20 - displayName: Upload Ubuntu 20 packages to repos + - job: UploadPackage_kernel5_15_debs + displayName: Upload kernel5_15 based DEB packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -218,20 +223,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2004_openssl - path: $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2004_openssl + artifact: drop_package_linux_distribution_kernel5_15 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel5_15 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.ubuntu20repos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2004_openssl/gen -r ${{ repo }} -n "*.deb" + - ${{ each repo in parameters.kernel5_15debrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel5_15/gen -r ${{ repo }} -n "*.deb" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_ubuntu22 - displayName: Upload Ubuntu 22 packages to repos + - job: UploadPackage_kernel6_8_debs + displayName: Upload kernel6_8 based DEB packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -248,20 +253,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2204_openssl3 - path: $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2204_openssl3 + artifact: drop_package_linux_distribution_kernel6_8 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel6_8 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.ubuntu22repos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2204_openssl3/gen -r ${{ repo }} -n "*.deb" + - ${{ each repo in parameters.kernel6_8debrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel6_8/gen -r ${{ repo }} -n "*.deb" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_ubuntu24 - displayName: Upload Ubuntu 24 packages to repos + - job: UploadPackage_kernel5_15_rpms + displayName: Upload kernel5_15 based RPM packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -278,20 +283,20 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2404_openssl3 - path: $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2404_openssl3 + artifact: drop_package_linux_distribution_kernel5_15 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel5_15 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.ubuntu24repos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/ubuntu_2404_openssl3/gen -r ${{ repo }} -n "*.deb" + - ${{ each repo in parameters.kernel5_15rpmrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel5_15/gen -r ${{ repo }} -n "*.rpm" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true - - job: UploadPackage_openssl3_rpms - displayName: Upload openSSL3 based RPM packages to repos + - job: UploadPackage_kernel6_8_rpms + displayName: Upload kernel6_8 based RPM packages to repos timeoutInMinutes: 120 workspace: clean: all @@ -308,15 +313,15 @@ stages: preferTriggeringPipeline: true runVersion: specific runId: $(resources.pipeline.onebranch.runID) - artifact: drop_package_linux_distribution_ubuntu_2204_openssl3 - path: $(Build.SourcesDirectory)/artifacts/signed/openssl3 + artifact: drop_package_linux_distribution_kernel6_8 + path: $(Build.SourcesDirectory)/artifacts/signed/kernel6_8 - task: DownloadSecureFile@1 name: pmcv4cert displayName: 'Download cert for PMC v4' inputs: secureFile: 'auth.pem' - - ${{ each repo in parameters.openssl3rpmrepos }}: - - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/openssl3/gen -r ${{ repo }} -n "*.rpm" + - ${{ each repo in parameters.kernel6_8rpmrepos }}: + - script: bash scripts/upload-linux-packages.sh -i $(PMCv4ClientId) -c $(pmcv4cert.secureFilePath) -f $(Build.SourcesDirectory)/artifacts/signed/kernel6_8/gen -r ${{ repo }} -n "*.rpm" condition: eq(${{ parameters.debug }}, false) displayName: ${{ repo }} continueOnError: true diff --git a/.azure/OneBranch.PullRequest.yml b/.azure/OneBranch.PullRequest.yml index ac9d0721a..99b011649 100644 --- a/.azure/OneBranch.PullRequest.yml +++ b/.azure/OneBranch.PullRequest.yml @@ -74,10 +74,10 @@ extends: - container: linux_build_container # Default container image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-20.04-cross' type: Linux - - container: ubuntu_2204_cross + - container: kernel5_15_cross image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-22.04-cross' type: Linux - - container: ubuntu_2404_cross + - container: kernel6_8_cross image: 'ghcr.io/microsoft/msquic/linux-build-xcomp:ubuntu-24.04-cross' type: Linux @@ -175,23 +175,23 @@ extends: - template: .azure/obtemplates/build-linux.yml@self parameters: config: Release - os: ubuntu_2204 + kernel: "kernel5_15" tls: openssl3 - template: .azure/obtemplates/build-linux.yml@self parameters: config: Debug - os: ubuntu_2204 + kernel: "kernel5_15" tls: openssl3 - template: .azure/obtemplates/build-linux.yml@self parameters: config: Release - os: ubuntu_2404 + kernel: "kernel6_8" tls: openssl3 xdp: "-UseXdp" - template: .azure/obtemplates/build-linux.yml@self parameters: config: Debug - os: ubuntu_2404 + kernel: "kernel6_8" tls: openssl3 xdp: "-UseXdp" diff --git a/.azure/obtemplates/build-distribution.yml b/.azure/obtemplates/build-distribution.yml index 377a6ce3b..da280a297 100644 --- a/.azure/obtemplates/build-distribution.yml +++ b/.azure/obtemplates/build-distribution.yml @@ -13,39 +13,39 @@ jobs: - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2004 + kernel: kernel5_4 config: Release tls: openssl - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2004 + kernel: kernel5_4 config: Debug tls: openssl - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2204 + kernel: kernel5_15 config: Release tls: openssl3 - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2204 + kernel: kernel5_15 config: Debug tls: openssl3 - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2404 + kernel: kernel6_8 config: Release tls: openssl3 - template: ./download-artifacts.yml parameters: platform: linux - linuxos: ubuntu_2404 + kernel: kernel6_8 config: Debug tls: openssl3 diff --git a/.azure/obtemplates/build-linux-packages.yml b/.azure/obtemplates/build-linux-packages.yml index 7dee1aa9f..d23327603 100644 --- a/.azure/obtemplates/build-linux-packages.yml +++ b/.azure/obtemplates/build-linux-packages.yml @@ -7,19 +7,16 @@ jobs: strategy: matrix: ubuntu_2004: - os: ubuntu_2004 - tls: openssl + kernel: "kernel5_4" ubuntu_2204: - os: ubuntu_2204 - tls: openssl3 + kernel: "kernel5_15" ubuntu_2404: - os: ubuntu_2404 - tls: openssl3 + kernel: "kernel6_8" pool: type: linux variables: ob_outputDirectory: $(Build.SourcesDirectory)/artifacts/dist - ob_artifactSuffix: _$(os)_$(tls) + ob_artifactSuffix: _$(kernel) # drop_package_linux_${{ jobname }}_${{ kernel }} steps: - task: PowerShell@2 displayName: Prepare Build Machine @@ -29,12 +26,12 @@ jobs: arguments: -ForContainerBuild - task: DownloadPipelineArtifact@2 inputs: - artifact: drop_build_linux_build_$(os)_$(tls)_Debug + artifact: drop_build_linux_$(kernel)_Debug path: $(Build.SourcesDirectory)/artifacts/bin/linux pattern: '*.tar' - task: DownloadPipelineArtifact@2 inputs: - artifact: drop_build_linux_build_$(os)_$(tls)_Release + artifact: drop_build_linux_$(kernel)_Release path: $(Build.SourcesDirectory)/artifacts/bin/linux pattern: '*.tar' - script: | # rebuild artifacts with correct permissions and symlink attributes. @@ -46,7 +43,6 @@ jobs: inputs: pwsh: false filePath: scripts/package-distribution.ps1 - arguments: -OS $(os) - script: | # prepare 2 sets of packages for signing with different keys (gen = general purpose, cbl = cbl-mariner) mkdir $(Build.SourcesDirectory)/artifacts/dist/gen find $(Build.SourcesDirectory)/artifacts/dist -type f -exec mv -t $(Build.SourcesDirectory)/artifacts/dist/gen/ {} + diff --git a/.azure/obtemplates/build-linux.yml b/.azure/obtemplates/build-linux.yml index 5fd6f8e05..6eb4e5981 100644 --- a/.azure/obtemplates/build-linux.yml +++ b/.azure/obtemplates/build-linux.yml @@ -2,12 +2,12 @@ parameters: config: '' tls: 'openssl' platform: 'linux' - os: 'ubuntu_2004' + kernel: 'kernel5_4' xdp: '' jobs: -- job: build_${{ parameters.os }}_${{ parameters.tls }}_${{ parameters.config }} - displayName: ${{ parameters.os }} ${{ parameters.tls }} ${{ parameters.config }} +- job: ${{ parameters.kernel }}_${{ parameters.config }} # this job name becomes artifact suffix in the build pipeline + displayName: ${{ parameters.platform }} ${{ parameters.kernel }} ${{ parameters.tls }} ${{ parameters.config }} pool: type: linux variables: @@ -17,48 +17,48 @@ jobs: steps: - task: PowerShell@2 displayName: Prepare Build Machine - ${{ if eq(parameters.os, 'ubuntu_2004') }}: + ${{ if eq(parameters.kernel, 'kernel5_4') }}: target: linux_build_container - ${{ elseif eq(parameters.os, 'ubuntu_2204') }}: - target: ubuntu_2204_cross + ${{ elseif eq(parameters.kernel, 'kernel5_15') }}: + target: kernel5_15_cross ${{ else }}: - target: ubuntu_2404_cross + target: kernel6_8_cross inputs: pwsh: true filePath: scripts/prepare-machine.ps1 arguments: -Tls ${{ parameters.tls }} -ForContainerBuild - task: PowerShell@2 displayName: x64 - ${{ if eq(parameters.os, 'ubuntu_2004') }}: + ${{ if eq(parameters.kernel, 'kernel5_4') }}: target: linux_build_container - ${{ elseif eq(parameters.os, 'ubuntu_2204') }}: - target: ubuntu_2204_cross + ${{ elseif eq(parameters.kernel, 'kernel5_15') }}: + target: kernel5_15_cross ${{ else }}: - target: ubuntu_2404_cross + target: kernel6_8_cross inputs: pwsh: true filePath: scripts/build.ps1 arguments: -Tls ${{ parameters.tls }} -Config ${{ parameters.config }} -Platform ${{ parameters.platform }} ${{ parameters.xdp }} -Arch x64 -CI -UseSystemOpenSSLCrypto -OneBranch -OfficialRelease - task: PowerShell@2 displayName: arm64 - ${{ if eq(parameters.os, 'ubuntu_2004') }}: + ${{ if eq(parameters.kernel, 'kernel5_4') }}: target: linux_build_container - ${{ elseif eq(parameters.os, 'ubuntu_2204') }}: - target: ubuntu_2204_cross + ${{ elseif eq(parameters.kernel, 'kernel5_15') }}: + target: kernel5_15_cross ${{ else }}: - target: ubuntu_2404_cross + target: kernel6_8_cross inputs: pwsh: true filePath: scripts/build.ps1 arguments: -Tls ${{ parameters.tls }} -Config ${{ parameters.config }} -Platform ${{ parameters.platform }} -Arch arm64 -CI -UseSystemOpenSSLCrypto -OneBranch -OfficialRelease - task: PowerShell@2 displayName: arm - ${{ if eq(parameters.os, 'ubuntu_2004') }}: + ${{ if eq(parameters.kernel, 'kernel5_4') }}: target: linux_build_container - ${{ elseif eq(parameters.os, 'ubuntu_2204') }}: - target: ubuntu_2204_cross + ${{ elseif eq(parameters.kernel, 'kernel5_15') }}: + target: kernel5_15_cross ${{ else }}: - target: ubuntu_2404_cross + target: kernel6_8_cross inputs: pwsh: true filePath: scripts/build.ps1 diff --git a/.azure/obtemplates/download-artifacts.yml b/.azure/obtemplates/download-artifacts.yml index d6a92f31d..795837951 100644 --- a/.azure/obtemplates/download-artifacts.yml +++ b/.azure/obtemplates/download-artifacts.yml @@ -2,7 +2,7 @@ parameters: platform: '' - linuxos: 'ubuntu_2004' + kernel: 'kernel5_4' tls: '' config: '' @@ -16,7 +16,7 @@ steps: runVersion: specific runId: $(resources.pipeline.onebranch.runID) ${{ if eq(parameters.platform, 'linux') }}: - artifact: drop_build_${{ parameters.platform }}_build_${{ parameters.linuxos }}_${{ parameters.tls }}_${{ parameters.config }} + artifact: drop_build_${{ parameters.platform }}_${{ parameters.kernel }}_${{ parameters.config }} ${{ else }}: artifact: drop_build_${{ parameters.platform }}_build_${{ parameters.platform }}_${{ parameters.tls }}_${{ parameters.config }} path: $(Build.SourcesDirectory)/artifacts/bin/${{ parameters.platform }} diff --git a/scripts/make-packages.sh b/scripts/make-packages.sh index c6afc2023..8ecbfefc6 100755 --- a/scripts/make-packages.sh +++ b/scripts/make-packages.sh @@ -141,7 +141,6 @@ mkdir -p ${OUTPUT} if [ "$OS" == "linux" ]; then # XDP is only validated on Ubuntu 24.04 and x64 if [ "$XDP" == "False" ] || [[ "$ARCH" == arm* ]]; then - echo "Building rpm package" # RedHat/CentOS FILES="${ARTIFACTS}/libmsquic.${LIBEXT}.${VER_MAJOR}.${VER_MINOR}.${VER_PATCH}=/usr/${LIBDIR}/libmsquic.${LIBEXT}.${VER_MAJOR}.${VER_MINOR}.${VER_PATCH}" FILES="${FILES} ${ARTIFACTS}/libmsquic.${LIBEXT}.${VER_MAJOR}=/usr/${LIBDIR}/libmsquic.${LIBEXT}.${VER_MAJOR}" @@ -151,25 +150,51 @@ if [ "$OS" == "linux" ]; then if [ "$PKGARCH" == 'aarch64' ] || [ "$PKGARCH" == 'x86_64' ]; then BITS='64bit' fi - fpm \ - --force \ - --input-type dir \ - --output-type rpm \ - --architecture ${PKGARCH} \ - --name ${NAME} \ - --provides ${NAME} \ - --depends "libcrypto.so.${TLSVERSION}()(${BITS})" \ - --depends "libnuma.so.1()(${BITS})" \ - --conflicts ${CONFLICTS} \ - --version ${VER_MAJOR}.${VER_MINOR}.${VER_PATCH} \ - --description "${DESCRIPTION}" \ - --vendor "${VENDOR}" \ - --maintainer "${MAINTAINER}" \ - --package "${OUTPUT}" \ - --license MIT \ - --url https://github.com/microsoft/msquic \ - --log error \ - ${FILES} + if [ "$XDP" == "True" ] && [[ "$ARCH" == x* ]]; then + echo "Building rpm package (XDP)" + fpm \ + --force \ + --input-type dir \ + --output-type rpm \ + --architecture ${PKGARCH} \ + --name ${NAME} \ + --provides ${NAME} \ + --depends "libcrypto.so.${TLSVERSION}()(${BITS})" \ + --depends "libnuma.so.1()(${BITS})" \ + --depends "libxdp.so.1.4.0" \ + --depends "libnl-route-3.so.200" \ + --conflicts ${CONFLICTS} \ + --version ${VER_MAJOR}.${VER_MINOR}.${VER_PATCH} \ + --description "${DESCRIPTION}" \ + --vendor "${VENDOR}" \ + --maintainer "${MAINTAINER}" \ + --package "${OUTPUT}" \ + --license MIT \ + --url https://github.com/microsoft/msquic \ + --log error \ + ${FILES} ${ARTIFACTS}/datapath_raw_xdp_kern.o=/usr/${LIBDIR}/datapath_raw_xdp_kern.o + else + echo "Building rpm package" + fpm \ + --force \ + --input-type dir \ + --output-type rpm \ + --architecture ${PKGARCH} \ + --name ${NAME} \ + --provides ${NAME} \ + --depends "libcrypto.so.${TLSVERSION}()(${BITS})" \ + --depends "libnuma.so.1()(${BITS})" \ + --conflicts ${CONFLICTS} \ + --version ${VER_MAJOR}.${VER_MINOR}.${VER_PATCH} \ + --description "${DESCRIPTION}" \ + --vendor "${VENDOR}" \ + --maintainer "${MAINTAINER}" \ + --package "${OUTPUT}" \ + --license MIT \ + --url https://github.com/microsoft/msquic \ + --log error \ + ${FILES} + fi fi # Debian/Ubuntu diff --git a/scripts/package-distribution.ps1 b/scripts/package-distribution.ps1 index 7a195d91a..e3dc4c5d2 100644 --- a/scripts/package-distribution.ps1 +++ b/scripts/package-distribution.ps1 @@ -6,32 +6,6 @@ #> -param ( - [Parameter(Mandatory = $false)] - [ValidateSet("ubuntu_2404", "ubuntu_2204", "ubuntu_2004", "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "")] - [string]$OS = "" -) - -# Convert GH Actions OS names to our internal names -if ($OS -eq "ubuntu-20.04") { - $OS = "ubuntu_2004" -} - -if ($OS -eq "ubuntu-22.04") { - $OS = "ubuntu_2204" -} - -if ($OS -eq "ubuntu-24.04") { - $OS = "ubuntu_2404" -} - -$UseXdp = $false -$Time64Distro = $false -if ($OS -eq "ubuntu_2404") { - $UseXdp = $true - $Time64Distro = $true -} - Set-StrictMode -Version 'Latest' $PSDefaultParameterValues['*:ErrorAction'] = 'Stop' @@ -141,6 +115,17 @@ foreach ($Build in $AllBuilds) { $Libraries += Join-Path $ArtifactsDir "msquic.lib" } + # if datapath_raw_xdp_kern.o exists under $ArtifactsDir, $UseXdp to be true + $Time64Distro = $false + $UseXdp = $false + if ($Platform -eq "linux") { + $XdpBin = Join-Path $ArtifactsDir "datapath_raw_xdp_kern.o" + if (Test-Path $XdpBin) { + $UseXdp = $true + $Time64Distro = $true + } + } + # Copy items into temp folder that can be zipped in 1 command $IncludeDir = Join-Path $TempDir "include"