Add debug symbols to packages (#7211)

This change updates our build pipelines to build edgelet with full debug symbols, and then have the various packaging systems generate a separate symbols package for each distro. For the deb-based distros, the new package is something like `aziot-edge-dbgsym_{ver}_{arch}.deb` (`.ddeb` on Ubuntu). For the rpm-based distros, the new package is something like `aziot-edge-debuginfo-{ver}.{distro}.{arch}.rpm`. This change also updates the test and release pipelines to handle the new packages. For the test pipelines, we don't want to copy/install the symbol packages. For the release pipeline, we _do_ want to sign and stage the symbol packages, but we only publish them to the GitHub release page for now, not to packages.microsoft.com. To test, I ran the various pipelines to make sure the work as expected. I also manually installed IoT Edge, installed the symbol packages, and attached gdb to the running edgelet process to verify that it loads the stack frames. ## Azure IoT Edge PR checklist:
2024-02-12 08:57:50 -08:00 · 2024-02-12 08:57:50 -08:00 · 36fc55afc7
--- a/builds/e2e/e2e.yaml
+++ b/builds/e2e/e2e.yaml
@ -209,7 +209,7 @@ jobs:
      arch: amd64
      artifactName: iotedged-centos7-amd64
      identityServiceArtifactName: packages_centos-7_amd64
-      identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+      identityServicePackageFilter: aziot-identity-service-?.*.x86_64.rpm

    steps:
    - template: templates/e2e-clean-directory.yaml
@ -232,7 +232,7 @@ jobs:
      arch: amd64
      artifactName: iotedged-redhat8-amd64
      identityServiceArtifactName: packages_redhat-ubi8-latest_amd64
-      identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+      identityServicePackageFilter: aziot-identity-service-?.*.x86_64.rpm

    steps:
    - template: templates/e2e-setup.yaml
@ -253,7 +253,7 @@ jobs:
      arch: amd64
      artifactName: iotedged-redhat9-amd64
      identityServiceArtifactName: packages_redhat-ubi9-latest_amd64
-      identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+      identityServicePackageFilter: aziot-identity-service-?.*.x86_64.rpm

    steps:
    - template: templates/e2e-setup.yaml
@ -307,7 +307,7 @@ jobs:
      arch: amd64
      artifactName: iotedged-mariner2-amd64
      identityServiceArtifactName: packages_mariner-2_amd64
-      identityServicePackageFilter: aziot-identity-service-*.cm2.x86_64.rpm
+      identityServicePackageFilter: aziot-identity-service-?.*.cm2.x86_64.rpm

    timeoutInMinutes: 90

--- a/builds/e2e/longhaul.yaml
+++ b/builds/e2e/longhaul.yaml
@ -364,7 +364,7 @@ jobs:
          edgelet.artifact.name: 'iotedged-mariner2-amd64'
          images.artifact.name: '$(images.artifact.name.linux)'
          aziotis.artifact.name: 'packages_mariner-2_amd64'
-          aziotis.package.filter: 'aziot-identity-service-*.x86_64.rpm'
+          aziotis.package.filter: 'aziot-identity-service-?.*.x86_64.rpm'
          identityServiceArtifactName: packages_mariner-2_amd64
          quickstart.artifactName: 'IotEdgeQuickstart.linux-x64.tar.gz'
      - task: DeleteFiles@1
--- a/builds/mariner/SPECS/aziot-edge/aziot-edge.spec
+++ b/builds/mariner/SPECS/aziot-edge/aziot-edge.spec
@ -1,4 +1,3 @@
-%global debug_package %{nil}
 %define iotedge_user iotedge
 %define iotedge_group %{iotedge_user}
 %define iotedge_home %{_localstatedir}/lib/aziot/edged
@ -38,6 +37,8 @@ securely and at scale—whether in the cloud or offline.

 This package contains the IoT Edge daemon and CLI tool.

+%debug_package
+
 %prep
 %setup -q
 # include rust toolchain that matches the one from aziotedge's pipeline
--- a/builds/misc/packages-release.yaml
+++ b/builds/misc/packages-release.yaml
@ -112,14 +112,19 @@ stages:
            SourceFolder: $(target.iotedged)
            Contents: |
              *.deb
+              *.ddeb
              *.rpm
+              !*-debugsource-*.rpm
            TargetFolder: '$(build.artifactstagingdirectory)'
        - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@3
          displayName: ESRP Binary CodeSigning
          inputs:
            ConnectedServiceName: "aziotedge-pmc-v4-prod"
            FolderPath: '$(build.artifactstagingdirectory)'
-            Pattern: "*.deb, *.rpm"
+            Pattern: |
+              *.deb
+              *.ddeb
+              *.rpm
            SessionTimeout: 20
            inlineOperation: |
              [
@ -137,7 +142,7 @@ stages:
            rm -f ./CodeSignSummary*
          displayName: Remove CodeSign Summary
        - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
-          displayName: 'SBOM Generation Task'
+          displayName: 'Generate SBOM'
          inputs:
              BuildDropPath: '$(build.artifactstagingdirectory)'
        - task: PublishBuildArtifacts@1
@ -193,14 +198,14 @@ stages:
            os: redhat8
            artifactName: iotedged-redhat8-amd64
            identityServiceArtifactName: packages_redhat-ubi8-latest_amd64
-            identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+            identityServicePackageFilter: aziot-identity-service-?.*.x86_64.rpm
            pmcRepoName: microsoft-rhel8.0-prod-yum
            pmcRelease: ''
          RedHat9-amd64:
            os: redhat9
            artifactName: iotedged-redhat9-amd64
            identityServiceArtifactName: packages_redhat-ubi9-latest_amd64
-            identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+            identityServicePackageFilter: aziot-identity-service-?.*.x86_64.rpm
            pmcRepoName: microsoft-rhel9.0-prod-yum
            pmcRelease: ''

@ -372,79 +377,79 @@ stages:
            identityServiceArtifactName: packages_centos-7_amd64
            identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
          RedHat8-amd64:
-              os: redhat8
-              artifactName: iotedged-redhat8-amd64
-              identityServiceArtifactName: packages_redhat-ubi8-latest_amd64
-              identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+            os: redhat8
+            artifactName: iotedged-redhat8-amd64
+            identityServiceArtifactName: packages_redhat-ubi8-latest_amd64
+            identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
          RedHat9-amd64:
-              os: redhat9
-              artifactName: iotedged-redhat9-amd64
-              identityServiceArtifactName: packages_redhat-ubi9-latest_amd64
-              identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm
+            os: redhat9
+            artifactName: iotedged-redhat9-amd64
+            identityServiceArtifactName: packages_redhat-ubi9-latest_amd64
+            identityServicePackageFilter: aziot-identity-service-*.x86_64.rpm

          Debian10-amd64:
            os: debian10
            artifactName: iotedged-debian10-amd64
            identityServiceArtifactName: packages_debian-10-slim_amd64
-            identityServicePackageFilter: aziot-identity-service_*_amd64.deb
+            identityServicePackageFilter: aziot-identity-service*_amd64.deb
          Debian10-arm32v7:
            os: debian10
            artifactName: iotedged-debian10-arm32v7
            identityServiceArtifactName: packages_debian-10-slim_arm32v7
-            identityServicePackageFilter: aziot-identity-service_*_armhf.deb
+            identityServicePackageFilter: aziot-identity-service*_armhf.deb
          Debian10-aarch64:
            os: debian10
            artifactName: iotedged-debian10-aarch64
            identityServiceArtifactName: packages_debian-10-slim_aarch64
-            identityServicePackageFilter: aziot-identity-service_*_arm64.deb
+            identityServicePackageFilter: aziot-identity-service*_arm64.deb

          Debian11-amd64:
            os: debian11
            artifactName: iotedged-debian11-amd64
            identityServiceArtifactName: packages_debian-11-slim_amd64
-            identityServicePackageFilter: aziot-identity-service_*_amd64.deb
+            identityServicePackageFilter: aziot-identity-service*_amd64.deb
          Debian11-arm32v7:
            os: debian11
            artifactName: iotedged-debian11-arm32v7
            identityServiceArtifactName: packages_debian-11-slim_arm32v7
-            identityServicePackageFilter: aziot-identity-service_*_armhf.deb
+            identityServicePackageFilter: aziot-identity-service*_armhf.deb
          Debian11-aarch64:
            os: debian11
            artifactName: iotedged-debian11-aarch64
            identityServiceArtifactName: packages_debian-11-slim_aarch64
-            identityServicePackageFilter: aziot-identity-service_*_arm64.deb
+            identityServicePackageFilter: aziot-identity-service*_arm64.deb

          Ubuntu2004-amd64:
            os: ubuntu20.04
            artifactName: iotedged-ubuntu20.04-amd64
            identityServiceArtifactName: packages_ubuntu-20.04_amd64
-            identityServicePackageFilter: aziot-identity-service_*_amd64.deb
+            identityServicePackageFilter: aziot-identity-service*_amd64.*deb
          Ubuntu2004-arm32v7:
            os: ubuntu20.04
            artifactName: iotedged-ubuntu20.04-arm32v7
            identityServiceArtifactName: packages_ubuntu-20.04_arm32v7
-            identityServicePackageFilter: aziot-identity-service_*_armhf.deb
+            identityServicePackageFilter: aziot-identity-service*_armhf.*deb
          Ubuntu2004-aarch64:
            os: ubuntu20.04
            artifactName: iotedged-ubuntu20.04-aarch64
            identityServiceArtifactName: packages_ubuntu-20.04_aarch64
-            identityServicePackageFilter: aziot-identity-service_*_arm64.deb
+            identityServicePackageFilter: aziot-identity-service*_arm64.*deb

          Ubuntu2204-amd64:
            os: ubuntu22.04
            artifactName: iotedged-ubuntu22.04-amd64
            identityServiceArtifactName: packages_ubuntu-22.04_amd64
-            identityServicePackageFilter: aziot-identity-service_*_amd64.deb
+            identityServicePackageFilter: aziot-identity-service*_amd64.*deb
          Ubuntu2204-arm32v7:
            os: ubuntu22.04
            artifactName: iotedged-ubuntu22.04-arm32v7
            identityServiceArtifactName: packages_ubuntu-22.04_arm32v7
-            identityServicePackageFilter: aziot-identity-service_*_armhf.deb
+            identityServicePackageFilter: aziot-identity-service*_armhf.*deb
          Ubuntu2204-aarch64:
            os: ubuntu22.04
            artifactName: iotedged-ubuntu22.04-aarch64
            identityServiceArtifactName: packages_ubuntu-22.04_aarch64
-            identityServicePackageFilter: aziot-identity-service_*_arm64.deb
+            identityServicePackageFilter: aziot-identity-service*_arm64.*deb

      steps:
        - task: AzureKeyVault@1
@ -462,6 +467,7 @@ stages:
            path: $(System.ArtifactsDirectory)
            patterns: |
                $(artifactName)/*.deb
+                $(artifactName)/*.ddeb
                $(artifactName)/*.rpm
        - task: PowerShell@2
          displayName: 'Download aziot-identity-service'
--- a/builds/misc/templates/build-packages.yaml
+++ b/builds/misc/templates/build-packages.yaml
@ -135,7 +135,9 @@ stages:
              SourceFolder: $(target.iotedged)
              Contents: |
                *.deb
+                *.ddeb
                *.rpm
+                !*-debugsource-*.rpm
              TargetFolder: '$(build.artifactstagingdirectory)'
          - task: PublishBuildArtifacts@1
            displayName: Publish Artifacts
@ -294,7 +296,7 @@ stages:
          # build iot-identity-service here since they cannot build native aarch64 packages
          - bash: |
              set -ex
-              git clone --recurse-submodules https://github.com/Azure/iot-identity-service.git
+              git clone --recurse-submodules --branch main https://github.com/Azure/iot-identity-service.git
              pushd iot-identity-service
              packageVersion=$(git tag | grep 1.3.[0-9]*$ | sort --version-sort -r | head -1)
              sudo docker run --rm \
@ -308,6 +310,7 @@ stages:
                '/src/ci/package.sh'
              popd
              sudo cp iot-identity-service/packages/mariner$(os_version)/$(arch)/aziot-identity-service-$packageVersion-1.cm$(os_version).$(arch).rpm .
+              sudo cp iot-identity-service/packages/mariner$(os_version)/$(arch)/aziot-identity-service-debuginfo-$packageVersion-1.cm$(os_version).$(arch).rpm .
              sudo rm -rf iot-identity-service
          
          - bash: |
--- a/edgelet/Cargo.toml
+++ b/edgelet/Cargo.toml
@ -20,3 +20,6 @@ panic = 'abort'

 [profile.release]
 panic = 'abort'
+# Release builds will have full symbols. The packaging phase will strip symbols from binaries and
+# make them available in a separate package.
+debug = 2
--- a/edgelet/build/linux/package-mariner.sh
+++ b/edgelet/build/linux/package-mariner.sh
@ -73,7 +73,9 @@ esac
 popd

 # get aziot-identity-service version
-IIS_VERSION=$(rpm -qp --queryformat '%{Version}' /src/aziot-identity-service-*.$PackageExtension.${MARINER_ARCH}.rpm | head -1)
+IIS_VERSION=$(
+    rpm -qp --queryformat '%{Version}' $(ls /src/aziot-identity-service-*.$PackageExtension.${MARINER_ARCH}.rpm | head -1)
+)

 # Update versions in specfiles
 pushd "${BUILD_REPOSITORY_LOCALPATH}"
--- a/edgelet/build/linux/package.sh
+++ b/edgelet/build/linux/package.sh
@ -385,4 +385,5 @@ docker run --rm \
    "

 find "$PROJECT_ROOT" -name '*.deb'
+find "$PROJECT_ROOT" -name '*.ddeb'
 find "$PROJECT_ROOT" -name '*.rpm'
--- a/edgelet/contrib/centos/aziot-edge.spec
+++ b/edgelet/contrib/centos/aziot-edge.spec
@ -9,7 +9,6 @@
 %define iotedge_agent_uid 13622
 %define iotedge_hub_user edgehubuser
 %define iotedge_hub_uid 13623
-%global debug_package %{nil}

 Name:           aziot-edge
 Version:        @version@
--- a/edgelet/contrib/debian/rules
+++ b/edgelet/contrib/debian/rules
@ -8,10 +8,7 @@
 %:
 	dh $@ --with systemd

-.PHONY: override_dh_strip
-override_dh_strip:
-	dh_strip --no-automatic-dbgsym
-
+.PHONY: override_dh_installinit
 override_dh_installinit:
 	dh_systemd_enable
 	dh_installinit
--- a/edgelet/contrib/enterprise-linux/aziot-edge.spec
+++ b/edgelet/contrib/enterprise-linux/aziot-edge.spec
@ -9,7 +9,6 @@
 %define iotedge_agent_uid 13622
 %define iotedge_hub_user edgehubuser
 %define iotedge_hub_uid 13623
-%global debug_package %{nil}

 Name:           aziot-edge
 Version:        @version@
--- a/scripts/local/test/DownloadIdentityService.ps1
+++ b/scripts/local/test/DownloadIdentityService.ps1
@ -85,7 +85,7 @@ for($page = 1; ; $page++)
    Write-Output "Extract aziot-identity-service.zip"
    Expand-Archive -Path aziot-identity-service.zip -DestinationPath aziot-identity-service -Force

-    $packages = Get-ChildItem -Recurse aziot-identity-service -Filter $env:PACKAGE_FILTER -Exclude "*debug*","*devel*"
+    $packages = Get-ChildItem -Recurse aziot-identity-service -Filter $env:PACKAGE_FILTER -Exclude "*devel*"
    $packagePath = Convert-Path $env:DOWNLOAD_PATH

    Write-Output "Copy $packages to $packagePath"
--- a/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/PackageManagement.cs
+++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/linux/PackageManagement.cs
@ -36,7 +36,10 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common.Linux
        {
            string[] packages = Directory
                .GetFiles(path, $"*.{this.packageExtension.ToString().ToLower()}")
-                .Where(p => !p.Contains("debug") && !p.Contains("devel"))
+                .Where(p => !p.Contains("debuginfo")
+                    && !p.Contains("dbgsym")
+                    && !p.Contains("devel")
+                    && !p.Contains("src.rpm"))
                .ToArray();

            return this.packageExtension switch
@ -47,7 +50,8 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common.Linux
                    $"apt-get install -y --option DPkg::Lock::Timeout=600 {string.Join(' ', packages)}",
                    $"apt-get install -f --option DPkg::Lock::Timeout=600"
                },
-                SupportedPackageExtension.Rpm => this.os switch {
+                SupportedPackageExtension.Rpm => this.os switch
+                {
                    "centos" => new[]
                    {
                        "set -e",
@ -102,7 +106,8 @@ namespace Microsoft.Azure.Devices.Edge.Test.Common.Linux
                    $"apt-get update",
                    $"apt-get install --option DPkg::Lock::Timeout=600 --yes aziot-edge"
                },
-                SupportedPackageExtension.Rpm => this.os switch {
+                SupportedPackageExtension.Rpm => this.os switch
+                {
                    "centos" => new[]
                    {
                        $"rpm -iv --replacepkgs https://packages.microsoft.com/config/{this.os}/{this.version}/packages-microsoft-prod.rpm",