Add a PR check for source signatures (#8542)

.github/workflows/check-entangled-specs.yml

@@ -23,10 +23,10 @@ jobs:
         uses: actions/checkout@v4
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.7
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt

.github/workflows/check-kernel-config.yml

@@ -42,10 +42,10 @@ jobs:
           echo "base_sha=${{ github.event.before }}" >> $GITHUB_ENV
           echo "Merging ${{ github.sha }} into ${{ github.event.before }}"
 
-      - name: Setup Python 3.11
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt

.github/workflows/check-license-map.yml

@@ -21,10 +21,10 @@ jobs:
       - name: Workflow trigger checkout
         uses: actions/checkout@v4
 
-      - name: Setup Python 3.9
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install python-rpm-spec

.github/workflows/check-signatures.yml

@@ -20,10 +20,10 @@ jobs:
         uses: actions/checkout@v4
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.9
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt
@@ -47,5 +47,5 @@ jobs:
           echo "Files changed: '$(git diff-tree --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})'"
           changed_folders=$(dirname $(git diff-tree --diff-filter=d --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }}) | sort --unique)
           echo "Folders to validate: '${changed_folders}'"
-          
+
           python3 toolkit/scripts/check_signatures.py ${changed_folders}

.github/workflows/check-source-signatures.yml

@@ -0,0 +1,117 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: Source Signature Check
+
+on:
+  push:
+    branches: [3.0*]
+  pull_request:
+    branches: [3.0*]
+
+jobs:
+  spec-check:
+    name: Source Signature Check
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout the branch of our repo that triggered this action
+      - name: Workflow trigger checkout
+        uses: actions/checkout@v4
+
+      # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
+      - name: Setup Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Get Python dependencies
+        run: python3 -m pip install -r toolkit/scripts/requirements.txt
+
+      - name: Get base commit for PRs
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          git fetch origin ${{ github.base_ref }}
+          echo "base_sha=$(git rev-parse origin/${{ github.base_ref }})" >> $GITHUB_ENV
+          echo "Merging ${{ github.sha }} into ${{ github.base_ref }}"
+
+      - name: Get base commit for Pushes
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          git fetch origin ${{ github.event.before }}
+          echo "base_sha=${{ github.event.before }}" >> $GITHUB_ENV
+          echo "Merging ${{ github.sha }} into ${{ github.event.before }}"
+
+      - name: Get changed packages
+        run: |
+          # Find the packages that have been modified in the current PR. They will be of the form '/path/to/SPECS/<pkgname>/**/.*', and we want to extract
+          # the package name (ie the folder inside ./SPECS).
+          changed_pkgs=$(git diff-tree --diff-filter=d --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }} | { grep "SPECS/.*" || test $? = 1; } | sed -n 's#SPECS/\([^/]*\)/.*#\1#p' | sort -u | xargs)
+          changed_pkgs_extended=$(git diff-tree --diff-filter=d --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }} | { grep "SPECS-EXTENDED/.*" || test $? = 1; } | sed -n 's#SPECS-EXTENDED/\([^/]*\)/.*#\1#p' | sort -u | xargs)
+          echo "Packages modified in this PR:"
+          echo "SPECS:          ${changed_pkgs}"
+          echo "SPECS-EXTENDED: ${changed_pkgs_extended}"
+          echo "changed_pkgs=${changed_pkgs}" >> $GITHUB_ENV
+          echo "changed_pkgs_extended=${changed_pkgs_extended}" >> $GITHUB_ENV
+
+      - name: Prepare the build environment
+        run: |
+          if [ -z "${{ env.changed_pkgs }}" ] && [ -z "${{ env.changed_pkgs_extended }}" ]; then
+            echo "No package changes detected."
+            exit 0
+          fi
+
+          echo "Checking for invalid signatures..."
+          # Call this script to sync the toolchain manifests with the LKG daily build.
+          ./toolkit/scripts/setuplkgtoolchain.sh
+          # Determine the LKG daily build ID.
+          LKG_BUILD_ID=$(wget -qO - https://mariner3dailydevrepo.blob.core.windows.net/lkg/lkg-3.0-dev.json | jq -r ".dailybuildid" | tr '\.' '-')
+          echo "LKG_BUILD_ID=${LKG_BUILD_ID}" >> $GITHUB_ENV
+
+          sudo make -C toolkit -j$(nproc) chroot-tools REBUILD_TOOLS=y DAILY_BUILD_ID=${LKG_BUILD_ID}
+
+      - name: Check for invalid source signatures
+        run: |
+          if [ -z "${{ env.changed_pkgs }}" ] && [ -z "${{ env.changed_pkgs_extended }}" ]; then
+            echo "No package changes detected."
+            exit 0
+          fi
+
+          # Core SPECs
+          if [ -n "${{ env.changed_pkgs }}" ]; then
+            # We want to ignore errors here, as we want to check all the packages that have been modified. Capture the error code and check it later.
+            set +e
+            set -x
+            sudo make -C toolkit -j$(nproc) input-srpms REBUILD_TOOLS=y DAILY_BUILD_ID=${{ env.LKG_BUILD_ID }} SRPM_PACK_LIST="${{ env.changed_pkgs }}"
+            core_err=$?
+            set +x
+            set -e
+          fi
+
+          # Extended SPECs
+          if [ -n "${{ env.changed_pkgs_extended }}" ]; then
+            # We want to ignore errors here, as we want to check all the packages that have been modified. Capture the error code and check it later.
+            set +e
+            set -x
+            sudo make -C toolkit -j$(nproc) input-srpms REBUILD_TOOLS=y DAILY_BUILD_ID=${{ env.LKG_BUILD_ID }} SRPM_PACK_LIST="${{ env.changed_pkgs_extended }}" SPECS_DIR=../SPECS-EXTENDED
+            extended_err=$?
+            set +x
+            set -e
+          fi
+
+          # Print results
+          if [ $core_err -ne 0 ] || [ $extended_err -ne 0 ]; then
+            printf "\n\n******************************"
+            echo "Failed to check the signatures of the modified packages."
+            echo "Check the logs above for details on the mismatches files and their expected hashes."
+            if [ $core_err -ne 0 ]; then
+              echo "Consider running: sudo make -C toolkit input-srpms REBUILD_TOOLS=y SRPM_PACK_LIST='${{ env.changed_pkgs }}'"
+            fi
+            if [ $extended_err -ne 0 ]; then
+              echo "Consider running: sudo make -C toolkit input-srpms REBUILD_TOOLS=y SRPM_PACK_LIST='${{ env.changed_pkgs_extended }}' SPECS_DIR=../SPECS-EXTENDED"
+            fi
+            printf "\n\n******************************"
+            exit 1
+          else
+            echo "All modified packages have valid source signatures."
+          fi

.github/workflows/check-spec.yml

@@ -20,10 +20,10 @@ jobs:
         uses: actions/checkout@v4
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.9
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt

.github/workflows/check-static-glibc.yml

@@ -21,10 +21,10 @@ jobs:
         uses: actions/checkout@v4
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.9
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt

.github/workflows/lint-specs.yml

@@ -60,10 +60,10 @@ jobs:
           path: 'spec-cleaner'
 
       # For consistency, we use the same major/minor version of Python that Azure Linux ships
-      - name: Setup Python 3.7
+      - name: Setup Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
 
       # We take our version of the linting tool from the master branch to ensure rules
       # are consistent across all branches