Codesigning support for scripts in Tools (#556)

* Codesigning tools scripts during release.
2023-06-16 15:13:32 +02:00 · 2023-06-16 15:13:32 +02:00 · 74bf8c5333
--- a/.github/workflows/create-prerelase.yml
+++ b/.github/workflows/create-prerelase.yml
@ -8,6 +8,7 @@ on:
  push:
    paths: 
      - 'Scripts/**'
+      - 'Tools/**'
    branches: [ dev ]

 jobs:
@ -16,22 +17,28 @@ jobs:
    if: "!contains(github.event.head_commit.message, '[no release]')"
    runs-on: windows-2019
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
-          fetch-depth: 0
+          fetch-depth: 1
      - name: Build scripts 
        id: build
        shell: powershell
        run: |
-          ./build.ps1 -Version prerelease
+          ./build.ps1 -Version dev
          $filename = "mslab_dev-$((Get-Date -Format "yyyyMMdd")).zip"
          mv ./Release.zip $filename
-          echo "::set-output name=filename::$filename"
-      - uses: "marvinpinto/action-automatic-releases@latest"
+          echo "filename=$filename" >> $env:GITHUB_OUTPUT
+      - name: Delete current dev prerelease
+        uses: cb80/delrel@latest
        with:
-          repo_token: "${{ secrets.GITHUB_TOKEN }}"
-          automatic_release_tag: "dev"
+          tag: dev
+      - name: Create new dev prerelease
+        uses: softprops/action-gh-release@v1
+        with:
+          tag_name: dev
+          name: dev branch preview
+          generate_release_notes: true
          prerelease: true
-          title: "dev branch preview"
          files: |
            ${{ steps.build.outputs.filename }}
+            Output/Tools/*.ps1
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@ -8,10 +8,12 @@ on:
  push:
    paths: 
      - 'Scripts/**'
+      - 'Tools/**'
    branches: [ master ]

 jobs:
  new-version:
+    environment: release
    name: Bump version
    if: "!contains(github.event.head_commit.message, '[no release]')"
    runs-on: windows-2019
@ -19,14 +21,15 @@ jobs:
      previous_tag: ${{ steps.bump.outputs.previous_tag }}
      new_tag: ${{ steps.bump.outputs.new_tag }}
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
      - id: bump
        name: Bump version
        run: |
          $today = Get-Date
          $newVersion = @($today.ToString("yy"), $today.ToString("MM"), "1")
          git fetch --tags
-          $hash = git rev-list --tags --topo-order --max-count=1
+          # Get the latest tag that matches our versioning schema (starts with letter v)
+          $hash = git rev-list --tags=v* --topo-order --max-count=1
          if($hash) {
            $currentTag = git describe --tags $hash
            $parts = $currentTag.Substring(1) -split '\.'
@ -35,72 +38,67 @@ jobs:
          
          $newTag = "v" + ($newVersion -join ".")
          git tag $newTag
+          if(-not $?) {
+            throw "Tagging of new release version failed!"
+          }
+          
+          git push origin $newTag
+
          "New version: $newTag"
-          echo "::set-output name=previous_tag::$currentTag"
-          echo "::set-output name=new_tag::$newTag"
-      
-      - name: Push version tag
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          tags: true
+          echo "previous_tag=$currentTag" >> $env:GITHUB_OUTPUT
+          echo "new_tag=$newTag" >> $env:GITHUB_OUTPUT

  new-release:
    name: Create release
    if: "!contains(github.event.head_commit.message, '[no release]')"
-    runs-on: windows-2019
+    runs-on: self-hosted
    needs: new-version
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
-      - uses: azure/login@v1
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-          enable-AzPSSession: true
-      - name: "Build scripts"
-        uses: azure/powershell@v1
+      - name: Build and sign release scripts
+        shell: pwsh
        env:
          SIGN_SCRIPT_URI: ${{ secrets.SIGN_SCRIPT_URI }}
-          CLIENT_ID: ${{ secrets.CLIENT_ID }} # just to ofusctate it in the output
-        with:
-          azPSVersion: "latest"
-          inlineScript: |
-            ./build.ps1 -Version ${{ needs.new-version.outputs.new_tag }} -SignScripts $true -SignScriptUri $env:SIGN_SCRIPT_URI -ClientId $env:CLIENT_ID
+          CLIENT_ID: ${{ secrets.CLIENT_ID }} # just to obfusctate it in the output
+        run: |
+          ./build.ps1 -Version ${{ needs.new-version.outputs.new_tag }} -SignScripts $true -SignScriptUri $env:SIGN_SCRIPT_URI -ClientId $env:CLIENT_ID
+          Move-Item ./Release.zip mslab_${{ needs.new-version.outputs.new_tag }}.zip
      - name: Create changelog
        id: changelog
        shell: powershell
        run: |
          if("${{ needs.new-version.outputs.previous_tag }}" -ne "") {
-            $changelog = (& { git log ${{ needs.new-version.outputs.previous_tag }}..HEAD --pretty=format:'- %s (%h)' --abbrev-commit -- Scripts }) -join '%0D%0A'
+            $changelog = (& { git log ${{ needs.new-version.outputs.previous_tag }}..HEAD --pretty=format:'- %s (%h)' --abbrev-commit -- Scripts Tools }) -join "`n"
            "Changes for ${{ needs.new-version.outputs.previous_tag }} are:"
            $changelog
          } else {
            $changelog = ""
          }
-          echo "::set-output name=changelog::$changelog"
-        
-      - name: Create Github Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          
+          $changeLogContent = @"
+          :package: All MSLab scripts are in **mslab_${{ needs.new-version.outputs.new_tag }}.zip** file. 
+            
+          :information_source: Remaining `.ps1` files in this release would be downloaded on-demand by MSLab scripts during deployment, only if needed.
+          "@
+          
+          if($changelog -ne "") {
+            $changeLogContent += @"
+          
+          :basket: Changes in this version:
+          $changelog
+          "@
+          }
+          
+          Set-Content -Value $changeLogContent -Path .\changelog.md
+      - name: Create new release
+        uses: softprops/action-gh-release@v1
        with:
          tag_name: ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
-          release_name: Release ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
-          body: |
-            Changes in this version:
-            ${{ steps.changelog.outputs.changelog }}
-          draft: false
-          prerelease: false
-
-      - name: Upload ZIP to Release
-        id: upload-scripts
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./Release.zip
-          asset_name: mslab_${{ needs.new-version.outputs.new_tag }}.zip
-          asset_content_type: application/zip
+          name: Release ${{ needs.new-version.outputs.new_tag }} # ${{ github.ref }}
+          generate_release_notes: true
+          body_path: changelog.md
+          files: |
+            mslab_${{ needs.new-version.outputs.new_tag }}.zip
+            Output/Tools/*.ps1
--- a/Scripts/1_Prereq.ps1
+++ b/Scripts/1_Prereq.ps1
@ -83,37 +83,58 @@ function  Get-WindowsBuildNumber {
 #region Download Scripts

 #add scripts for VMM
-    $Filenames="1_SQL_Install","2_ADK_Install","3_SCVMM_Install"
-    foreach ($Filename in $filenames){
-        $Path="$PSScriptRoot\Temp\ToolsVHD\SCVMM\$Filename.ps1"
-        If (Test-Path -Path $Path){
+    $filenames = "1_SQL_Install", "2_ADK_Install", "3_SCVMM_Install"
+    foreach ($filename in $filenames) {
+        $Path = "$PSScriptRoot\Temp\ToolsVHD\SCVMM\$filename.ps1"
+        if (Test-Path -Path $Path) {
            WriteSuccess "`t $Filename is present, skipping download"
-        }else{
-            $FileContent=$null
-            $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/Microsoft/MSLab/master/Tools/$Filename.ps1").Content
-            if ($FileContent){
+        } else {
+            $FileContent = $null
+
+            try {
+                # try to download tagged version first
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/$mslabVersion/Tools/$filename.ps1").Content
+            } catch {
+                WriteInfo "Download $filename failed with $($_.Exception.Message), trying master branch now"
+                # if that fails, try master branch
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/$filename.ps1").Content
+            }
+
+            if ($FileContent) {
                $script = New-Item $Path -type File -Force
                $FileContent=$FileContent -replace "PasswordGoesHere",$LabConfig.AdminPassword #only applies to 1_SQL_Install and 3_SCVMM_Install.ps1
                $FileContent=$FileContent -replace "DomainNameGoesHere",$LabConfig.DomainNetbiosName #only applies to 1_SQL_Install and 3_SCVMM_Install.ps1
                Set-Content -path $script -value $FileContent
-            }else{
+            } else {
                WriteErrorAndExit "Unable to download $Filename."
            }
        }
    }

 # add createparentdisks, DownloadLatestCU and PatchParentDisks scripts to Parent Disks folder
-    $FileNames = "CreateParentDisk", "DownloadLatestCUs", "PatchParentDisks", "CreateVMFleetDisk"
+    $fileNames = "CreateParentDisk", "DownloadLatestCUs", "PatchParentDisks", "CreateVMFleetDisk"
    if($LabConfig.Linux) {
-        $FileNames += "CreateLinuxParentDisk"
+        $fileNames += "CreateLinuxParentDisk"
    }
-    foreach ($filename in $filenames) {
+    foreach ($filename in $fileNames) {
        $Path="$PSScriptRoot\ParentDisks\$FileName.ps1"
        If (Test-Path -Path $Path) {
            WriteSuccess "`t $Filename is present, skipping download"
        } else {
            $FileContent = $null
-            $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/Microsoft/MSLab/master/Tools/$FileName.ps1").Content
+
+            try {
+                # try to download release version first
+                $file = (Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/$Filename.ps1")
+                if($file.Headers["Content-Type"] -eq "application/octet-stream") {
+                    $FileContent = [System.Text.Encoding]::UTF8.GetString($file.Content)
+                }
+            } catch {
+                WriteInfo "Download $filename failed with $($_.Exception.Message), trying master branch now"
+                # if that fails, try main branch
+                $FileContent = (Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/$FileName.ps1").Content
+            }
+
            if ($FileContent) {
                $script = New-Item "$PSScriptRoot\ParentDisks\$FileName.ps1" -type File -Force
                Set-Content -path $script -value $FileContent
@ -124,17 +145,23 @@ function  Get-WindowsBuildNumber {
    }

 # Download convert-windowsimage into Temp
-WriteInfoHighlighted "Testing Convert-windowsimage presence"
-If ( Test-Path -Path "$PSScriptRoot\Temp\Convert-WindowsImage.ps1" ) {
-    WriteSuccess "`t Convert-windowsimage.ps1 is present, skipping download"
-}else{ 
-    WriteInfo "`t Downloading Convert-WindowsImage"
-    try {
-        Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile "$PSScriptRoot\Temp\Convert-WindowsImage.ps1"
-    } catch {
-        WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+    WriteInfoHighlighted "Testing Convert-windowsimage presence"
+    $convertWindowsImagePath = "$PSScriptRoot\Temp\Convert-WindowsImage.ps1"
+    If (Test-Path -Path $convertWindowsImagePath) {
+        WriteSuccess "`t Convert-windowsimage.ps1 is present, skipping download"
+    } else {
+        WriteInfo "`t Downloading Convert-WindowsImage"
+        try {
+            Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+        } catch {
+            try {
+                WriteInfo "Download Convert-windowsimage.ps1 failed with $($_.Exception.Message), trying master branch now"
+                Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+            } catch {
+                WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+            }
+        }
    }
-}
 #endregion

 #region some tools to download
--- a/Tools/CreateParentDisk.ps1
+++ b/Tools/CreateParentDisk.ps1
@ -40,13 +40,18 @@ If (-not $isAdmin) {
    #endregion

    #region download convert-windowsimage if needed and load it
-
-    if (!(Test-Path "$PSScriptRoot\Convert-WindowsImage.ps1")){
+    $convertWindowsImagePath = "$PSScriptRoot\Convert-WindowsImage.ps1"
+    if (-not (Test-Path -Path $convertWindowsImagePath)) {
        WriteInfo "`t Downloading Convert-WindowsImage"
        try {
-            Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile "$PSScriptRoot\Convert-WindowsImage.ps1"
+            Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/microsoft/MSLab/releases/download/$mslabVersion/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
        } catch {
-            WriteErrorAndExit "`t Failed to download Convert-WindowsImage.ps1!"
+            try {
+                WriteInfo "Download Convert-windowsimage.ps1 from releases ($mslabVersion) failed with $($_.Exception.Message), trying master branch now"
+                Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/microsoft/MSLab/master/Tools/Convert-WindowsImage.ps1" -OutFile $convertWindowsImagePath
+            } catch {
+                WriteError "`t Failed to download Convert-WindowsImage.ps1!"
+            }
        }
    }

--- a/build.ps1
+++ b/build.ps1
@ -11,12 +11,19 @@ param(
    [string]$ClientId
 )

+$toolsDir = ".\Tools\"
 $baseDir = ".\Scripts\"
-$outputDir = ".\Output"
+$outputBaseDir = ".\Output\"
+$outputDir = "$($outputBaseDir)\Compiled"
+$signedOutputDir = "$($outputBaseDir)\Signed"
+$toolsOutputDir = "$($outputBaseDir)\Tools"
 $outputFile = "Release.zip"
+
 [array]$ignoredFiles = "0_Shared.ps1"
 [array]$ignoredFilesToSign = @() #"LabConfig.ps1"
+[array]$toolsIgnoredFilesToSign = @("1_SQL_Install.ps1", "2_ADK_Install.ps1", "3_SCVMM_Install.ps1")

+#region Build (and optionally sign) Scripts
 if(Test-Path -Path $outputDir) {
    Remove-Item -Path $outputDir -Recurse -Force
 }
@ -63,31 +70,58 @@ foreach($file in $files) {
 $outputFullPath = $releaseDirectory.FullName

 if($SignScripts) {
-    # Download signing script
-    Invoke-WebRequest -Uri $SignScriptUri -OutFile .\sign.ps1
-
+    # Download signing script with Managed Identity
+    $token = Invoke-RestMethod -Uri "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fstorage.azure.com%2F" -Headers @{ "Metadata" = "true" }
+    Invoke-WebRequest -Headers @{ "x-ms-version" = "2017-11-09"; "Authorization" = "Bearer $($token.access_token)" } -Uri $SignScriptUri -OutFile .\sign.ps1
+    
    . .\sign.ps1
-
-    $signedOutputDir = "$($outputDir)\Signed"
-    if(Test-Path -Path $signedOutputDir) {
-        Remove-Item -Path $signedOutputDir -Recurse -Force
-    }
-
-    $signedReleaseDirectory = New-Item -ItemType "Directory" -Path ".\" -Name $signedOutputDir
-    $files = Get-ChildItem -Path $releaseDirectory -File | Where-Object Name -NotIn $ignoredFilesToSign 
-
-    # sign scripts
-    Invoke-CodeSign -Files $files -OutputPath $signedReleaseDirectory -ClientId $ClientId
-
-    $signedFiles = Get-ChildItem -Path $signedReleaseDirectory.FullName
-    if($files.Length -ne $signedFiles.Length) {
-        throw "Signing files failed (source count: $($files.Length), signedCount: $($signedFiles.Length))"
-    }
-
-    # and copy scripts that are ignored from signing
-    Get-ChildItem -Path $releaseDirectory -File | Where-Object Name -In $ignoredFilesToSign | Copy-Item -Destination $signedReleaseDirectory.FullName
-
-    $outputFullPath = $signedReleaseDirectory.FullName
 }

+if(Test-Path -Path $signedOutputDir) {
+    Remove-Item -Path $signedOutputDir -Recurse -Force
+}
+
+$signedReleaseDirectory = New-Item -ItemType "Directory" -Path ".\" -Name $signedOutputDir
+$files = Get-ChildItem -Path $releaseDirectory -File | Where-Object Name -NotIn $ignoredFilesToSign 
+
+if($SignScripts) {
+    # sign scripts
+    Invoke-CodeSign -Files $files -OutputPath $signedReleaseDirectory -ClientId $ClientId
+} else {
+    # if not signing, just copy files over as is
+    $files | Select-Object -ExpandProperty FullName | Copy-Item -Destination $signedReleaseDirectory
+}
+
+$signedFiles = Get-ChildItem -Path $signedReleaseDirectory.FullName
+if($files.Length -ne $signedFiles.Length) {
+    throw "Signing files failed (source count: $($files.Length), signedCount: $($signedFiles.Length))"
+}
+#endregion
+
+#region Build (and optionally sign) Tools
+if(Test-Path -Path $ToolsOutputDir) {
+    Remove-Item -Path $ToolsOutputDir -Recurse -Force
+}
+# and copy scripts that are ignored from signing
+Get-ChildItem -Path $releaseDirectory -File | Where-Object Name -In $ignoredFilesToSign | Copy-Item -Destination $signedReleaseDirectory.FullName
+
+$outputFullPath = $signedReleaseDirectory.FullName
+
+$toolsSignedDirectory = New-Item -ItemType "Directory" -Path ".\" -Name $toolsOutputDir
+$toolsFiles = Get-ChildItem -Path $toolsDir -File | Where-Object Name -NotIn $toolsIgnoredFilesToSign
+
+if($SignScripts) {
+    # Sign scripts in Tools folder
+    Invoke-CodeSign -Files $toolsFiles -OutputPath $toolsSignedDirectory -ClientId $ClientId
+} else {
+    # or just copy tools scripts over
+    $toolsFiles | Select-Object -ExpandProperty FullName | Copy-Item -Destination $toolsSignedDirectory
+}
+
+$signedToolsFiles = Get-ChildItem -Path $toolsSignedDirectory.FullName
+if($toolsFiles.Length -ne $signedToolsFiles.Length) {
+    throw "Signing files failed (source count: $($toolsFiles.Length), signedCount: $($signedToolsFiles.Length))"
+}
+#endregion
+
 Compress-Archive -Path "$($outputFullPath)\*" -DestinationPath $outputFile -CompressionLevel Optimal -Force