Add role assignment cleanup and workaround for the exit code 100 error. (#2049)

* Add role assignment cleanup

* Add verbose logging for troubleshooting the exit code 100 issue

* bypass the error for grub-efi-amd64-signed

* Adding the redirections back

.github/workflows/automated-cleanup-resources.yml

@@ -26,7 +26,12 @@ jobs:
       uses: azure/login@v1
       with:
         creds: ${{secrets.AZUREML_CREDENTIALS}}
+        enable-AzPSSession: true
       continue-on-error: true
+    - name: "Install Az Modules"
+      shell: pwsh
+      run: |
+        Install-Module Az -Force; Import-Module Az -Force;
     - name: setup-cli
       run: |
           source "${{ github.workspace }}/infra/sdk_helpers.sh";
@@ -41,3 +46,8 @@ jobs:
           bash cleanup.sh
       working-directory: infra
       continue-on-error: true
+    - name: Perform additional Cleanup
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          pwsh -File '${{ github.workspace }}/infra/scripts/call_all_scripts.ps1'

infra/scripts/call_all_scripts.ps1

@@ -0,0 +1,5 @@
+####################################################################################################
+# All the post cleanup scripts are called from this file                                           #
+####################################################################################################
+
+&"$PSScriptroot\remove_role_assignments.ps1" -ResourceGroupName "$env:RESOURCE_GROUP_NAME"

infra/scripts/remove_role_assignments.ps1

@@ -0,0 +1,38 @@
+####################################################################################################
+# This script allows you to remove stale role assignments listed as 'Identity not found'           #
+####################################################################################################
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory=$true)]
+    [ValidateNotNullOrEmpty()]
+    [string]$ResourceGroupName
+)
+function Get-RoleAssignmentCount {
+    $OBJTYPE = "Unknown"
+    Write-Output "Determing unknown Role Assignment count in the ResourceGroup:$ResourceGroupName..."
+    $stale = Get-AzRoleAssignment -ResourceGroupName "$ResourceGroupName" | Where-Object { $_.ObjectType -eq $OBJTYPE}
+    $unknownRoleAssignmentCount = $stale.Count
+    Write-Output "Total Unknown Role Assignment Count: $unknownRoleAssignmentCount in the ResourceGroup:$ResourceGroupName..."
+
+    return $unknownRoleAssignmentCount
+}
+
+try
+{
+    $OBJTYPE = "Unknown"
+    Write-Output "Pre-checking the RoleAssignment count..."
+    Get-RoleAssignmentCount
+    # Remove only limited RoleDefinitions
+    $staleRoleAssignments = Get-AzRoleAssignment -ResourceGroupName "$ResourceGroupName" | Where-Object {($_.ObjectType -eq $OBJTYPE) -and ($_.RoleDefinitionName -match "Storage Blob Data Reader|AzureML Metrics Writer (preview)|AcrPull")}
+    $unknownRoleAssignmentCount = $staleRoleAssignments.Count
+    Write-Output "Initiating the cleanup of unknownRole in the ResourceGroup:$ResourceGroupName having count as $unknownRoleAssignmentCount..."
+    $staleRoleAssignments | Remove-AzRoleAssignment
+    Write-Output "Check the Role Assignment count after cleanup"
+    Get-RoleAssignmentCount
+    Write-Output "Role Assignment clean-up complete."
+}
+catch
+{
+    Write-Error "There was an issue in cleaning-up the Role Assignment. See details: $($_.Exception.Message)"
+    Exit 1
+}

infra/sdk_helpers.sh

@@ -266,9 +266,14 @@ function install_packages() {
     echo_info ">>> Updating packages index"
     echo_info "------------------------------------------------"
 
-    sudo apt-get update > /dev/null 2>&1
-    sudo apt-get upgrade -y > /dev/null 2>&1
-    sudo apt-get dist-upgrade -y > /dev/null 2>&1
+    echo_info ">>> Executing: apt-get update"
+    # https://github.com/orgs/community/discussions/47863
+    sudo apt-mark hold grub-efi-amd64-signed
+    sudo apt-get update --fix-missing
+    echo_info ">>> Executing: apt-get upgrade"
+    sudo apt-get upgrade -y
+    echo_info ">>> Executing: apt-get dist-upgrade"
+    sudo apt-get dist-upgrade -y
 
     echo_info ">>> Installing packages"