msquic/docs/AZP.md

Azure Pipelines Machine Setup

The following are a set of instructions to set up an Azure Pipeline test machine for running the latest Windows builds on private/custom hardware. Both VMs and HW are supported/allowed.

Windows OS Install

• Install the latest Windows Server Datacenter (desktop) SKU, ideally from the RS_PRERELEASE branch if possible, since symbols persist the longest for that branch.
• Log in as Administrator.
• Make sure to activate it. If necessary grab a product key from the internal website.

Machine Setup

• Copy sfpcopy.exe into C:\Windows\System32
• Disable Windows Defender
• Install the latest PowerShell: iex "& { $(irm https://aka.ms/install-powershell.ps1) } -UseMSI"
• Download and install VS build tools:
  • https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools&rel=16
  • Specifically make sure C++ Build Tools is installed
• Enable the kernel debugger:

bcdedit /dbgsettings NET HOSTIP:192.168.1.160 PORT:50000 /noumex
bcdedit -deletevalue {dbgsettings} hostip
bcdedit /debug on

• Start windbg on debug server machine to connect with autogenerated key from above.
• Reboot

Agent Setup

• Download the agent: https://vstsagentpackage.azureedge.net/agent/2.172.2/vsts-agent-win-x64-2.172.2.zip
• Unzip the contents to C:\a
• Run config.cmd
  • Connect to https://dev.azure.com/ms
  • Use the PAT (ping me to ask for it)
  • Add to MsQuic-Win-Latest pool
  • Use w for the work folder name
  • Run it as a service, using the default account
• Open 'Services' and find the Azure Pipelines service that was created
• Change the service to run a Local System
  • Then stop and restart it
• Change the service to run as Administrator
  • Then stop and restart it

Script for preparing VMs for use in the AZP pool

param (
    [Parameter(Mandatory = $true)]
    [ValidateSet("prepareseed", "create", "config", "delete", "launchdebug")]
    [string]$Mode,

    [Parameter(Mandatory = $false)]
    [string]$SeedVhdLocation = 'c:\SeedVHDs',

    [Parameter(Mandatory = $false)]
    [string]$SeedVhd = 'seed.vhdx',

    [Parameter(Mandatory = $false)]
    [string]$VhdLocation = 'c:\users\public\documents\Hyper-V\Virtual hard disks',

    [Parameter(Mandatory = $false)]
    [long]$VhdSize = 40GB,

    [Parameter(Mandatory = $false)]
    [string]$UnattendLocation = "$($PSScriptRoot)",

    [Parameter(Mandatory = $false)]
    [string]$PAT = '',

    [Parameter(Mandatory = $false)]
    [long]$VmCores = 4,

    [Parameter(Mandatory = $false)]
    [long]$MaximumVmMemory = 4096,

    [Parameter(Mandatory = $false)]
    [long]$VmsToProcess = 14,

    [Parameter(Mandatory = $false)]
    [long]$VmNumberStart = 1,

    [Parameter(Mandatory = $false)]
    [string]$VmNamePrefix = "AZP",

    [Parameter(Mandatory = $false)]
    [string]$VmPassword = "Very!SecurePassword2", <#[SuppressMessage("Microsoft.Security", "CS001:SecretInline", Justification="Example password for demo")]#>

    [Parameter(Mandatory = $false)]
    [string]$AgentPool = 'MsQuic-Win-Latest',

    [Parameter(Mandatory = $false)]
    [string]$AgentUrl = 'https://dev.azure.com/ms',

    [Parameter(Mandatory = $false)]
    [long]$DebugPortBase = 50000
)

$ProgressPreference = 'SilentlyContinue'

$AzpDebugKey = "something.very.very.secret"

function ExecuteScriptWithMountedVhd($vhd, $scriptblock)
{
    $vhdname = (Split-Path $vhd -Leaf).substring(0,8)
    $vhdmountdir = "$($Env:Temp)\vhdmount\$vhdname"
    if (Test-Path $vhdmountdir) {
        throw "VHD mount path already exists."
    }
    mkdir $vhdmountdir > $null
    dism.exe /mount-image /ImageFile:$vhd /index:1 /mountdir:$vhdmountdir
    & $scriptblock
    dism.exe /unmount-image /mountdir:$vhdmountdir /commit
    Remove-Item -Path $vhdmountdir -Force > $null
}

function AddUnattendFile($vhd, $unattendfile)
{
    ExecuteScriptWithMountedVhd $vhd {
        dism.exe /image:$vhdmountdir /apply-unattend:$unattendfile
        Copy-Item $unattendfile $vhdmountdir
    }
}

function CreateVM($VmName, $VmNumber)
{
    $VhdExtension = $SeedVhd.split('.')[-1]
    $VhdPath = "$(Join-Path $VhdLocation $VmName).$($VhdExtension)"

    if (-not (Test-Path $VhdPath)) {
        New-VHD -Path $VhdPath -ParentPath (Join-Path $SeedVhdLocation $SeedVhd) -Differencing
    }

    $VMGeneration = 1
    if ($VhdExtension.Contains("vhdx")) {
        $VmGeneration = 2
    }

    New-Vm -Name $VmName -VHDPath $VhdPath -Generation $VMGeneration -SwitchName "ExternalSwitch"

    Set-VMProcessor -VMName $VmName -Count $VmCores

    Set-VMMemory -VMName $VmName -DynamicMemoryEnabled $true -MinimumBytes (512 * 1024 * 1024) -StartupBytes (1024 *1024 * 1024) -MaximumBytes ($MaximumVmMemory * 1024 * 1024)

    if ($VMGeneration -eq 2) {
        Set-VMFirmware -VMName $VmName -FirstBootDevice (Get-VMHardDiskDrive -VMName $VmName) -EnableSecureBoot Off
    }

    Start-VM $VmName

    # Wait for VM to start - never gets to "Healthy"
    # while ((Get-VM -Name $VmName).HeartBeat -ne  'OkApplicationsHealthy') {
    #     Write-Host "VM state is: $((Get-VM -Name $VmName).HeartBeat) Sleeping 5 seconds..."
    #     Start-Sleep -Seconds 5
    # }
}

function ConfigureVM($VmName, $VmNumber)
{
    $block = {
        param($Port, $HostIP, $VmName, $Token, $DebugKey, $Password, $PoolName, $URL)
        Set-MpPreference -DisableRealtimeMonitoring $true
        Copy-Item c:\sfpcopy.exe c:\Windows\System32
        Invoke-Expression "& { $(Invoke-RestMethod https://aka.ms/install-powershell.ps1) } -UseMSI"
        bcdedit.exe /dbgsettings NET HOSTIPV6:$HostIP PORT:$Port KEY:$DebugKey /noumex
        bcdedit.exe /debug on
        # Install VS build tools
        c:\vs_buildtools.exe --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools --IncludeRecommended -q --force --wait
        # Add agent to pool
        c:\a\config.cmd --unattended --URL "$($URL)" --auth pat --token "$($Token)" --pool "$($PoolName)" --agent "$($VmName)" --replace --work "c:\a\w" --runAsService
        # Enable services to run as Administrator
        secedit.exe /export /cfg "$($Env:Temp)\secconfig.cfg"
        (Get-Content "$($Env:Temp)\secconfig.cfg") -replace 'SeServiceLogonRight = (?<SIDs>.*)', 'SeServiceLogonRight = Administrator,$1' | Set-Content "$($Env:Temp)\secconfig.cfg"
        secedit.exe /configure /db c:\windows\security\local.sdb /cfg "$($Env:Temp)\secconfig.cfg" /areas USER_RIGHTS
        # Set AZP service to run as Administrator
        #$cred = New-Object System.Management.Automation.PSCredential ('Administrator', (ConvertTo-SecureString $Password -AsPlainText -Force))
        #& "C:\Program Files\Powershell\7\pwsh.exe" -Command Set-Service -Name "vsts*" -Credential (New-Object System.Management.Automation.PSCredential ('Administrator', (ConvertTo-SecureString $Password -AsPlainText -Force)))
        #Get-Service vsts* | Restart-Service
        $ServiceName = (get-service vsts*).name
        $WmiObject = Get-WMIObject Win32_Service -filter "name='$ServiceName'"
        $StopStatus = $WmiObject.StopService()
        If ($StopStatus.ReturnValue -eq "0") {
            Write-host "The service '$ServiceName' Stopped successfully"
        }
        $ChangeStatus = $WmiObject.change($null,$null,$null,$null,$null,$null,'.\Administrator',$Password,$null,$null,$null)
        If ($ChangeStatus.ReturnValue -eq "0") {
            Write-host "Set User Name sucessfully for the service '$ServiceName'"
        }
        Start-Sleep 5
        $StartStatus = $WmiObject.StartService()
        If ($StartStatus.ReturnValue -eq "0") {
            Write-host "The service '$ServiceName' Started successfully"
        }
        # Visual studio installer doesn't respect the --wait parameter, so we can't automatically reboot
    }

    $DebugPort = $DebugPortBase + $VmNumber
    $HostIP =  (get-netipaddress -AddressFamily IPv6 -AddressState Preferred -InterfaceAlias "vEthernet (ExternalSwitch)" -ValidLifetime ([TimeSpan]::MaxValue)).IPAddress.Split('%')[0]

    $password = ConvertTo-SecureString $VmPassword -AsPlainText -Force
    $credential = New-Object System.Management.Automation.PSCredential ('Administrator', $password)

    #start up WinDbgX
    windbgx.exe -y c:\symbols -k net:port=$DebugPort,key=$AzpDebugKey

    #Configure VM
    Invoke-Command -VmName $VmName -Credential $credential -ScriptBlock $block -ArgumentList $DebugPort,$HostIP,$VmName,$PAT, $AzpDebugKey,$VmPassword,$AgentPool,$AgentUrl

    Write-Host "Finished setting up $VmName"
}

function DeleteVM($VmName)
{
    if ((Get-VM -Name $VmName).State -eq 'Running') {

        $block = {
            param($Token)
            C:\a\config.cmd remove --unattended --auth pat --token "$($Token)"
        }

        $password = ConvertTo-SecureString $VmPassword -AsPlainText -Force
        $credential = New-Object System.Management.Automation.PSCredential ('Administrator', $password)

        Invoke-Command -VmName $VmName -Credential $credential -ScriptBlock $block -ArgumentList $PAT

        # Turn off running VM
        Stop-VM -Name $VmName -TurnOff -Force
    }
   # Remove VHDs (and snapshot VHDs) before the VM
   Remove-Item -Path "$(Join-Path $VhdLocation $VmName)*" -Force
   Remove-VM -Name $VmName -Force
}

#
# Actually run the script
#

if ($Mode -eq "prepareseed") {
    $Vhd = Join-Path $SeedVhdLocation $SeedVhd
    if ((Get-VHD $Vhd).Size -lt $VhdSize) {
        Resize-VHD -Path $Vhd -SizeBytes $VhdSize
    }
    AddUnattendFile $Vhd "$($UnattendLocation)\unattend.xml"
    ExecuteScriptWithMountedVhd $Vhd {
        Invoke-WebRequest -Uri "https://vstsagentpackage.azureedge.net/agent/2.172.2/vsts-agent-win-x64-2.172.2.zip" -OutFile "$($vhdmountdir)\vsts-agent-win-x64-2.172.2.zip"
        Expand-Archive -Path "$($vhdmountdir)\vsts-agent-win-x64-2.172.2.zip" -DestinationPath "$($vhdmountdir)\a" -Force
        Copy-Item "vs_buildtools.exe" "$($vhdmountdir)\vs_buildtools.exe"
        Copy-Item "sfpcopy.exe" "$($vhdmountdir)\sfpcopy.exe"
    }
} else {
    for ($i = $VmNumberStart; $i -le $VmsToProcess; $i++) {
        if ($Mode -eq "create") {
            CreateVM "$($VmNamePrefix)-$($i)" $i
        } elseif ($Mode -eq "config") {
            ConfigureVM "$($VmNamePrefix)-$($i)" $i
        } elseif ($Mode -eq "delete") {
             $VmName = "$($VmNamePrefix)-$($i)"
             DeleteVm $VmName
        } elseif ($Mode -eq "launchdebug") {
            $DebugPort = $DebugPortBase + $i
            windbgx.exe -y c:\symbols -k net:port=$DebugPort,key=$AzpDebugKey
        }
    }
}