Add kernel regression tests (#3421)

* add tests * fix * add traces to scripts * add temp test * Revert "add temp test" This reverts commit 9ca282a62e. * code cleanup * pass version as param * fix * fix. --------- Co-authored-by: Shankar Seal <sseal@microsoft.com> Co-authored-by: Shankar Seal <74580197+shankarseal@users.noreply.github.com>
2024-04-03 11:52:02 -07:00 · 2024-04-03 11:52:02 -07:00 · 37b7cc5970
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@ -251,6 +251,25 @@ jobs:
      code_coverage: false
      configurations: '["NativeOnlyDebug", "NativeOnlyRelease"]'

+  # Run the regression driver tests on self-hosted runners (only for 2022).
+  regression_driver_ws2022:
+    # Always run this job.
+    # Only run this on repos that have self-host runners.
+    needs: regular
+    if: github.repository == 'microsoft/ebpf-for-windows' && (github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch')
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" -TestMode "Regression" -RegressionArtifactsVersion "0.15.0"
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Regression"
+      post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true
+      name: driver_ws2022
+      build_artifact: Build-x64
+      environment: ebpf_cicd_tests_ws2022
+      # driver test copies dumps to testlog folder.
+      gather_dumps: false
+      # driver tests manually gather code coverage
+      code_coverage: false
+
  ossar:
    # Always run this job.
    needs: regular
--- a/.github/workflows/reusable-test.yml
+++ b/.github/workflows/reusable-test.yml
@ -202,7 +202,7 @@ jobs:
      id: run_pre_test_command_self_hosted
      working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
      run: |
-        ${{env.PRE_COMMAND}} -LogFileName ${{ runner.name }}.log -SelfHostedRunnerName ${{ runner.name }}
+        ${{env.PRE_COMMAND}} -LogFileName ${{ runner.name }}.log -SelfHostedRunnerName ${{ runner.name }} -RegressionArtifactsConfiguration ${{env.BUILD_CONFIGURATION}}

    # TODO: Clean up the combination of options: https://github.com/microsoft/ebpf-for-windows/issues/1590
    - name: Run test with Code Coverage in VS Dev environment
--- a/scripts/config_test_vm.psm1
+++ b/scripts/config_test_vm.psm1
@ -217,6 +217,7 @@ function Export-BuildArtifactsToVMs
 function Install-eBPFComponentsOnVM
 {
    param([parameter(Mandatory=$true)][string] $VMName,
+          [parameter(Mandatory=$true)][string] $TestMode,
          [parameter(Mandatory=$true)][bool] $KmTracing,
          [parameter(Mandatory=$true)][string] $KmTraceType)

@ -227,13 +228,14 @@ function Install-eBPFComponentsOnVM
        param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
              [Parameter(Mandatory=$True)] [string] $LogFileName,
              [Parameter(Mandatory=$true)] [bool] $KmTracing,
-              [Parameter(Mandatory=$true)] [string] $KmTraceType)
+              [Parameter(Mandatory=$true)] [string] $KmTraceType,
+              [parameter(Mandatory=$true)][string] $TestMode)
        $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
        Import-Module $WorkingDirectory\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue

-        Install-eBPFComponents -KmTracing $KmTracing -KmTraceType $KmTraceType -KMDFVerifier $true -ErrorAction Stop
-    } -ArgumentList ("eBPF", $LogFileName, $KmTracing, $KmTraceType) -ErrorAction Stop
+        Install-eBPFComponents -KmTracing $KmTracing -KmTraceType $KmTraceType -KMDFVerifier $true -TestMode $TestMode -ErrorAction Stop
+    } -ArgumentList ("eBPF", $LogFileName, $KmTracing, $KmTraceType, $TestMode) -ErrorAction Stop
    Write-Log "eBPF components installed on $VMName" -ForegroundColor Green
 }

@ -540,7 +542,7 @@ function Initialize-NetworkInterfacesOnVMs
    }
 }

-function Get-RegressionTestArtifacts
+function Get-LegacyRegressionTestArtifacts
 {
    $ArifactVersionList = @("0.11.0")
    $RegressionTestArtifactsPath = "$pwd\regression"
@ -559,6 +561,7 @@ function Get-RegressionTestArtifacts
    # Download regression test artifacts for each version.
    foreach ($ArtifactVersion in $ArifactVersionList)
    {
+        Write-Log "Downloading legacy regression test artifacts for version $ArtifactVersion"
        $DownloadPath = "$RegressionTestArtifactsPath\$ArtifactVersion"
        mkdir $DownloadPath
        $ArtifactName = "v$ArtifactVersion/Build-x64-native-only-Release.$ArtifactVersion.zip"
@ -576,6 +579,52 @@ function Get-RegressionTestArtifacts
    }
 }

+function Get-RegressionTestArtifacts
+{
+    param([Parameter(Mandatory=$True)][string] $Configuration,
+          [Parameter(Mandatory=$True)][string] $ArtifactVersion)
+
+    $RegressionTestArtifactsPath = "$pwd\regression"
+    $OriginalPath = $pwd
+    if (Test-Path -Path $RegressionTestArtifactsPath) {
+        Remove-Item -Path $RegressionTestArtifactsPath -Recurse -Force
+    }
+    mkdir $RegressionTestArtifactsPath
+
+    # Verify artifacts' folder presence
+    if (-not (Test-Path -Path $RegressionTestArtifactsPath)) {
+        $ErrorMessage = "*** ERROR *** Regression test artifacts folder not found: $RegressionTestArtifactsPath)"
+        Write-Log $ErrorMessage
+        throw $ErrorMessage
+    }
+
+    # Download regression test artifacts for each version.
+    $DownloadPath = "$RegressionTestArtifactsPath"
+    $ArtifactName = "Release-v$ArtifactVersion/Build-x64.$Configuration.zip"
+    $ArtifactUrl = "https://github.com/microsoft/ebpf-for-windows/releases/download/" + $ArtifactName
+
+    Write-Log "Downloading regression test artifacts for version $ArtifactVersion" -ForegroundColor Green
+    $ProgressPreference = 'SilentlyContinue'
+    Invoke-WebRequest -Uri $ArtifactUrl -OutFile "$DownloadPath\artifact.zip"
+
+    Write-Log "Extracting $ArtifactName"
+    Expand-Archive -Path "$DownloadPath\artifact.zip" -DestinationPath $DownloadPath -Force
+
+    # Copy all the drivers, DLLs and exe to pwd.
+    Write-Log "Copy regression test artifacts to main folder" -ForegroundColor Green
+    $ArtifactPath = "$DownloadPath\Build-x64 $Configuration"
+    Push-Location $ArtifactPath
+    Get-ChildItem -Path .\* -Include *.sys | Move-Item -Destination $OriginalPath -Force
+    Get-ChildItem -Path .\* -Include *.dll | Move-Item -Destination $OriginalPath -Force
+    Get-ChildItem -Path .\* -Include *.exe | Move-Item -Destination $OriginalPath -Force
+    Pop-Location
+
+    Remove-Item -Path $DownloadPath -Force -Recurse
+
+    # Delete ebpfapi.dll from the artifacts. ebpfapi.dll from the MSI installation should be used instead.
+    Remove-Item -Path ".\ebpfapi.dll" -Force
+}
+
 # Copied from https://github.com/microsoft/msquic/blob/main/scripts/prepare-machine.ps1
 function Get-Duonic {
    # Download and extract https://github.com/microsoft/corenet-ci.
--- a/scripts/execute_ebpf_cicd_tests.ps1
+++ b/scripts/execute_ebpf_cicd_tests.ps1
@ -16,6 +16,11 @@ param ([parameter(Mandatory = $false)][string] $AdminTarget = "TEST_VM",

 Push-Location $WorkingDirectory

+# For test execution, "Regression" and "CI/CD" have same behavior.
+if ($TestMode -eq "Regression") {
+    $TestMode = "CI/CD"
+}
+
 $AdminTestVMCredential = Get-StoredCredential -Target $AdminTarget -ErrorAction Stop
 $StandardUserTestVMCredential = Get-StoredCredential -Target $StandardUserTarget -ErrorAction Stop

--- a/scripts/install_ebpf.psm1
+++ b/scripts/install_ebpf.psm1
@ -16,21 +16,25 @@ $EbpfDrivers = @{
        "Name" = "ebpfcore.sys"
        "IsDriver" = $true
        "InstalledByMsi" = $true
+        "ReplaceForRegressionTest" = $false
    }
    "NetEbpfExt" = [PSCustomObject]@{
        "Name" = "netebpfext.sys"
        "IsDriver" = $true
        "InstalledByMsi" = $true
+        "ReplaceForRegressionTest" = $true
    }
    "SampleEbpfExt" = [PSCustomObject]@{
        "Name" = "sample_ebpf_ext.sys"
        "IsDriver" = $true
        "InstalledByMsi" = $false
+        "ReplaceForRegressionTest" = $true
    }
    "EbpfSvc" = [PSCustomObject]@{
        "Name" = "ebpfsvc.exe"
        "IsDriver" = $false
        "InstalledByMsi" = $true
+        "ReplaceForRegressionTest" = $false
    }
 }

@ -159,7 +163,8 @@ function Install-eBPFComponents
 {
    param([parameter(Mandatory=$true)] [bool] $KmTracing,
          [parameter(Mandatory=$true)] [string] $KmTraceType,
-          [parameter(Mandatory=$false)] [bool] $KMDFVerifier = $false)
+          [parameter(Mandatory=$false)] [bool] $KMDFVerifier = $false,
+          [parameter(Mandatory=$true)] [string] $TestMode)

    # Print the status of the eBPF drivers and services before installation.
    # This is useful for detecting issues with the runner baselines.
@ -239,6 +244,64 @@ function Install-eBPFComponents
        }
    }

+    # If TestMode is "Regression", reinstall the extension drivers from the regression test artifacts.
+    if ($TestMode -eq "Regression") {
+        Write-Log("Reinstalling the extension drivers from the regression test artifacts...") -ForegroundColor Green
+        $EbpfDrivers.GetEnumerator() | ForEach-Object {
+            if ($_.Value.InstalledByMsi) {
+                if ($_.Value.ReplaceForRegressionTest) {
+
+                    # Stop and delete the service.
+                    Write-Log("Regression Tests: Stopping $($_.Key) service...") -ForegroundColor Green
+                    sc.exe stop $_.Key 2>&1 | Write-Log
+                    if ($LASTEXITCODE -ne 0) {
+                        throw ("Failed to stop $($_.Key) service.")
+                    } else {
+                        Write-Log("$($_.Key) service stopped.") -ForegroundColor Green
+                        Write-Log("Deleting $($_.Key) service...") -ForegroundColor Green
+                        sc.exe delete $_.Key 2>&1 | Write-Log
+                        if ($LASTEXITCODE -ne 0) {
+                            throw ("Failed to delete $($_.Key) service.")
+                        } else {
+                            Write-Log("$($_.Key) service deleted.") -ForegroundColor Green
+                        }
+                    }
+
+                    # Install the driver.
+                    $driverPath = if (Test-Path -Path ("$pwd\{0}" -f $_.Value.Name)) {
+                        "$pwd\{0}" -f $_.Value.Name
+                    } elseif (Test-Path -Path ("$pwd\drivers\{0}" -f $_.Value.Name)) {
+                        "$pwd\drivers\{0}" -f $_.Value.Name
+                    } else {
+                        throw ("Driver file not found for $($_.Key).")
+                    }
+                    Write-Log("Installing $($_.Key) from path $driverPath ...") -ForegroundColor Green
+                    sc.exe create $_.Key type=kernel start=demand binpath=$driverPath 2>&1 | Write-Log
+                    if ($LASTEXITCODE -ne 0) {
+                        throw ("Failed to create $($_.Key) driver.")
+                    } else {
+                        Write-Log("$($_.Key) driver created.") -ForegroundColor Green
+                        # Start the service.
+                        Write-Log("Starting $($_.Key) service...") -ForegroundColor Green
+                        sc.exe start $_.Key 2>&1 | Write-Log
+                        if ($LASTEXITCODE -ne 0) {
+                            throw ("Failed to start $($_.Key) service.")
+                        } else {
+                            Write-Log("$($_.Key) service started.") -ForegroundColor Green
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    # Copy ebpfapi.dll from installation directory to current directory.
+    # This ensures latest version of ebpfapi is used by the tests.
+    $ebpfApiDllPath = Join-Path $env:ProgramFiles "ebpf-for-windows\ebpfapi.dll"
+    Write-Log("Copying ebpfapi.dll from '$ebpfApiDllPath' to '$pwd'...")
+    Copy-Item -Path $ebpfApiDllPath -Destination $pwd -Force
+
    # Export program info for the sample driver.
    Write-Log("Running 'export_program_info_sample.exe'...")
    if (Test-Path -Path "export_program_info_sample.exe") {
--- a/scripts/setup_ebpf_cicd_tests.ps1
+++ b/scripts/setup_ebpf_cicd_tests.ps1
@ -7,6 +7,8 @@ param ([parameter(Mandatory=$false)][string] $Target = "TEST_VM",
       [parameter(Mandatory=$false)][string] $TestMode = "CI/CD",
       [parameter(Mandatory=$false)][string] $LogFileName = "TestLog.log",
       [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(),
+       [parameter(Mandatory=$false)][string] $RegressionArtifactsVersion = "",
+       [parameter(Mandatory=$false)][string] $RegressionArtifactsConfiguration = "",
       [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json",
       [parameter(Mandatory=$false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName())

@ -33,10 +35,16 @@ Remove-Item ".\TestLogs" -Recurse -Confirm:$false -ErrorAction SilentlyContinue
 # Get all VMs to ready state.
 Initialize-AllVMs -VMList $VMList -ErrorAction Stop

-if ($TestMode -eq "CI/CD") {
+if ($TestMode -eq "Regression") {

    # Download the release artifacts for regression tests.
-    Get-RegressionTestArtifacts
+    Get-RegressionTestArtifacts -ArtifactVersion $RegressionArtifactsVersion -Configuration $RegressionArtifactsConfiguration
+}
+
+if ($TestMode -eq "CI/CD" -or $TestMode -eq "Regression") {
+
+    # Download the release artifacts for legacy regression tests.
+    Get-LegacyRegressionTestArtifacts
 }

 Get-Duonic
@ -51,7 +59,7 @@ Initialize-NetworkInterfacesOnVMs $VMList -ErrorAction Stop
 # Install eBPF Components on the test VM.
 foreach($VM in $VMList) {
    $VMName = $VM.Name
-    Install-eBPFComponentsOnVM -VMName $VMname -KmTracing $KmTracing -KmTraceType $KmTraceType -ErrorAction Stop
+    Install-eBPFComponentsOnVM -VMName $VMname -TestMode $TestMode -KmTracing $KmTracing -KmTraceType $KmTraceType -ErrorAction Stop
 }

 Pop-Location