Implement Kernel Mode Performance Testing (#722)

Changed required include using a private kernel library for CI testing, as msquic.sys can't be properly unloaded.
2020-08-21 13:46:39 -07:00 · 2020-08-21 13:46:39 -07:00 · c62d90db53
--- a/.azure/azure-pipelines.ci.yml
+++ b/.azure/azure-pipelines.ci.yml
@ -178,6 +178,7 @@ stages:
  dependsOn:
  - build_windows
  - build_linux
+  - build_winkernel
  jobs:
  - template: ./templates/run-performance.yml
    parameters:
@ -186,6 +187,14 @@ stages:
      localTls: schannel
      remoteTls: schannel
      extraArgs: '-Publish'
+  - template: ./templates/run-performance.yml
+    parameters:
+      pool: MsQuic-Win-Perf
+      platform: windows
+      localTls: schannel
+      remoteTls: schannel
+      extraName: 'Kernel'
+      kernelMode: -Kernel
  - template: ./templates/run-performance.yml
    parameters:
      pool: MsQuic-Win-Perf
--- a/.azure/azure-pipelines.perf.yml
+++ b/.azure/azure-pipelines.perf.yml
@ -9,6 +9,10 @@ pr: none
 name: 0.$(Date:yyyy).$(Date:MM).$(DayOfMonth).$(Rev:rr).0

 parameters:
+- name: kernelmode
+  type: boolean
+  displayName: Kernel Mode
+  default: false
 - name: mode
  type: string
  displayName: Mode
@ -32,17 +36,23 @@ stages:
    runCodesignValidationInjection: false
  jobs:
  # Officially supported configurations.
-  - template: ./templates/build-config-user.yml
-    parameters:
-      image: windows-latest
-      platform: windows
-      arch: x86
-      tls: schannel
-      config: Release
-      ${{ if eq(parameters.mode, 'PGO') }}:
-        extraBuildArgs: -DisableTest -DisableTools -PGO 
-      ${{ if ne(parameters.mode, 'PGO') }}:
-        extraBuildArgs: -DisableTest -DisableTools
+  - ${{ if eq(parameters.kernelmode, false) }}:
+    - template: ./templates/build-config-user.yml
+      parameters:
+        image: windows-latest
+        platform: windows
+        arch: x86
+        tls: schannel
+        config: Release
+        ${{ if eq(parameters.mode, 'PGO') }}:
+          extraBuildArgs: -DisableTest -DisableTools -PGO
+        ${{ if ne(parameters.mode, 'PGO') }}:
+          extraBuildArgs: -DisableTest -DisableTools
+  - ${{ if eq(parameters.kernelmode, true) }}:
+    - template: ./templates/build-config-winkernel.yml
+      parameters:
+        arch: x64
+        config: Release
  - template: ./templates/build-config-user.yml
    parameters:
      image: windows-latest
@ -70,53 +80,56 @@ stages:
      platform: windows
      localTls: schannel
      remoteTls: schannel
+      kernelMode: -Kernel
      ${{ if eq(parameters.mode, 'PGO') }}:
        extraArgs: -PGO
      ${{ if eq(parameters.mode, 'Record') }}:
        extraArgs: -RecordStack
      ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
        extraArgs: -RecordQUIC
-  - template: ./templates/run-performance.yml
-    parameters:
-      pool: MsQuic-Win-Perf
-      platform: windows
-      localTls: schannel
-      remoteTls: schannel
-      arch: x86
-      ${{ if eq(parameters.mode, 'PGO') }}:
-        extraArgs: -PGO
-      ${{ if eq(parameters.mode, 'Record') }}:
-        extraArgs: -RecordStack
-      ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
-        extraArgs: -RecordQUIC
-  - template: ./templates/run-performance.yml
-    parameters:
-      pool: MsQuic-Win-Perf
-      platform: windows
-      localTls: schannel
-      remoteTls: schannel
-      extraName: Loopback
-      ${{ if eq(parameters.mode, 'PGO') }}:
-        extraArgs: -PGO -Local
-      ${{ if eq(parameters.mode, 'Record') }}:
-        extraArgs: -RecordStack -Local
-      ${{ if eq(parameters.mode, 'Normal') }}:
-        extraArgs: -Local
-      ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
-        extraArgs: -RecordQUIC -Local
-  - template: ./templates/run-performance.yml
-    parameters:
-      pool: MsQuic-Win-Perf
-      platform: windows
-      localTls: schannel
-      remoteTls: schannel
-      extraName: Loopback
-      arch: x86
-      ${{ if eq(parameters.mode, 'PGO') }}:
-        extraArgs: -PGO -Local
-      ${{ if eq(parameters.mode, 'Record') }}:
-        extraArgs: -RecordStack -Local
-      ${{ if eq(parameters.mode, 'Normal') }}:
-        extraArgs: -Local
-      ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
-        extraArgs: -RecordQUIC -Local
+  # Skip 32 bit and loopback in kernel mode
+  - ${{ if eq(parameters.kernelmode, false) }}:
+    - template: ./templates/run-performance.yml
+      parameters:
+        pool: MsQuic-Win-Perf
+        platform: windows
+        localTls: schannel
+        remoteTls: schannel
+        arch: x86
+        ${{ if eq(parameters.mode, 'PGO') }}:
+          extraArgs: -PGO
+        ${{ if eq(parameters.mode, 'Record') }}:
+          extraArgs: -RecordStack
+        ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
+          extraArgs: -RecordQUIC
+    - template: ./templates/run-performance.yml
+      parameters:
+        pool: MsQuic-Win-Perf
+        platform: windows
+        localTls: schannel
+        remoteTls: schannel
+        extraName: Loopback
+        ${{ if eq(parameters.mode, 'PGO') }}:
+          extraArgs: -PGO -Local
+        ${{ if eq(parameters.mode, 'Record') }}:
+          extraArgs: -RecordStack -Local
+        ${{ if eq(parameters.mode, 'Normal') }}:
+          extraArgs: -Local
+        ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
+          extraArgs: -RecordQUIC -Local
+    - template: ./templates/run-performance.yml
+      parameters:
+        pool: MsQuic-Win-Perf
+        platform: windows
+        localTls: schannel
+        remoteTls: schannel
+        extraName: Loopback
+        arch: x86
+        ${{ if eq(parameters.mode, 'PGO') }}:
+          extraArgs: -PGO -Local
+        ${{ if eq(parameters.mode, 'Record') }}:
+          extraArgs: -RecordStack -Local
+        ${{ if eq(parameters.mode, 'Normal') }}:
+          extraArgs: -Local
+        ${{ if eq(parameters.mode, 'RecordQUICTrace') }}:
+          extraArgs: -RecordQUIC -Local
--- a/.azure/templates/build-config-winkernel.yml
+++ b/.azure/templates/build-config-winkernel.yml
@ -2,6 +2,7 @@

 parameters:
  arch: ''
+  config: 'Debug,Release'

 jobs:
 - job: build_winkernel_${{ parameters.arch }}
@ -26,12 +27,16 @@ jobs:
      restoreSolution: msquic.kernel.sln

  - task: VSBuild@1
+    displayName: Build Source Code (Debug)
+    condition: contains('${{ parameters.config }}', 'Debug')
    inputs:
      solution: msquic.kernel.sln
      platform: ${{ parameters.arch }}
      configuration: debug

  - task: VSBuild@1
+    displayName: Build Source Code (Release)
+    condition: contains('${{ parameters.config }}', 'Release')
    inputs:
      solution: msquic.kernel.sln
      platform: ${{ parameters.arch }}
--- a/.azure/templates/run-performance.yml
+++ b/.azure/templates/run-performance.yml
@ -10,6 +10,7 @@ parameters:
  remoteTls: ''
  extraName: ''
  extraArgs: '-Publish'
+  kernelMode: ''

 jobs:
 - job: performance_${{ parameters.platform }}_${{ parameters.arch }}_${{ parameters.localTls }}_${{ parameters.remoteTls }}_${{ parameters.extraName }}
@ -38,6 +39,7 @@ jobs:
      platform: ${{ parameters.platform }}
      arch: ${{ parameters.arch }}
      tls: ${{ parameters.localTls }}
+      includeKernel: ${{ ne(parameters.kernelMode, '') }}

  - ${{ if ne(parameters.localTls, parameters.remoteTls) }}:
    - template: ./download-artifacts.yml
@ -45,6 +47,7 @@ jobs:
        platform: ${{ parameters.platform }}
        arch: ${{ parameters.arch }}
        tls: ${{ parameters.remoteTls }}
+        includeKernel: ${{ ne(parameters.kernelMode, '') }}

  - task: PowerShell@2
    displayName: Prepare Test Machine
@ -60,7 +63,7 @@ jobs:
    inputs:
      pwsh: true
      filePath: scripts/performance.ps1
-      arguments: -Config ${{ parameters.config }} -LocalTls ${{ parameters.localTls }} -RemoteTls ${{ parameters.remoteTls }} -LocalArch ${{ parameters.arch }} -RemoteArch ${{ parameters.arch }} ${{ parameters.extraArgs }}
+      arguments: -Config ${{ parameters.config }} -LocalTls ${{ parameters.localTls }} -RemoteTls ${{ parameters.remoteTls }} -LocalArch ${{ parameters.arch }} -RemoteArch ${{ parameters.arch }} ${{ parameters.kernelMode }} ${{ parameters.extraArgs }}

  - task: CopyFiles@2
    displayName: Move Performance Results
--- a/msquic.kernel.sln
+++ b/msquic.kernel.sln
@ -22,6 +22,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "perflib.kernel", "src\perf\
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "performance.kernel", "src\perf\bin\performance.kernel.vcxproj", "{2BE64DBF-60E6-4FE8-96B0-5F2526405096}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msquicpriv.kernel", "src\bin\winkernel\msquicpriv.kernel.vcxproj", "{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msquictestpriv.kernel", "src\test\bin\winkernel\msquictestpriv.kernel.vcxproj", "{C8491270-B0BE-440C-B88D-7B441A8CB67E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|ARM = Debug|ARM
@ -226,6 +230,54 @@ Global
 		{2BE64DBF-60E6-4FE8-96B0-5F2526405096}.Release|x86.ActiveCfg = Release|Win32
 		{2BE64DBF-60E6-4FE8-96B0-5F2526405096}.Release|x86.Build.0 = Release|Win32
 		{2BE64DBF-60E6-4FE8-96B0-5F2526405096}.Release|x86.Deploy.0 = Release|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM.ActiveCfg = Debug|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM.Build.0 = Debug|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM.Deploy.0 = Debug|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM64.Build.0 = Debug|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|ARM64.Deploy.0 = Debug|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x64.ActiveCfg = Debug|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x64.Build.0 = Debug|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x64.Deploy.0 = Debug|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x86.ActiveCfg = Debug|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x86.Build.0 = Debug|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Debug|x86.Deploy.0 = Debug|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM.ActiveCfg = Release|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM.Build.0 = Release|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM.Deploy.0 = Release|ARM
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM64.ActiveCfg = Release|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM64.Build.0 = Release|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|ARM64.Deploy.0 = Release|ARM64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x64.ActiveCfg = Release|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x64.Build.0 = Release|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x64.Deploy.0 = Release|x64
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x86.ActiveCfg = Release|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x86.Build.0 = Release|Win32
+		{E2DDB0A8-594D-424D-9ADD-4EF211F7FC3F}.Release|x86.Deploy.0 = Release|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM.ActiveCfg = Debug|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM.Build.0 = Debug|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM.Deploy.0 = Debug|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM64.Build.0 = Debug|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|ARM64.Deploy.0 = Debug|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x64.ActiveCfg = Debug|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x64.Build.0 = Debug|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x64.Deploy.0 = Debug|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x86.ActiveCfg = Debug|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x86.Build.0 = Debug|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Debug|x86.Deploy.0 = Debug|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM.ActiveCfg = Release|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM.Build.0 = Release|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM.Deploy.0 = Release|ARM
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM64.ActiveCfg = Release|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM64.Build.0 = Release|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|ARM64.Deploy.0 = Release|ARM64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x64.ActiveCfg = Release|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x64.Build.0 = Release|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x64.Deploy.0 = Release|x64
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x86.ActiveCfg = Release|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x86.Build.0 = Release|Win32
+		{C8491270-B0BE-440C-B88D-7B441A8CB67E}.Release|x86.Deploy.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/scripts/performance-helper.psm1
+++ b/scripts/performance-helper.psm1
@ -87,7 +87,7 @@ $WpaQUICLogProfileXml = `

 function Set-ScriptVariables {
    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '')]
-    param ($Local, $LocalTls, $LocalArch, $RemoteTls, $RemoteArch, $Config, $Publish, $Record, $RecordQUIC, $RemoteAddress, $Session)
+    param ($Local, $LocalTls, $LocalArch, $RemoteTls, $RemoteArch, $Config, $Publish, $Record, $RecordQUIC, $RemoteAddress, $Session, $Kernel)
    $script:Local = $Local
    $script:LocalTls = $LocalTls
    $script:LocalArch = $LocalArch
@ -99,6 +99,7 @@ function Set-ScriptVariables {
    $script:RecordQUIC = $RecordQUIC
    $script:RemoteAddress = $RemoteAddress
    $script:Session = $Session
+    $script:Kernel = $Kernel
    if ($null -ne $Session) {
        Invoke-Command -Session $Session -ScriptBlock {
            $ErrorActionPreference = "Stop"
@ -197,6 +198,7 @@ function Wait-ForRemote {
 function Copy-Artifacts {
    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingEmptyCatchBlock', '')]
    param ([string]$From, [string]$To)
+    Remove-PerfServices
    Invoke-TestCommand $Session -ScriptBlock {
        param ($To)
        try {
@ -258,6 +260,27 @@ function Get-ExeName {
    }
 }

+function Remove-PerfServices {
+    if ($IsWindows) {
+        Invoke-TestCommand -Session $Session -ScriptBlock {
+            if ($null -ne (Get-Service -Name "quicperf" -ErrorAction Ignore)) {
+                try {
+                    net.exe stop quicperf /y | Out-Null
+                }
+                catch {}
+                sc.exe delete quicperf /y | Out-Null
+            }
+            if ($null -ne (Get-Service -Name "msquicpriv" -ErrorAction Ignore)) {
+                try {
+                    net.exe stop msquicpriv /y | Out-Null
+                }
+                catch {}
+                sc.exe delete msquicpriv /y | Out-Null
+            }
+        }
+    }
+}
+
 function Invoke-RemoteExe {
    param ($Exe, $RunArgs)

@ -271,6 +294,10 @@ function Invoke-RemoteExe {
        return $null
    } -ArgumentList $Exe

+    if ($Kernel) {
+        $RunArgs = "--kernel $RunArgs"
+    }
+
    Write-Debug "Running Remote: $Exe $RunArgs"

    $WpaXml = $WpaStackWalkProfileXml
@ -279,7 +306,7 @@ function Invoke-RemoteExe {
    }

    return Invoke-TestCommand -Session $Session -ScriptBlock {
-        param ($Exe, $RunArgs, $BasePath, $Record, $WpaXml)
+        param ($Exe, $RunArgs, $BasePath, $Record, $WpaXml, $Kernel)
        if ($null -ne $BasePath) {
            $env:LD_LIBRARY_PATH = $BasePath
        }
@ -291,13 +318,31 @@ function Invoke-RemoteExe {
            wpr.exe -start $EtwXmlName -filemode 2> $null
        }

+        $Arch = Split-Path (Split-Path $Exe -Parent) -Leaf
+        $RootBinPath = Split-Path (Split-Path (Split-Path $Exe -Parent) -Parent) -Parent
+        $KernelDir = Join-Path $RootBinPath "winkernel" $Arch
+
+        if ($Kernel) {
+            Copy-Item (Join-Path $KernelDir "quicperf.sys") (Split-Path $Exe -Parent)
+            Copy-Item (Join-Path $KernelDir "msquicpriv.sys") (Split-Path $Exe -Parent)
+            sc.exe create "msquicpriv" type= kernel binpath= (Join-Path (Split-Path $Exe -Parent) "msquicpriv.sys") start= demand | Out-Null
+            net.exe start msquicpriv
+        }
+
        & $Exe ($RunArgs).Split(" ")

+        # Uninstall the kernel mode test driver and revert the msquic driver.
+        if ($Kernel) {
+            net.exe stop msquicpriv /y | Out-Null
+            sc.exe delete quicperf | Out-Null
+            sc.exe delete msquicpriv | Out-Null
+        }
+
        if ($Record -and $IsWindows) {
            $EtwName = $Exe + ".remote.etl"
            wpr.exe -stop $EtwName 2> $null
        }
-    } -AsJob -ArgumentList $Exe, $RunArgs, $BasePath, $Record, $WpaXml
+    } -AsJob -ArgumentList $Exe, $RunArgs, $BasePath, $Record, $WpaXml, $Kernel
 }

 function Get-RemoteFile {
@ -678,7 +723,11 @@ class TestRunDefinition {
        if ($this.VariableName -eq "Default") {
            $VarVal = ""
        }
-        $RetString = "$($this.TestName)_$($this.Remote.Platform)_$($script:RemoteArch)_$($script:RemoteTls)_$($this.VariableName)$VarVal"
+        $Platform = $this.Remote.Platform
+        if ($script:Kernel -and $this.Remote.Platform -eq "Windows") {
+            $Platform = 'Winkernel'
+        }
+        $RetString = "$($this.TestName)_$($Platform)_$($script:RemoteArch)_$($script:RemoteTls)_$($this.VariableName)$VarVal"
        if ($this.Loopback) {
            $RetString += "_Loopback"
        }
@ -686,7 +735,11 @@ class TestRunDefinition {
    }

    [string]ToTestPlatformString() {
-        $RetString = "$($this.Remote.Platform)_$($script:RemoteArch)_$($script:RemoteTls)"
+        $Platform = $this.Remote.Platform
+        if ($script:Kernel -and $this.Remote.Platform -eq "Windows") {
+            $Platform = 'Winkernel'
+        }
+        $RetString = "$($Platform)_$($script:RemoteArch)_$($script:RemoteTls)"
        return $RetString
    }
 }
--- a/scripts/performance.ps1
+++ b/scripts/performance.ps1
@ -12,6 +12,9 @@ This script runs performance tests locally for a period of time.
 .PARAMETER RemoteArch
    Specifies what the remote arch is

+.PARAMETER Kernel
+    Run the remote in kernel mode
+
 .PARAMETER LocalTls
    Specifies what local TLS provider to use

@ -83,6 +86,9 @@ param (
    [Parameter(Mandatory = $false)]
    [string]$WinRMUser = "",

+    [Parameter(Mandatory = $false)]
+    [switch]$Kernel = $false,
+
    [Parameter(Mandatory = $false)]
    [switch]$SkipDeploy = $false,

@ -111,6 +117,15 @@ param (
 Set-StrictMode -Version 'Latest'
 $PSDefaultParameterValues['*:ErrorAction'] = 'Stop'

+# Validate the the kernel switch.
+if ($Kernel -and !$IsWindows) {
+    Write-Error "-Kernel switch only supported on Windows"
+}
+
+if ($Kernel -and $PGO) {
+    Write-Error "PGO is currently not supported in kernel mode"
+}
+
 # Root directory of the project.
 $RootDir = Split-Path $PSScriptRoot -Parent

@ -193,7 +208,8 @@ Set-ScriptVariables -Local $Local `
                    -Record $Record `
                    -RecordQUIC $RecordQUIC `
                    -RemoteAddress $RemoteAddress `
-                    -Session $Session
+                    -Session $Session `
+                    -Kernel $Kernel

 $RemotePlatform = Invoke-TestCommand -Session $Session -ScriptBlock {
    if ($IsWindows) {
@ -303,7 +319,7 @@ function Invoke-Test {

    # Starting the server
    $RemoteJob = Invoke-RemoteExe -Exe $RemoteExe -RunArgs $RemoteArguments
-    $ReadyToStart = Wait-ForRemoteReady -Job $RemoteJob
+    $ReadyToStart = Wait-ForRemoteReady -Job $RemoteJob -Matcher $Test.RemoteReadyMatcher

    if (!$ReadyToStart) {
        Stop-Job -Job $RemoteJob
--- a/scripts/run-gtest.ps1
+++ b/scripts/run-gtest.ps1
@ -311,7 +311,7 @@ function Start-TestCase([String]$Name) {
        $Arguments += " --gtest_break_on_failure"
    }
    if ($Kernel -ne "") {
-        $Arguments += " --kernel"
+        $Arguments += " --kernel --privateLibrary"
    }

    # Start the test process and return some information about the test case.
@ -345,7 +345,7 @@ function Start-AllTestCases {
        $Arguments += " --gtest_break_on_failure"
    }
    if ($Kernel -ne "") {
-        $Arguments += " --kernel"
+        $Arguments += " --kernel --privateLibrary"
    }

    # Start the test process and return some information about the test case.
@ -566,12 +566,25 @@ if ($IsWindows -and $EnableAppVerifier) {

 # Install the kernel mode drivers.
 if ($Kernel -ne "") {
-    net.exe stop msquic /y | Out-Null
-    Copy-Item C:\Windows\system32\drivers\msquic.sys C:\Windows\system32\drivers\msquic.sys.old
-    Copy-Item (Join-Path $Kernel "msquictest.sys") (Split-Path $Path -Parent)
-    verifier.exe /volatile /adddriver afd.sys msquic.sys msquictest.sys netio.sys tcpip.sys /flags 0x9BB
-    sfpcopy.exe (Join-Path $Kernel "msquic.sys") C:\Windows\system32\drivers\msquic.sys
-    net.exe start msquic
+    if ($null -ne (Get-Service -Name "msquicpriv" -ErrorAction Ignore)) {
+        try {
+            net.exe stop msquicpriv /y | Out-Null
+        }
+        catch {}
+        sc.exe delete msquicpriv /y | Out-Null
+    }
+    if ($null -ne (Get-Service -Name "msquictestpriv" -ErrorAction Ignore)) {
+        try {
+            net.exe stop msquictestpriv /y | Out-Null
+        }
+        catch {}
+        sc.exe delete msquictestpriv /y | Out-Null
+    }
+    Copy-Item (Join-Path $Kernel "msquictestpriv.sys") (Split-Path $Path -Parent)
+    Copy-Item (Join-Path $Kernel "msquicpriv.sys") (Split-Path $Path -Parent)
+    sc.exe create "msquicpriv" type= kernel binpath= (Join-Path (Split-Path $Path -Parent) "msquicpriv.sys") start= demand | Out-Null
+    verifier.exe /volatile /adddriver afd.sys msquicpriv.sys msquictestpriv.sys netio.sys tcpip.sys /flags 0x9BB
+    net.exe start msquicpriv
 }

 try {
@ -673,12 +686,10 @@ try {

    # Uninstall the kernel mode test driver and revert the msquic driver.
    if ($Kernel -ne "") {
-        net.exe stop msquic /y | Out-Null
-        sc.exe delete msquictest | Out-Null
-        verifier.exe /volatile /removedriver afd.sys msquic.sys msquictest.sys netio.sys tcpip.sys
+        net.exe stop msquicpriv /y | Out-Null
+        sc.exe delete msquictestpriv | Out-Null
+        sc.exe delete msquicpriv | Out-Null
+        verifier.exe /volatile /removedriver afd.sys msquicpriv.sys msquictestpriv.sys netio.sys tcpip.sys
        verifier.exe /volatile /flags 0x0
-        sfpcopy.exe C:\Windows\system32\drivers\msquic.sys.old C:\Windows\system32\drivers\msquic.sys
-        net.exe start msquic
-        Remove-Item C:\Windows\system32\drivers\msquic.sys.old -Force
    }
 }
--- a/scripts/test.ps1
+++ b/scripts/test.ps1
@ -210,7 +210,7 @@ if (!(Test-Path $MsQuicTest)) {
    Write-Error "Build does not exist!`n `nRun the following to generate it:`n `n    $(Join-Path $RootDir "scripts" "build.ps1") -Config $Config -Arch $Arch -Tls $Tls`n"
 }
 if ($Kernel) {
-    if (!(Test-Path (Join-Path $KernelPath "msquictest.sys"))) {
+    if (!(Test-Path (Join-Path $KernelPath "msquictestpriv.sys"))) {
        Write-Error "Kernel binaries do not exist!"
    }
 }
--- a/src/bin/winkernel/msquicpriv.kernel.vcxproj
+++ b/src/bin/winkernel/msquicpriv.kernel.vcxproj
@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.props" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.props')" />
+  <Import Project="..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.props" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.props')" />
+  <Import Project="..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.props" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.props')" />
+  <Import Project="..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.props" Condition="Exists('..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.props')" />
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="driver.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="msquicpriv.rc" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="resource.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\core\core.kernel.vcxproj">
+      <Project>{E862BE76-298B-4021-8105-A3E115FA2900}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\..\platform\platform.kernel.vcxproj">
+      <Project>{5f99f713-bf5f-44eb-90fe-fea03906bba9}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{e2ddb0a8-594d-424d-9add-4ef211f7fc3f}</ProjectGuid>
+    <TemplateGuid>{1bc93793-694f-48fe-9372-81e2b05556fd}</TemplateGuid>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <MinimumVisualStudioVersion>12.0</MinimumVisualStudioVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <TargetVersion>Windows10</TargetVersion>
+    <PlatformToolset>WindowsKernelModeDriver10.0</PlatformToolset>
+    <ConfigurationType>Driver</ConfigurationType>
+    <DriverType>KMDF</DriverType>
+    <DriverTargetPlatform>Universal</DriverTargetPlatform>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <UseDebugLibraries>false</UseDebugLibraries>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings" />
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <TargetName>msquicpriv</TargetName>
+    <DebuggerFlavor>DbgengKernelDebugger</DebuggerFlavor>
+    <EnableInf2cat>false</EnableInf2cat>
+    <OutDir>$(SolutionDir)artifacts\bin\winkernel\$(Platform)_$(Configuration)_schannel\</OutDir>
+    <IntDir>$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\obj\$(ProjectName)\</IntDir>
+    <SignMode>Off</SignMode>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <WholeProgramOptimization>true</WholeProgramOptimization>
+      <AdditionalOptions Condition="'$(Platform)'!='x64'">/Gw /kernel /ZH:SHA_256</AdditionalOptions>
+      <AdditionalOptions Condition="'$(Platform)'=='x64'">/Gw /kernel /ZH:SHA_256 -d2jumptablerdata -d2epilogunwindrequirev2</AdditionalOptions>
+    </ClCompile>
+    <Link>
+      <ModuleDefinitionFile>msquicpriv.src</ModuleDefinitionFile>
+      <AdditionalDependencies>cng.lib;ksecdd.lib;msnetioid.lib;ndis.lib;netio.lib;uuid.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalOptions>/kernel /NOOPTIDATA /pdbcompress /MERGE:.gfids=GFIDS /MERGE:.orpc=.text /MERGE:_PAGE=PAGE /MERGE:_RDATA=.rdata /MERGE:_TEXT=.text /section:GFIDS,d</AdditionalOptions>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <ResourceCompile>
+      <AdditionalIncludeDirectories>..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(UM_IncludePath);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ResourceCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <PreprocessorDefinitions>QUIC_EVENTS_MANIFEST_ETW;QUIC_LOGS_MANIFEST_ETW;SECURITY_KERNEL;SECURITY_WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <PreprocessorDefinitions>QUIC_EVENTS_MANIFEST_ETW;QUIC_LOGS_MANIFEST_ETW;SECURITY_KERNEL;SECURITY_WIN32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Link>
+      <AdditionalOptions>%(AdditionalOptions) /FORCE:PGOREPRO /USEPROFILE:PGD=$(SolutionDir)src\bin\winkernel\pgo_x64\msquic.pgd</AdditionalOptions>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <FilesToPackage Include="$(TargetPath)" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <PropertyGroup>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+    <CodeAnalysisTreatWarningsAsErrors>true</CodeAnalysisTreatWarningsAsErrors>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.targets" Condition="Exists('..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.targets')" />
+    <Import Project="..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.targets" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.targets')" />
+    <Import Project="..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.targets" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.targets')" />
+    <Import Project="..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.targets" Condition="Exists('..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.targets')" />
+  </ImportGroup>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.props'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.Build.Tasks.Git.1.0.0\build\Microsoft.Build.Tasks.Git.targets'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.props'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.Common.1.0.0\build\Microsoft.SourceLink.Common.targets'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.props'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.GitHub.1.0.0\build\Microsoft.SourceLink.GitHub.targets'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.props'))" />
+    <Error Condition="!Exists('..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\Microsoft.SourceLink.AzureRepos.Git.1.0.0\build\Microsoft.SourceLink.AzureRepos.Git.targets'))" />
+  </Target>
+</Project>
--- a/src/bin/winkernel/msquicpriv.rc
+++ b/src/bin/winkernel/msquicpriv.rc
@ -0,0 +1,32 @@
+//
+//    Copyright (c) Microsoft Corporation.
+//    Licensed under the MIT License.
+//
+
+#include <windows.h>
+
+#define VER_FILETYPE                VFT_DRV
+#define VER_FILESUBTYPE             VFT2_DRV_NETWORK
+#define VER_ORIGINALFILENAME_STR    "msquicpriv.sys"
+
+#ifdef QUIC_WINDOWS_INTERNAL
+
+#include <ntverp.h>
+#define VER_FILEDESCRIPTION_STR     "Microsoft QUIC Private Test Library"
+#define VER_INTERNALNAME_STR        "msquicpriv"
+#include "common.ver"
+
+#else
+
+#include "msquic.ver"
+
+#endif // QUIC_WINDOWS_INTERNAL
+
+#include "resource.h"
+STRINGTABLE DISCARDABLE
+BEGIN
+    IDS_MSQUIC_DISPLAY_NAME         "MSQUICPRIVATE"
+    IDS_MSQUIC_DESCRIPTION_NAME     "This service implements the QUIC transport protocol, for private use."
+END
+
+#include "MsQuicEtw.rc"
--- a/src/bin/winkernel/msquicpriv.src
+++ b/src/bin/winkernel/msquicpriv.src
@ -0,0 +1,5 @@
+NAME msquicpriv.sys
+
+EXPORTS
+    MsQuicOpen
+    MsQuicClose
--- a/src/inc/quic_driver_helpers.h
+++ b/src/inc/quic_driver_helpers.h
@ -16,8 +16,9 @@ Abstract:

 #ifdef _WIN32

-#define QUIC_DRIVER_FILE_NAME  QUIC_DRIVER_NAME ".sys"
-#define QUIC_IOCTL_PATH        "\\\\.\\\\" QUIC_DRIVER_NAME
+//#define QUIC_DRIVER_FILE_NAME  QUIC_DRIVER_NAME ".sys"
+//#define QUIC_IOCTL_PATH        "\\\\.\\\\" QUIC_DRIVER_NAME
+

 class QuicDriverService {
    SC_HANDLE ScmHandle;
@ -27,7 +28,10 @@ public:
        ScmHandle(nullptr),
        ServiceHandle(nullptr) {
    }
-    bool Initialize() {
+    bool Initialize(
+        _In_z_ const char* DriverName,
+        _In_z_ const char* DependentFileNames
+        ) {
        uint32_t Error;
        ScmHandle = OpenSCManager(nullptr, nullptr, SC_MANAGER_ALL_ACCESS);
        if (ScmHandle == nullptr) {
@ -43,7 +47,7 @@ public:
        ServiceHandle =
            OpenServiceA(
                ScmHandle,
-                QUIC_DRIVER_NAME,
+                DriverName,
                SERVICE_ALL_ACCESS);
        if (ServiceHandle == nullptr) {
            QuicTraceEvent(
@ -54,27 +58,40 @@ public:
            char DriverFilePath[MAX_PATH] = {0};
            GetModuleFileNameA(NULL, DriverFilePath, MAX_PATH);
            char* PathEnd = strrchr(DriverFilePath, '\\');
-            if (!PathEnd ||
-                sizeof(DriverFilePath) - (PathEnd - DriverFilePath) < sizeof(QUIC_DRIVER_FILE_NAME)) {
+            if (!PathEnd) {
                QuicTraceEvent(
                    LibraryError,
                    "[ lib] ERROR, %s.",
-                    "Can't build " QUIC_DRIVER_FILE_NAME " full path");
+                    "Failed to get currently executing module path");
+                return false;
+            }
+            PathEnd++;
+            size_t RemainingLength = sizeof(DriverFilePath) - (PathEnd - DriverFilePath);
+            int PathResult =
+                snprintf(
+                    PathEnd,
+                    RemainingLength,
+                    "%s.sys",
+                    DriverName);
+            if (PathResult <= 0 || (size_t)PathResult > RemainingLength) {
+                QuicTraceEvent(
+                    LibraryError,
+                    "[ lib] ERROR, %s.",
+                    "Failed to create driver on disk file path");
                return false;
            }
-            memcpy(PathEnd + 1, QUIC_DRIVER_FILE_NAME, sizeof(QUIC_DRIVER_FILE_NAME));
            if (GetFileAttributesA(DriverFilePath) == INVALID_FILE_ATTRIBUTES) {
                QuicTraceEvent(
                    LibraryError,
                    "[ lib] ERROR, %s.",
-                    "Failed to find " QUIC_DRIVER_FILE_NAME);
+                    "Failed to find driver on disk");
                return false;
            }
            ServiceHandle =
                CreateServiceA(
                    ScmHandle,
-                    QUIC_DRIVER_NAME,
-                    QUIC_DRIVER_NAME,
+                    DriverName,
+                    DriverName,
                    SC_MANAGER_ALL_ACCESS,
                    SERVICE_KERNEL_DRIVER,
                    SERVICE_DEMAND_START,
@ -82,7 +99,7 @@ public:
                    DriverFilePath,
                    nullptr,
                    nullptr,
-                    "msquic\0",
+                    DependentFileNames,
                    nullptr,
                    nullptr);
            if (ServiceHandle == nullptr) {
@ -129,12 +146,27 @@ class QuicDriverClient {
 public:
    QuicDriverClient() : DeviceHandle(INVALID_HANDLE_VALUE) { }
    bool Initialize(
-        _In_ QUIC_SEC_CONFIG_PARAMS* SecConfigParams
+        _In_ QUIC_SEC_CONFIG_PARAMS* SecConfigParams,
+        _In_z_ const char* DriverName
        ) {
        uint32_t Error;
+        char IoctlPath[MAX_PATH];
+        int PathResult =
+            snprintf(
+                IoctlPath,
+                sizeof(IoctlPath),
+                "\\\\.\\\\%s",
+                DriverName);
+        if (PathResult < 0 || PathResult >= sizeof(IoctlPath)) {
+            QuicTraceEvent(
+                LibraryError,
+                "[ lib] ERRROR, %s",
+                "Creating Driver File Path failed");
+            return false;
+        }
        DeviceHandle =
            CreateFileA(
-                QUIC_IOCTL_PATH,
+                IoctlPath,
                GENERIC_READ | GENERIC_WRITE,
                0,
                nullptr,                // no SECURITY_ATTRIBUTES structure
@ -187,7 +219,7 @@ public:
        }
        QuicTraceLogVerbose(
            TestSendIoctl,
-            "[test] Sending IOCTL %u with %u bytes.",
+            "[test] Sending Write IOCTL %u with %u bytes.",
            IoGetFunctionCodeFromCtlCode(IoControlCode),
            InBufferSize);
        if (!DeviceIoControl(
@ -204,7 +236,7 @@ public:
                    LibraryErrorStatus,
                    "[ lib] ERROR, %u, %s.",
                    Error,
-                    "DeviceIoControl failed");
+                    "DeviceIoControl Write failed");
                return false;
            }
        }
@ -224,7 +256,7 @@ public:
                LibraryErrorStatus,
                "[ lib] ERROR, %u, %s.",
                Error,
-                "GetOverlappedResultEx failed");
+                "GetOverlappedResultEx Write failed");
        } else {
            Error = ERROR_SUCCESS;
        }
@ -267,7 +299,7 @@ public:
        }
        QuicTraceLogVerbose(
            TestSendIoctl,
-            "[test] Sending IOCTL %u.",
+            "[test] Sending Read IOCTL %u.",
            IoGetFunctionCodeFromCtlCode(IoControlCode));
        if (!DeviceIoControl(
                DeviceHandle,
@ -283,7 +315,7 @@ public:
                    LibraryErrorStatus,
                    "[ lib] ERROR, %u, %s.",
                    Error,
-                    "DeviceIoControl failed");
+                    "DeviceIoControl Write failed");
                return false;
            }
        }
@ -297,13 +329,15 @@ public:
            Error = GetLastError();
            if (Error == WAIT_TIMEOUT) {
                Error = ERROR_TIMEOUT;
-                CancelIoEx(DeviceHandle, &Overlapped);
+                if (CancelIoEx(DeviceHandle, &Overlapped)) {
+                    GetOverlappedResult(DeviceHandle, &Overlapped, &dwBytesReturned, true);
+                }
            } else {
                QuicTraceEvent(
                    LibraryErrorStatus,
                    "[ lib] ERROR, %u, %s.",
                    Error,
-                    "GetOverlappedResultEx failed");
+                    "GetOverlappedResultEx Read failed");
            }
        } else {
            Error = ERROR_SUCCESS;
@ -318,7 +352,14 @@ public:

 class QuicDriverService {
 public:
-    bool Initialize() { return false; }
+    bool Initialize(
+        _In_z_ const char* DriverName,
+        _In_z_ const char* DependentFileNames
+        ) {
+        UNREFERENCED_PARAMETER(DriverName);
+        UNREFERENCED_PARAMETER(DependentFileNames);
+        return false;
+        }
    void Uninitialize() { }
    bool Start() { return false; }
 };
@ -326,9 +367,11 @@ public:
 class QuicDriverClient {
 public:
    bool Initialize(
-        _In_ QUIC_SEC_CONFIG_PARAMS* SecConfigParams
+        _In_ QUIC_SEC_CONFIG_PARAMS* SecConfigParams,
+        _In_z_ const char* DriverName
    ) {
        UNREFERENCED_PARAMETER(SecConfigParams);
+        UNREFERENCED_PARAMETER(DriverName);
        return false;
    }
    void Uninitialize() { }
--- a/src/inc/quic_platform_winkernel.h
+++ b/src/inc/quic_platform_winkernel.h
@ -505,7 +505,7 @@ QuicRefDecrement(
 typedef KEVENT QUIC_EVENT;
 #define QuicEventInitialize(Event, ManualReset, InitialState) \
    KeInitializeEvent(Event, ManualReset ? NotificationEvent : SynchronizationEvent, InitialState)
-#define QuicEventUninitialize(Event)
+#define QuicEventUninitialize(Event) UNREFERENCED_PARAMETER(Event)
 #define QuicEventSet(Event) KeSetEvent(&(Event), IO_NO_INCREMENT, FALSE)
 #define QuicEventReset(Event) KeResetEvent(&(Event))
 #define QuicEventWaitForever(Event) \
--- a/src/inc/quic_platform_winuser.h
+++ b/src/inc/quic_platform_winuser.h
@ -907,6 +907,8 @@ QuicRandom(

 #define QuicSetCurrentThreadAffinityMask(Mask) SetThreadAffinityMask(GetCurrentThread(), Mask)

+#define QuicSetCurrentThreadGroupAffinityMask(Group) SetThreadGroupAffinity(GetCurrentThread(), Group, NULL)
+
 #define QuicCompartmentIdGetCurrent() GetCurrentThreadCompartmentId()
 #define QuicCompartmentIdSetCurrent(CompartmentId) \
    HRESULT_FROM_WIN32(SetCurrentThreadCompartmentId(CompartmentId))
--- a/src/perf/bin/appmain.cpp
+++ b/src/perf/bin/appmain.cpp
@ -17,25 +17,18 @@ Abstract:

 #ifdef _WIN32

-//
-// Name of the driver service for quicperf.sys.
-// Must be defined before quic_driver_helpers.h is included
-//
-#define QUIC_DRIVER_NAME   "quicperf"
-
 #include <winioctl.h>
 #include "PerfIoctls.h"
 #include "quic_driver_helpers.h"

+#define QUIC_DRIVER_NAME   "quicperf"
+
 #endif

-#include "quic_datapath.h"
+

 extern "C" _IRQL_requires_max_(PASSIVE_LEVEL) void QuicTraceRundown(void) { }

-QUIC_DATAPATH_RECEIVE_CALLBACK DatapathReceiveUserMode;
-QUIC_DATAPATH_UNREACHABLE_CALLBACK DatapathUnreachable;
-
 QUIC_STATUS
 QuicUserMain(
    _In_ int argc,
@ -45,33 +38,10 @@ QuicUserMain(
    ) {
    EventScope StopEvent {true};

-    uint8_t ServerMode = 0;
-    TryGetValue(argc, argv, "ServerMode", &ServerMode);
-
    QUIC_STATUS Status;
-    QUIC_DATAPATH* Datapath = nullptr;
-    QUIC_DATAPATH_BINDING* Binding = nullptr;
-
-    if (ServerMode) {
-        Status = QuicDataPathInitialize(0, DatapathReceiveUserMode, DatapathUnreachable, &Datapath);
-        if (QUIC_FAILED(Status)) {
-            return Status;
-        }
-
-        QuicAddr LocalAddress {AF_INET, (uint16_t)9999};
-        Status = QuicDataPathBindingCreate(Datapath, &LocalAddress.SockAddr, nullptr, &StopEvent.Handle, &Binding);
-        if (QUIC_FAILED(Status)) {
-            QuicDataPathUninitialize(Datapath);
-            return Status;
-        }
-    }

    Status = QuicMainStart(argc, argv, &StopEvent.Handle, SelfSignedConfig);
    if (QUIC_FAILED(Status)) {
-        if (ServerMode) {
-            QuicDataPathBindingDelete(Binding);
-            QuicDataPathUninitialize(Datapath);
-        }
        return Status;
    }

@ -86,11 +56,6 @@ QuicUserMain(

    Status = QuicMainStop(0);

-    if (ServerMode) {
-        QuicDataPathBindingDelete(Binding);
-        QuicDataPathUninitialize(Datapath);
-    }
-
    return Status;
 }

@ -109,6 +74,9 @@ QuicKernelMain(
    // Get total length
    //
    for (int i = 0; i < argc; ++i) {
+        if (strcmp("--kernel", argv[i]) == 0) {
+            continue;
+        }
        TotalLength += strlen(argv[i]) + 1;
    }

@ -125,18 +93,20 @@ QuicKernelMain(

    char* DataCurrent = Data;

-    QuicCopyMemory(DataCurrent, &argc, sizeof(TotalLength));
+    QuicCopyMemory(DataCurrent, &argc, sizeof(argc));

    DataCurrent += sizeof(argc);

    for (int i = 0; i < argc; ++i) {
-        size_t ArgLen = strlen(argv[i]) + 1;
+        if (strcmp("--kernel", argv[i]) == 0) {
+            continue;
+        }
+        size_t ArgLen = strlen(argv[i]);
        QuicCopyMemory(DataCurrent, argv[i], ArgLen);
        DataCurrent += ArgLen;
        DataCurrent[0] = '\0';
        ++DataCurrent;
    }
-
    QUIC_DBG_ASSERT(DataCurrent == (Data + TotalLength));

    constexpr uint32_t OutBufferSize = 1024 * 1000;
@ -149,41 +119,73 @@ QuicKernelMain(

    QuicDriverService DriverService;
    QuicDriverClient DriverClient;
-    if (!DriverService.Initialize()) {
+
+    if (!DriverService.Initialize(QUIC_DRIVER_NAME, "msquicpriv\0")) {
        printf("Failed to initialize driver service\n");
        QUIC_FREE(Data);
        return QUIC_STATUS_INVALID_STATE;
    }
-    DriverService.Start();
-
-    if (!DriverClient.Initialize(SelfSignedParams)) {
-        printf("Failed to initialize driver client\n");
+    if (!DriverService.Start()) {
+        printf("Starting Driver Service Failed\n");
+        DriverService.Uninitialize();
        QUIC_FREE(Data);
        return QUIC_STATUS_INVALID_STATE;
    }

-    if (!DriverClient.Run(IOCTL_QUIC_RUN_PERF, Data, (uint32_t)TotalLength)) {
+    if (!DriverClient.Initialize(SelfSignedParams, QUIC_DRIVER_NAME)) {
+        printf("Intializing Driver Client Failed.\n");
+        DriverService.Uninitialize();
        QUIC_FREE(Data);
+        return QUIC_STATUS_INVALID_STATE;
+    }
+
+    printf("Right before run\n");
+    uint32_t OutBufferWritten = 0;
+    bool RunSuccess = false;
+    if (!DriverClient.Run(IOCTL_QUIC_RUN_PERF, Data, (uint32_t)TotalLength, 30000)) {
+        printf("Failed To Run\n");
+        QUIC_FREE(Data);
+
+        RunSuccess =
+            DriverClient.Read(
+                IOCTL_QUIC_READ_DATA,
+                OutBuffer,
+                OutBufferSize,
+                &OutBufferWritten,
+                10000);
+        printf("OutBufferWritten %d\n", OutBufferWritten);
+        if (RunSuccess) {
+            printf("%s\n", OutBuffer);
+        } else {
+            printf("Failed to exit\n");
+        }
        QUIC_FREE(OutBuffer);
+        DriverClient.Uninitialize();
+        DriverService.Uninitialize();
        return QUIC_STATUS_INVALID_STATE;
    }
    printf("Started!\n\n");
    fflush(stdout);

-    uint32_t OutBufferWritten = 0;
-    bool RunSuccess =
+    RunSuccess =
        DriverClient.Read(
            IOCTL_QUIC_READ_DATA,
            OutBuffer,
            OutBufferSize,
-            &OutBufferWritten);
+            &OutBufferWritten,
+            240000);
    if (RunSuccess) {
-        printf("%s", OutBuffer);
+        printf("%s\n", OutBuffer);
+    } else {
+        printf("Run end failed\n");
    }

    QUIC_FREE(Data);
    QUIC_FREE(OutBuffer);

+    DriverClient.Uninitialize();
+    DriverService.Uninitialize();
+
    return RunSuccess ? QUIC_STATUS_SUCCESS : QUIC_STATUS_INTERNAL_ERROR;
 }

@ -236,6 +238,7 @@ main(

    if (TestingKernelMode) {
 #ifdef _WIN32
+        printf("Entering kernel mode main\n");
        RetVal = QuicKernelMain(argc, argv, KeyboardWait, SelfSignedParams);
 #else
        QUIC_FRE_ASSERT(FALSE);
@ -254,26 +257,3 @@ Exit:

    return RetVal;
 }
-
-void
-DatapathReceiveUserMode(
-    _In_ QUIC_DATAPATH_BINDING*,
-    _In_ void* Context,
-    _In_ QUIC_RECV_DATAGRAM*
-    )
-{
-    QUIC_EVENT* Event = static_cast<QUIC_EVENT*>(Context);
-    QuicEventSet(*Event);
-}
-
-void
-DatapathUnreachable(
-    _In_ QUIC_DATAPATH_BINDING*,
-    _In_ void*,
-    _In_ const QUIC_ADDR*
-    )
-{
-    //
-    // Do nothing, we never send
-    //
-}
--- a/src/perf/bin/drvmain.cpp
+++ b/src/perf/bin/drvmain.cpp
@ -17,8 +17,8 @@ Abstract:
 #include "drivermain.cpp.clog.h"
 #endif

-DECLARE_CONST_UNICODE_STRING(QuicPerfCtlDeviceName, L"\\Device\\quicperformance");
-DECLARE_CONST_UNICODE_STRING(QuicPerfCtlDeviceSymLink, L"\\DosDevices\\quicperformance");
+DECLARE_CONST_UNICODE_STRING(QuicPerfCtlDeviceName, L"\\Device\\quicperf");
+DECLARE_CONST_UNICODE_STRING(QuicPerfCtlDeviceSymLink, L"\\DosDevices\\quicperf");

 typedef struct QUIC_DEVICE_EXTENSION {
    EX_PUSH_LOCK Lock;
@ -33,14 +33,19 @@ WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(QUIC_DEVICE_EXTENSION, QuicPerfCtlGetDeviceCo

 typedef struct QUIC_DRIVER_CLIENT {
    LIST_ENTRY Link;
-    bool TestFailure;
-
+    PerfSelfSignedConfiguration SelfSignedConfiguration;
+    bool SelfSignedValid;
+    QUIC_EVENT StopEvent;
+    WDFREQUEST Request;
+    QUIC_THREAD Thread;
+    bool Canceled;
 } QUIC_DRIVER_CLIENT;

 WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(QUIC_DRIVER_CLIENT, QuicPerfCtlGetFileContext);

 EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL QuicPerfCtlEvtIoDeviceControl;
-EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE QuicPerfCtlEvtIoCanceled;
+EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE QuicPerfCtlEvtIoQueueCanceled;
+EVT_WDF_REQUEST_CANCEL QuicPerfCtlEvtIoCanceled;

 PAGEDX EVT_WDF_DEVICE_FILE_CREATE QuicPerfCtlEvtFileCreate;
 PAGEDX EVT_WDF_FILE_CLOSE QuicPerfCtlEvtFileClose;
@ -242,6 +247,10 @@ QuicPerfCtlInitialize(
        goto Error;
    }

+    QuicTraceLogVerbose(
+        PerfControlInitialized,
+        "[perf] Control interface initialized with %.*S", QuicPerfCtlDeviceName.Length, QuicPerfCtlDeviceName.Buffer);
+
    WDF_FILEOBJECT_CONFIG_INIT(
        &FileConfig,
        QuicPerfCtlEvtFileCreate,
@ -287,7 +296,7 @@ QuicPerfCtlInitialize(

    WDF_IO_QUEUE_CONFIG_INIT_DEFAULT_QUEUE(&QueueConfig, WdfIoQueueDispatchParallel);
    QueueConfig.EvtIoDeviceControl = QuicPerfCtlEvtIoDeviceControl;
-    QueueConfig.EvtIoCanceledOnQueue = QuicPerfCtlEvtIoCanceled;
+    QueueConfig.EvtIoCanceledOnQueue = QuicPerfCtlEvtIoQueueCanceled;

    __analysis_assume(QueueConfig.EvtIoStop != 0);
    Status =
@ -393,14 +402,16 @@ QuicPerfCtlEvtFileCreate(
        QuicPerfCtlExtension->ClientListSize++;

        QuicTraceLogInfo(
-            TestControlClientCreated,
-            "[test] Client %p created",
+            PerfControlClientCreated,
+            "[perf] Client %p created",
            Client);

        //
        // Update globals. (TODO: Add multiple device client support)
        //
        QuicPerfClient = Client;
+        InterlockedExchange((volatile LONG*)&BufferCurrent, 0);
+        QuicEventInitialize(&Client->StopEvent, true, false);
    } while (false);

    ExfReleasePushLockExclusive(&QuicPerfCtlExtension->Lock);
@ -444,10 +455,19 @@ QuicPerfCtlEvtFileCleanup(
        ExfReleasePushLockExclusive(&QuicPerfCtlExtension->Lock);

        QuicTraceLogInfo(
-            TestControlClientCleaningUp,
-            "[test] Client %p cleaning up",
+            PerfControlClientCleaningUp,
+            "[perf] Client %p cleaning up",
            Client);

+        Client->Canceled = true;
+        QuicEventSet(Client->StopEvent);
+
+        if (Client->Thread != nullptr) {
+            QuicThreadWait(&Client->Thread);
+            QuicThreadDelete(&Client->Thread);
+        }
+        QuicEventUninitialize(Client->StopEvent);
+
        //
        // Clean up globals.
        //
@ -458,50 +478,186 @@ QuicPerfCtlEvtFileCleanup(
 }

 VOID
-QuicPerfCtlEvtIoCanceled(
+QuicPerfCtlEvtIoQueueCanceled(
    _In_ WDFQUEUE /* Queue */,
    _In_ WDFREQUEST Request
    )
+{
+    QuicPerfCtlEvtIoCanceled(Request);
+}
+
+VOID
+QuicPerfCtlEvtIoCanceled(
+    _In_ WDFREQUEST Request
+    )
 {
    NTSTATUS Status;

    WDFFILEOBJECT FileObject = WdfRequestGetFileObject(Request);
    if (FileObject == nullptr) {
        Status = STATUS_DEVICE_NOT_READY;
-        goto error;
+        goto Error;
    }

    QUIC_DRIVER_CLIENT* Client = QuicPerfCtlGetFileContext(FileObject);
    if (Client == nullptr) {
        Status = STATUS_DEVICE_NOT_READY;
-        goto error;
+        goto Error;
    }

+    Client->Canceled = true;
+    QuicEventSet(Client->StopEvent);
+
    QuicTraceLogWarning(
-        TestControlClientCanceledRequest,
-        "[test] Client %p canceled request %p",
+        PerfControlClientCanceledRequest,
+        "[perf] Client %p canceled request %p",
        Client,
        Request);

-    Status = STATUS_CANCELLED;
-
-error:
-
+    return;
+Error:
    WdfRequestComplete(Request, Status);
 }

+NTSTATUS
+QuicPerfCtlSetSecurityConfig(
+    _Inout_ QUIC_DRIVER_CLIENT* Client,
+    _In_ const QUIC_CERTIFICATE_HASH* CertHash
+    )
+{
+    Client->SelfSignedConfiguration.SelfSignedSecurityHash = *CertHash;
+    Client->SelfSignedValid = true;
+    return QUIC_STATUS_SUCCESS;
+}
+
 size_t QUIC_IOCTL_BUFFER_SIZES[] =
 {
    0,
    sizeof(QUIC_CERTIFICATE_HASH),
-    0,
+    SIZE_MAX,
    0
 };

+typedef union {
+    struct {
+        int Length;
+        char Data;
+    };
+    QUIC_CERTIFICATE_HASH CertHash;
+} QUIC_IOCTL_PARAMS;
+
 static_assert(
    QUIC_PERF_MAX_IOCTL_FUNC_CODE + 1 == (sizeof(QUIC_IOCTL_BUFFER_SIZES) / sizeof(size_t)),
    "QUIC_IOCTL_BUFFER_SIZES must be kept in sync with the IOTCLs");

+//
+// Since the test is long running, we can't just wait in the Ioctl directly,
+// otherwise we can't cancel. Instead, move the wait into a separate thread
+// so the Ioctl returns into user mode.
+//
+QUIC_THREAD_CALLBACK(PerformanceWaitForStopThreadCb, Context)
+{
+    QUIC_DRIVER_CLIENT* Client = (QUIC_DRIVER_CLIENT*)Context;
+    WDFREQUEST Request = Client->Request;
+
+    char* LocalBuffer = nullptr;
+    DWORD ReturnedLength = 0;
+    QUIC_STATUS StopStatus;
+
+    StopStatus =
+        QuicMainStop(0);
+
+    if (Client->Canceled) {
+        QuicTraceLogInfo(
+            PerformanceStopCancelled,
+            "[perf] Performance Stop Cancelled");
+        WdfRequestComplete(Request, STATUS_CANCELLED);
+        return;
+    }
+
+    WdfRequestUnmarkCancelable(Request);
+
+    NTSTATUS Status =
+        WdfRequestRetrieveOutputBuffer(
+            Request,
+            (size_t)BufferCurrent + 1,
+            (void**)&LocalBuffer,
+            nullptr);
+
+    if (!NT_SUCCESS(Status)) {
+        goto Exit;
+    }
+
+    QuicCopyMemory(LocalBuffer, Buffer, BufferCurrent);
+    LocalBuffer[BufferCurrent] = '\0';
+
+    QuicTraceLogInfo(
+        PrintBufferReturn,
+        "[perf] Print Buffer %d %s\n",
+        BufferCurrent,
+        LocalBuffer);
+
+    ReturnedLength = BufferCurrent + 1;
+
+Exit:
+    WdfRequestCompleteWithInformation(
+        Request,
+        StopStatus,
+        ReturnedLength);
+}
+
+void
+QuicPerfCtlReadPrints(
+    _In_ WDFREQUEST Request,
+    _In_ QUIC_DRIVER_CLIENT* Client
+    )
+{
+    QUIC_STATUS Status;
+    QUIC_THREAD_CONFIG ThreadConfig;
+    QuicZeroMemory(&ThreadConfig, sizeof(ThreadConfig));
+    ThreadConfig.Name = "PerfWait";
+    ThreadConfig.Callback = PerformanceWaitForStopThreadCb;
+    ThreadConfig.Context = Client;
+    Client->Request = Request;
+    if (QUIC_FAILED(Status = QuicThreadCreate(&ThreadConfig, &Client->Thread))) {
+        WdfRequestCompleteWithInformation(
+            Request,
+            Status,
+            0);
+    } else {
+        WdfRequestMarkCancelable(Request, QuicPerfCtlEvtIoCanceled);
+    }
+}
+
+NTSTATUS
+QuicPerfCtlStart(
+    _In_ QUIC_DRIVER_CLIENT* Client,
+    _In_ char* Arguments,
+    _In_ int Length
+    ) {
+    char** Argv = new(std::nothrow) char* [Length];
+    if (!Argv) {
+        return QUIC_STATUS_OUT_OF_MEMORY;
+    }
+
+    Arguments += sizeof(Length);
+    for (int i = 0; i < Length; i++) {
+        Argv[i] = Arguments;
+        Arguments += strlen(Arguments);
+        Arguments++;
+    }
+
+    NTSTATUS Status =
+        QuicMainStart(
+            (int)Length,
+            Argv,
+            &Client->StopEvent,
+            &Client->SelfSignedConfiguration);
+    delete[] Argv;
+
+    return Status;
+}
+
 VOID
 QuicPerfCtlEvtIoDeviceControl(
    _In_ WDFQUEUE Queue,
@ -515,6 +671,8 @@ QuicPerfCtlEvtIoDeviceControl(
    WDFFILEOBJECT FileObject = nullptr;
    QUIC_DRIVER_CLIENT* Client = nullptr;
    ULONG FunctionCode = 0;
+    QUIC_IOCTL_PARAMS* Params = nullptr;
+    size_t Length = 0;

    if (KeGetCurrentIrql() > PASSIVE_LEVEL) {
        Status = STATUS_NOT_SUPPORTED;
@ -541,10 +699,20 @@ QuicPerfCtlEvtIoDeviceControl(
        QuicTraceEvent(
            LibraryError,
            "[ lib] ERROR, %s.",
-            "QuicTestCtlGetFileContext failed");
+            "QuicPerfCtlGetFileContext failed");
        goto Error;
    }

+    //
+    // Handle IOCTL for read
+    //
+    if (IoControlCode == IOCTL_QUIC_READ_DATA) {
+        QuicPerfCtlReadPrints(
+            Request,
+            Client);
+        return;
+    }
+
    FunctionCode = IoGetFunctionCodeFromCtlCode(IoControlCode);
    if (FunctionCode == 0 || FunctionCode > QUIC_PERF_MAX_IOCTL_FUNC_CODE) {
        Status = STATUS_NOT_IMPLEMENTED;
@ -556,10 +724,71 @@ QuicPerfCtlEvtIoDeviceControl(
        goto Error;
    }

-    // TODO Input Buffer Length
+    Status =
+        WdfRequestRetrieveInputBuffer(
+            Request,
+            0,
+            (void**)&Params,
+            &Length);
+    if (!NT_SUCCESS(Status)) {
+        QuicTraceEvent(
+            LibraryErrorStatus,
+            "[ lib] Error, %u, %s.",
+            Status,
+            "WfdRequestRetreiveInputBuffer failed");
+    } else if (Params == nullptr) {
+        QuicTraceEvent(
+            LibraryError,
+            "[ lib] ERROR, %s.",
+            "WdfRequestRetrieveInputBuffer failed to return parameter buffer");
+        Status = STATUS_INVALID_PARAMETER;
+        goto Error;
+    }

+    QuicTraceLogInfo(
+        PerfControlClientIoctl,
+        "[perf] Client %p executing write IOCTL %u",
+        Client,
+        FunctionCode);
+
+    if (IoControlCode != IOCTL_QUIC_SEC_CONFIG &&
+        !Client->SelfSignedValid) {
+        Status = STATUS_INVALID_DEVICE_STATE;
+        QuicTraceEvent(
+            LibraryError,
+            "[ lib] ERROR, %s.",
+            "Client didn't set Security Config");
+        goto Error;
+    }
+
+    switch (IoControlCode) {
+    case IOCTL_QUIC_SEC_CONFIG:
+        QUIC_FRE_ASSERT(Params != nullptr);
+        Status =
+            QuicPerfCtlSetSecurityConfig(
+                Client,
+                &Params->CertHash);
+        break;
+    case IOCTL_QUIC_RUN_PERF:
+        Status =
+            QuicPerfCtlStart(
+                Client,
+                &Params->Data,
+                Params->Length);
+        break;
+    default:
+        Status = STATUS_NOT_IMPLEMENTED;
+        break;
+    }

 Error:
+    QuicTraceLogInfo(
+        PerfControlClientIoctlComplete,
+        "[perf] Client %p completing request, 0x%x",
+        Client,
+        Status);
+
+    WdfRequestComplete(Request, Status);
    UNREFERENCED_PARAMETER(InputBufferLength);
    UNREFERENCED_PARAMETER(OutputBufferLength);
    UNREFERENCED_PARAMETER(Queue);
--- a/src/perf/bin/performance.kernel.vcxproj
+++ b/src/perf/bin/performance.kernel.vcxproj
@ -44,8 +44,8 @@
    <ProjectReference Include="..\lib\perflib.kernel.vcxproj">
      <Project>{11633785-79cc-4c7d-ab6a-aecdf29a1fa7}</Project>
    </ProjectReference>
-    <ProjectReference Include="..\..\bin\winkernel\msquic.kernel.vcxproj">
-      <Project>{C31B028C-E91C-4CF7-A8E7-F385B2AF5F85}</Project>
+    <ProjectReference Include="..\..\bin\winkernel\msquicpriv.kernel.vcxproj">
+      <Project>{e2ddb0a8-594d-424d-9add-4ef211f7fc3f}</Project>
    </ProjectReference>
  </ItemGroup>
  <PropertyGroup Label="Globals">
@ -93,7 +93,7 @@
    </ClCompile>
    <Link>
      <AdditionalLibraryDirectories>$(SolutionDir)artifacts\bin\winkernel\$(Platform)_$(Configuration)_schannel\</AdditionalLibraryDirectories>
-      <AdditionalDependencies>cng.lib;ksecdd.lib;msnetioid.lib;netio.lib;wdmsec.lib;uuid.lib;msquic.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>cng.lib;ksecdd.lib;msnetioid.lib;netio.lib;wdmsec.lib;uuid.lib;msquicpriv.lib;ndis.lib;%(AdditionalDependencies)</AdditionalDependencies>
      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
    </Link>
  </ItemDefinitionGroup>
--- a/src/perf/lib/PerfHelpers.h
+++ b/src/perf/lib/PerfHelpers.h
@ -42,7 +42,7 @@ extern const QuicApiTable* MsQuic;

 struct PerfSelfSignedConfiguration {
 #ifdef _KERNEL_MODE
-    uint8_t SelfSignedSecurityHash[20];
+    QUIC_CERTIFICATE_HASH SelfSignedSecurityHash;
 #else
    QUIC_SEC_CONFIG_PARAMS* SelfSignedParams;
 #endif
@ -65,6 +65,10 @@ QuicMainStop(
    _In_ int Timeout
    );

+extern volatile int BufferCurrent;
+constexpr int BufferLength = 40 * 1024 * 1024;
+extern char Buffer[BufferLength];
+
 inline
 int
 #ifndef _WIN32
@ -82,8 +86,31 @@ WriteOutput(
    va_end(args);
    return rval;
 #else
-    UNREFERENCED_PARAMETER(format);
-    return 0;
+    char Buf[256];
+    char* BufEnd;
+    va_list args;
+    va_start(args, format);
+    NTSTATUS Status = RtlStringCbVPrintfExA(Buf, sizeof(Buf), &BufEnd, nullptr, 0, format, args);
+    va_end(args);
+
+    if (Status == STATUS_INVALID_PARAMETER) {
+        // Write error
+        Status = RtlStringCbPrintfExA(Buf, sizeof(Buf), &BufEnd, nullptr, 0, "Invalid Format: %s\n", format);
+        if (Status != STATUS_SUCCESS) {
+            return 0;
+        }
+    }
+
+    int Length = (int)(BufEnd - Buf);
+    int End = InterlockedAdd((volatile LONG*)&BufferCurrent, Length);
+    if (End > BufferLength) {
+        return 0;
+    }
+    int Start = End - Length;
+    QuicCopyMemory(Buffer + Start, Buf, Length);
+    
+    
+    return Length;
 #endif
 }

--- a/src/perf/lib/RpsClient.cpp
+++ b/src/perf/lib/RpsClient.cpp
@ -201,7 +201,7 @@ RpsClient::Start(
            }
        }

-#ifdef _WIN32
+#if defined(_WIN32) && !defined(_KERNEL_MODE)
        uint8_t Processor = (uint8_t)(i % QuicProcActiveCount());
        const QUIC_PROCESSOR_INFO* ProcInfo = &QuicProcessorInfo[Processor];
        GROUP_AFFINITY Group = {0};
@ -318,7 +318,7 @@ RpsClient::StreamCallback(
        break;
    case QUIC_STREAM_EVENT_PEER_SEND_ABORTED:
    case QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED:
-        printf("Peer stream aborted!\n");
+        WriteOutput("Peer stream aborted!\n");
        MsQuic->StreamShutdown(
            StreamHandle,
            QUIC_STREAM_SHUTDOWN_FLAG_ABORT,
--- a/src/perf/lib/ThroughputClient.cpp
+++ b/src/perf/lib/ThroughputClient.cpp
@ -149,6 +149,8 @@ ThroughputClient::Start(
    _In_ QUIC_EVENT* StopEvnt
    ) {
    ShutdownWrapper Shutdown;
+    this->StopEvent = StopEvnt;
+
    ConnectionData* ConnData = ConnectionDataAllocator.Alloc(this);
    if (!ConnData) {
        return QUIC_STATUS_OUT_OF_MEMORY;
@ -226,17 +228,6 @@ ThroughputClient::Start(
            &LocalIpAddr);
    }

-    Status =
-        MsQuic->ConnectionStart(
-            ConnData->Connection,
-            RemoteFamily,
-            TargetData.get(),
-            Port);
-    if (QUIC_FAILED(Status)) {
-        WriteOutput("Failed ConnectionStart 0x%x\n", Status);
-        return Status;
-    }
-
    StreamData* StrmData = StreamDataAllocator.Alloc(this, ConnData->Connection);

    Status =
@ -268,7 +259,6 @@ ThroughputClient::Start(
        return Status;
    }

-    this->StopEvent = StopEvnt;
    StrmData->StartTime = QuicTimeUs64();

    if (Length == 0) {
@ -299,6 +289,17 @@ ThroughputClient::Start(
            return Status;
        }
    }
+
+    Status =
+        MsQuic->ConnectionStart(
+            ConnData->Connection,
+            RemoteFamily,
+            TargetData.get(),
+            Port);
+    if (QUIC_FAILED(Status)) {
+        WriteOutput("Failed ConnectionStart 0x%x\n", Status);
+        return Status;
+    }
    WriteOutput("Started!\n");
    Shutdown.ConnHandle = nullptr;
    return Status;
--- a/src/perf/lib/perflib.kernel.vcxproj
+++ b/src/perf/lib/perflib.kernel.vcxproj
@ -38,6 +38,8 @@
    <ClCompile Include="quicmain.cpp" />
    <ClCompile Include="ThroughputClient.cpp" />
    <ClCompile Include="ThroughputServer.cpp" />
+    <ClCompile Include="RpsClient.cpp" />
+    <ClCompile Include="RpsServer.cpp" />
  </ItemGroup>
  <ItemGroup>
    <ClInclude Include="PerfHelpers.h" />
--- a/src/perf/lib/quicmain.cpp
+++ b/src/perf/lib/quicmain.cpp
@ -20,9 +20,19 @@ Abstract:
 #endif

 const QuicApiTable* MsQuic;
+volatile int BufferCurrent;
+char Buffer[BufferLength];

 PerfBase* TestToRun;

+#include "quic_datapath.h"
+
+QUIC_DATAPATH_RECEIVE_CALLBACK DatapathReceive;
+QUIC_DATAPATH_UNREACHABLE_CALLBACK DatapathUnreachable;
+QUIC_DATAPATH* Datapath;
+QUIC_DATAPATH_BINDING* Binding;
+uint8_t ServerMode = 0;
+
 static
 void
 PrintHelp(
@ -57,13 +67,31 @@ QuicMainStart(
    const char* TestName = GetValue(argc, argv, "TestName");
    argc--; argv++;

-    uint8_t ServerMode = 0;
+    ServerMode = 0;
    if (argc != 0 && IsArg(argv[0], "ServerMode")) {
        TryGetValue(argc, argv, "ServerMode", &ServerMode);
        argc--; argv++;
    }

    QUIC_STATUS Status;
+
+    if (ServerMode) {
+        Datapath = nullptr;
+        Binding = nullptr;
+        Status = QuicDataPathInitialize(0, DatapathReceive, DatapathUnreachable, &Datapath);
+        if (QUIC_FAILED(Status)) {
+            return Status;
+        }
+
+        QuicAddr LocalAddress {AF_INET, (uint16_t)9999};
+        Status = QuicDataPathBindingCreate(Datapath, &LocalAddress.SockAddr, nullptr, StopEvent, &Binding);
+        if (QUIC_FAILED(Status)) {
+            QuicDataPathUninitialize(Datapath);
+            return Status;
+        }
+    }
+
+
    MsQuic = new(std::nothrow) QuicApiTable;
    if (MsQuic == nullptr) {
        return QUIC_STATUS_OUT_OF_MEMORY;
@ -104,7 +132,9 @@ QuicMainStart(
    }

    delete TestToRun;
+    TestToRun = nullptr;
    delete MsQuic;
+    MsQuic = nullptr;
    return Status;
 }

@ -113,11 +143,48 @@ QuicMainStop(
    _In_ int Timeout
    ) {
    if (TestToRun == nullptr) {
+        if (ServerMode) {
+            QuicDataPathBindingDelete(Binding);
+            QuicDataPathUninitialize(Datapath);
+            Datapath = nullptr;
+            Binding = nullptr;
+        }
        return QUIC_STATUS_SUCCESS;
    }

    QUIC_STATUS Status = TestToRun->Wait(Timeout);
    delete TestToRun;
    delete MsQuic;
+    if (ServerMode) {
+        QuicDataPathBindingDelete(Binding);
+        QuicDataPathUninitialize(Datapath);
+        Datapath = nullptr;
+        Binding = nullptr;
+    }
+    MsQuic = nullptr;
+    TestToRun = nullptr;
    return Status;
 }
+
+void
+DatapathReceive(
+    _In_ QUIC_DATAPATH_BINDING*,
+    _In_ void* Context,
+    _In_ QUIC_RECV_DATAGRAM*
+    )
+{
+    QUIC_EVENT* Event = static_cast<QUIC_EVENT*>(Context);
+    QuicEventSet(*Event);
+}
+
+void
+DatapathUnreachable(
+    _In_ QUIC_DATAPATH_BINDING*,
+    _In_ void*,
+    _In_ const QUIC_ADDR*
+    )
+{
+    //
+    // Do nothing, we never send
+    //
+}
--- a/src/test/MsQuicTests.h
+++ b/src/test/MsQuicTests.h
@ -303,7 +303,8 @@ LogTestFailure(
 //
 // Name of the driver service for msquictest.sys.
 //
-#define QUIC_DRIVER_NAME   "msquictest"
+#define QUIC_DRIVER_NAME            "msquictest"
+#define QUIC_DRIVER_NAME_PRIVATE    "msquictestpriv"

 #ifdef _WIN32

--- a/src/test/bin/quic_gtest.cpp
+++ b/src/test/bin/quic_gtest.cpp
@ -11,6 +11,7 @@
 #endif

 bool TestingKernelMode = false;
+bool PrivateTestLibrary = false;
 const QUIC_API_TABLE* MsQuic;
 HQUIC Registration;
 QUIC_SEC_CONFIG_PARAMS* SelfSignedCertParams;
@ -33,9 +34,18 @@ public:
                )) != nullptr);
        if (TestingKernelMode) {
            printf("Initializing for Kernel Mode tests\n");
-            ASSERT_TRUE(DriverService.Initialize());
+            const char* DriverName;
+            const char* DependentDriverNames;
+            if (PrivateTestLibrary) {
+                DriverName = QUIC_DRIVER_NAME_PRIVATE;
+                DependentDriverNames = "msquicpriv\0";
+            } else {
+                DriverName = QUIC_DRIVER_NAME;
+                DependentDriverNames = "msquic\0";
+            }
+            ASSERT_TRUE(DriverService.Initialize(DriverName, DependentDriverNames));
            ASSERT_TRUE(DriverService.Start());
-            ASSERT_TRUE(DriverClient.Initialize(SelfSignedCertParams));
+            ASSERT_TRUE(DriverClient.Initialize(SelfSignedCertParams, DriverName));
        } else {
            printf("Initializing for User Mode tests\n");
            ASSERT_TRUE(QUIC_SUCCEEDED(MsQuicOpen(&MsQuic)));
@ -1015,7 +1025,9 @@ int main(int argc, char** argv) {
    for (int i = 0; i < argc; ++i) {
        if (strcmp("--kernel", argv[i]) == 0) {
            TestingKernelMode = true;
-            break;
+        }
+        if (strcmp("--privateLibrary", argv[i]) == 0) {
+            PrivateTestLibrary = true;
        }
    }
    ::testing::AddGlobalTestEnvironment(new QuicTestEnvironment);
--- a/src/test/bin/winkernel/control.cpp
+++ b/src/test/bin/winkernel/control.cpp
@ -23,8 +23,13 @@ QUIC_SEC_CONFIG* SecurityConfig;

 QUIC_SEC_CONFIG_CREATE_COMPLETE QuicTestSecConfigCreated;

-DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceName, L"\\Device\\msquictest");
-DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceSymLink, L"\\DosDevices\\msquictest");
+#ifdef PRIVATE_LIBRARY
+DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceName, L"\\Device\\" QUIC_DRIVER_NAME_PRIVATE);
+DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceSymLink, L"\\DosDevices\\" QUIC_DRIVER_NAME_PRIVATE);
+#else
+DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceName, L"\\Device\\" QUIC_DRIVER_NAME);
+DECLARE_CONST_UNICODE_STRING(QuicTestCtlDeviceSymLink, L"\\DosDevices\\" QUIC_DRIVER_NAME);
+#endif

 typedef struct QUIC_DEVICE_EXTENSION {
    EX_PUSH_LOCK Lock;
--- a/src/test/bin/winkernel/msquictest.kernel.vcxproj
+++ b/src/test/bin/winkernel/msquictest.kernel.vcxproj
@ -121,4 +121,4 @@
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
  <ImportGroup Label="ExtensionTargets">
  </ImportGroup>
-</Project>
+</Project>
--- a/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj
+++ b/src/test/bin/winkernel/msquictestpriv.kernel.vcxproj
@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="driver.cpp" />
+    <ClCompile Include="control.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\MsQuicTests.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\platform\platform.kernel.vcxproj">
+      <Project>{5f99f713-bf5f-44eb-90fe-fea03906bba9}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\..\lib\testlib.kernel.vcxproj">
+      <Project>{A295E4D7-A50A-4771-8383-608258C23442}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\..\..\bin\winkernel\msquicpriv.kernel.vcxproj">
+      <Project>{e2ddb0a8-594d-424d-9add-4ef211f7fc3f}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{c8491270-b0be-440c-b88d-7b441a8cb67e}</ProjectGuid>
+    <TemplateGuid>{1bc93793-694f-48fe-9372-81e2b05556fd}</TemplateGuid>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <MinimumVisualStudioVersion>12.0</MinimumVisualStudioVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <TargetVersion>Windows10</TargetVersion>
+    <PlatformToolset>WindowsKernelModeDriver10.0</PlatformToolset>
+    <ConfigurationType>Driver</ConfigurationType>
+    <DriverType>KMDF</DriverType>
+    <DriverTargetPlatform>Universal</DriverTargetPlatform>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <UseDebugLibraries>false</UseDebugLibraries>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings" />
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <TargetName>msquictestpriv</TargetName>
+    <DebuggerFlavor>DbgengKernelDebugger</DebuggerFlavor>
+    <EnableInf2cat>false</EnableInf2cat>
+    <OutDir>$(SolutionDir)artifacts\bin\winkernel\$(Platform)_$(Configuration)_schannel\</OutDir>
+    <IntDir>$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\obj\$(ProjectName)\</IntDir>
+    <SignMode>Off</SignMode>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..;..\..\..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
+      <WholeProgramOptimization>true</WholeProgramOptimization>
+      <AdditionalOptions Condition="'$(Platform)'!='x64'">/Gw /kernel /ZH:SHA_256</AdditionalOptions>
+      <AdditionalOptions Condition="'$(Platform)'=='x64'">/Gw /kernel /ZH:SHA_256 -d2jumptablerdata -d2epilogunwindrequirev2</AdditionalOptions>
+    </ClCompile>
+    <Link>
+      <AdditionalLibraryDirectories>$(SolutionDir)artifacts\bin\winkernel\$(Platform)_$(Configuration)_schannel\</AdditionalLibraryDirectories>
+      <AdditionalDependencies>cng.lib;ksecdd.lib;msnetioid.lib;netio.lib;wdmsec.lib;uuid.lib;msquicpriv.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <PreprocessorDefinitions>PRIVATE_LIBRARY;QUIC_EVENTS_MANIFEST_ETW;QUIC_LOGS_MANIFEST_ETW;QUIC_DISABLE_0RTT_TESTS;SECURITY_KERNEL;SECURITY_WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <PreprocessorDefinitions>PRIVATE_LIBRARY;QUIC_EVENTS_MANIFEST_ETW;QUIC_LOGS_MANIFEST_ETW;QUIC_DISABLE_0RTT_TESTS;SECURITY_KERNEL;SECURITY_WIN32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <FilesToPackage Include="$(TargetPath)" />
+  </ItemGroup>
+  <PropertyGroup>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+    <CodeAnalysisTreatWarningsAsErrors>true</CodeAnalysisTreatWarningsAsErrors>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>