This commit is contained in:
Nick Banks 2024-09-09 06:39:50 -07:00 коммит произвёл GitHub
Родитель 8ee995eb04
Коммит add8a3aa41
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
13 изменённых файлов: 219 добавлений и 110 удалений

Просмотреть файл

@ -103,9 +103,24 @@ QuicWorkerInitialize(
} else
#endif // _KERNEL_MODE
{
const uint16_t ThreadFlags =
ExecProfile == QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME ?
CXPLAT_THREAD_FLAG_SET_AFFINITIZE : CXPLAT_THREAD_FLAG_NONE;
uint16_t ThreadFlags;
switch (ExecProfile) {
default:
case QUIC_EXECUTION_PROFILE_LOW_LATENCY:
case QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT:
ThreadFlags = CXPLAT_THREAD_FLAG_SET_IDEAL_PROC;
break;
case QUIC_EXECUTION_PROFILE_TYPE_SCAVENGER:
ThreadFlags = CXPLAT_THREAD_FLAG_NONE;
break;
case QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME:
ThreadFlags = CXPLAT_THREAD_FLAG_SET_AFFINITIZE | CXPLAT_THREAD_FLAG_HIGH_PRIORITY;
break;
}
if (MsQuicLib.ExecutionConfig && MsQuicLib.ExecutionConfig->Flags & QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY) {
ThreadFlags |= CXPLAT_THREAD_FLAG_HIGH_PRIORITY;
}
CXPLAT_THREAD_CONFIG ThreadConfig = {
ThreadFlags,

Просмотреть файл

@ -217,6 +217,7 @@ namespace Microsoft.Quic
RIO = 0x0002,
XDP = 0x0004,
NO_IDEAL_PROC = 0x0008,
HIGH_PRIORITY = 0x0010,
}
internal unsafe partial struct QUIC_EXECUTION_CONFIG

Просмотреть файл

@ -82,11 +82,11 @@ tracepoint(CLOG_PLATFORM_WINUSER_C, WindowsUserProcessorStateV3 , arg2, arg3, ar
/*----------------------------------------------------------
// Decoder Ring for ProcessorInfoV2
// [ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu
// Decoder Ring for ProcessorInfoV3
// [ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu
// QuicTraceLogInfo(
ProcessorInfoV2,
"[ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu",
ProcessorInfoV3,
"[ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu",
Proc,
(uint16_t)Group,
CxPlatProcessorInfo[Proc].Index,
@ -96,9 +96,9 @@ tracepoint(CLOG_PLATFORM_WINUSER_C, WindowsUserProcessorStateV3 , arg2, arg3, ar
// arg4 = arg4 = CxPlatProcessorInfo[Proc].Index = arg4
// arg5 = arg5 = (uint8_t)!!(CxPlatProcessorGroupInfo[Group].Mask & (1ULL << CxPlatProcessorInfo[Proc].Index)) = arg5
----------------------------------------------------------*/
#ifndef _clog_6_ARGS_TRACE_ProcessorInfoV2
#define _clog_6_ARGS_TRACE_ProcessorInfoV2(uniqueId, encoded_arg_string, arg2, arg3, arg4, arg5)\
tracepoint(CLOG_PLATFORM_WINUSER_C, ProcessorInfoV2 , arg2, arg3, arg4, arg5);\
#ifndef _clog_6_ARGS_TRACE_ProcessorInfoV3
#define _clog_6_ARGS_TRACE_ProcessorInfoV3(uniqueId, encoded_arg_string, arg2, arg3, arg4, arg5)\
tracepoint(CLOG_PLATFORM_WINUSER_C, ProcessorInfoV3 , arg2, arg3, arg4, arg5);\
#endif

Просмотреть файл

@ -65,11 +65,11 @@ TRACEPOINT_EVENT(CLOG_PLATFORM_WINUSER_C, WindowsUserProcessorStateV3,
/*----------------------------------------------------------
// Decoder Ring for ProcessorInfoV2
// [ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu
// Decoder Ring for ProcessorInfoV3
// [ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu
// QuicTraceLogInfo(
ProcessorInfoV2,
"[ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu",
ProcessorInfoV3,
"[ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu",
Proc,
(uint16_t)Group,
CxPlatProcessorInfo[Proc].Index,
@ -79,16 +79,16 @@ TRACEPOINT_EVENT(CLOG_PLATFORM_WINUSER_C, WindowsUserProcessorStateV3,
// arg4 = arg4 = CxPlatProcessorInfo[Proc].Index = arg4
// arg5 = arg5 = (uint8_t)!!(CxPlatProcessorGroupInfo[Group].Mask & (1ULL << CxPlatProcessorInfo[Proc].Index)) = arg5
----------------------------------------------------------*/
TRACEPOINT_EVENT(CLOG_PLATFORM_WINUSER_C, ProcessorInfoV2,
TRACEPOINT_EVENT(CLOG_PLATFORM_WINUSER_C, ProcessorInfoV3,
TP_ARGS(
unsigned int, arg2,
unsigned short, arg3,
unsigned int, arg4,
unsigned char, arg4,
unsigned char, arg5),
TP_FIELDS(
ctf_integer(unsigned int, arg2, arg2)
ctf_integer(unsigned short, arg3, arg3)
ctf_integer(unsigned int, arg4, arg4)
ctf_integer(unsigned char, arg4, arg4)
ctf_integer(unsigned char, arg5, arg5)
)
)

Просмотреть файл

@ -271,6 +271,7 @@ typedef enum QUIC_EXECUTION_CONFIG_FLAGS {
QUIC_EXECUTION_CONFIG_FLAG_RIO = 0x0002,
QUIC_EXECUTION_CONFIG_FLAG_XDP = 0x0004,
QUIC_EXECUTION_CONFIG_FLAG_NO_IDEAL_PROC = 0x0008,
QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY = 0x0010,
#endif
} QUIC_EXECUTION_CONFIG_FLAGS;

Просмотреть файл

@ -785,7 +785,7 @@ CxPlatEventQDequeue(
)
{
ULONG out_count = 0;
if (!GetQueuedCompletionStatusEx(*queue, events, count, &out_count, wait_time, FALSE)) return FALSE;
if (!GetQueuedCompletionStatusEx(*queue, events, count, &out_count, wait_time, FALSE)) return 0;
CXPLAT_DBG_ASSERT(out_count != 0);
CXPLAT_DBG_ASSERT(events[0].lpOverlapped != NULL || out_count == 1);
#if DEBUG
@ -1001,10 +1001,13 @@ CxPlatTimeAtOrBefore32(
//
typedef struct CXPLAT_PROCESSOR_INFO {
uint32_t Index; // Index in the current group
uint16_t Group; // The group number this processor is a part of
uint8_t Index; // Index in the current group
uint8_t PADDING; // Here to align with PROCESSOR_NUMBER struct
} CXPLAT_PROCESSOR_INFO;
CXPLAT_STATIC_ASSERT(sizeof(CXPLAT_PROCESSOR_INFO) == sizeof(PROCESSOR_NUMBER), "Size check");
typedef struct CXPLAT_PROCESSOR_GROUP_INFO {
KAFFINITY Mask; // Bit mask of active processors in the group
uint32_t Count; // Count of active processors in the group
@ -1108,90 +1111,11 @@ CXPLAT_THREAD_CALLBACK(CxPlatThreadCustomStart, CustomContext); // CXPLAT_THREAD
#endif // CXPLAT_USE_CUSTOM_THREAD_CONTEXT
inline
QUIC_STATUS
CxPlatThreadCreate(
_In_ CXPLAT_THREAD_CONFIG* Config,
_Out_ CXPLAT_THREAD* Thread
)
{
#ifdef CXPLAT_USE_CUSTOM_THREAD_CONTEXT
CXPLAT_THREAD_CUSTOM_CONTEXT* CustomContext =
CXPLAT_ALLOC_NONPAGED(sizeof(CXPLAT_THREAD_CUSTOM_CONTEXT), QUIC_POOL_CUSTOM_THREAD);
if (CustomContext == NULL) {
QuicTraceEvent(
AllocFailure,
"Allocation of '%s' failed. (%llu bytes)",
"Custom thread context",
sizeof(CXPLAT_THREAD_CUSTOM_CONTEXT));
return QUIC_STATUS_OUT_OF_MEMORY;
}
CustomContext->Callback = Config->Callback;
CustomContext->Context = Config->Context;
*Thread =
CreateThread(
NULL,
0,
CxPlatThreadCustomStart,
CustomContext,
0,
NULL);
if (*Thread == NULL) {
CXPLAT_FREE(CustomContext, QUIC_POOL_CUSTOM_THREAD);
return GetLastError();
}
#else // CXPLAT_USE_CUSTOM_THREAD_CONTEXT
*Thread =
CreateThread(
NULL,
0,
Config->Callback,
Config->Context,
0,
NULL);
if (*Thread == NULL) {
return GetLastError();
}
#endif // CXPLAT_USE_CUSTOM_THREAD_CONTEXT
CXPLAT_DBG_ASSERT(Config->IdealProcessor < CxPlatProcCount());
const CXPLAT_PROCESSOR_INFO* ProcInfo = &CxPlatProcessorInfo[Config->IdealProcessor];
GROUP_AFFINITY Group = {0};
if (Config->Flags & CXPLAT_THREAD_FLAG_SET_AFFINITIZE) {
Group.Mask = (KAFFINITY)(1ull << ProcInfo->Index); // Fixed processor
} else {
Group.Mask = CxPlatProcessorGroupInfo[ProcInfo->Group].Mask;
}
Group.Group = ProcInfo->Group;
SetThreadGroupAffinity(*Thread, &Group, NULL);
if (Config->Flags & CXPLAT_THREAD_FLAG_SET_IDEAL_PROC) {
SetThreadIdealProcessor(*Thread, ProcInfo->Index);
}
if (Config->Flags & CXPLAT_THREAD_FLAG_HIGH_PRIORITY) {
SetThreadPriority(*Thread, THREAD_PRIORITY_HIGHEST);
}
if (Config->Name) {
WCHAR WideName[64] = L"";
size_t WideNameLength;
mbstowcs_s(
&WideNameLength,
WideName,
ARRAYSIZE(WideName) - 1,
Config->Name,
_TRUNCATE);
#if defined(QUIC_RESTRICTED_BUILD)
SetThreadDescription(*Thread, WideName);
#else
THREAD_NAME_INFORMATION_PRIVATE ThreadNameInfo;
RtlInitUnicodeString(&ThreadNameInfo.ThreadName, WideName);
NtSetInformationThread(
*Thread,
ThreadNameInformationPrivate,
&ThreadNameInfo,
sizeof(ThreadNameInfo));
#endif
}
return QUIC_STATUS_SUCCESS;
}
);
#define CxPlatThreadDelete(Thread) CxPlatCloseHandle(*(Thread))
#define CxPlatThreadWait(Thread) WaitForSingleObject(*(Thread), INFINITE)
typedef uint32_t CXPLAT_THREAD_ID;

Просмотреть файл

@ -8874,6 +8874,30 @@
],
"macroName": "QuicTraceLogInfo"
},
"ProcessorInfoV3": {
"ModuleProperites": {},
"TraceString": "[ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu",
"UniqueId": "ProcessorInfoV3",
"splitArgs": [
{
"DefinationEncoding": "u",
"MacroVariableName": "arg2"
},
{
"DefinationEncoding": "hu",
"MacroVariableName": "arg3"
},
{
"DefinationEncoding": "hhu",
"MacroVariableName": "arg4"
},
{
"DefinationEncoding": "hhu",
"MacroVariableName": "arg5"
}
],
"macroName": "QuicTraceLogInfo"
},
"ProviderAttachClient": {
"ModuleProperites": {},
"TraceString": "[ nmr][%p] Client attached Ver %hu Size %hu Number %u ModuleID { %x-%x-%x-%llx }",
@ -16002,6 +16026,11 @@
"TraceID": "ProcessorInfoV2",
"EncodingString": "[ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu"
},
{
"UniquenessHash": "c224289c-70bc-0be5-bc16-ea902175e280",
"TraceID": "ProcessorInfoV3",
"EncodingString": "[ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu"
},
{
"UniquenessHash": "6509226e-b945-543f-bf07-0147bcb93ca5",
"TraceID": "ProviderAttachClient",

Просмотреть файл

@ -294,8 +294,15 @@ PerfClient::Start(
//
// Configure and start all the workers.
//
uint16_t ThreadFlags =
AffinitizeWorkers ?
(uint16_t)CXPLAT_THREAD_FLAG_SET_AFFINITIZE :
(uint16_t)CXPLAT_THREAD_FLAG_SET_IDEAL_PROC;
if (PerfDefaultHighPriority) {
ThreadFlags |= CXPLAT_THREAD_FLAG_HIGH_PRIORITY;
}
CXPLAT_THREAD_CONFIG ThreadConfig = {
(uint16_t)(AffinitizeWorkers ? CXPLAT_THREAD_FLAG_SET_AFFINITIZE : CXPLAT_THREAD_FLAG_SET_IDEAL_PROC),
ThreadFlags,
0,
"Perf Worker",
PerfClientWorker::s_WorkerThread,

Просмотреть файл

@ -52,6 +52,7 @@ extern TCP_EXECUTION_PROFILE TcpDefaultExecutionProfile;
extern QUIC_CONGESTION_CONTROL_ALGORITHM PerfDefaultCongestionControl;
extern uint8_t PerfDefaultEcnEnabled;
extern uint8_t PerfDefaultQeoAllowed;
extern uint8_t PerfDefaultHighPriority;
extern CXPLAT_DATAPATH* Datapath;

Просмотреть файл

@ -27,6 +27,7 @@ TCP_EXECUTION_PROFILE TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LAT
QUIC_CONGESTION_CONTROL_ALGORITHM PerfDefaultCongestionControl = QUIC_CONGESTION_CONTROL_ALGORITHM_CUBIC;
uint8_t PerfDefaultEcnEnabled = false;
uint8_t PerfDefaultQeoAllowed = false;
uint8_t PerfDefaultHighPriority = false;
#ifdef _KERNEL_MODE
volatile int BufferCurrent;
@ -94,11 +95,12 @@ PrintHelp(
" -pollidle:<time_us> Amount of time to poll while idle before sleeping (default: 0).\n"
" -ecn:<0/1> Enables/disables sender-side ECN support. (def:0)\n"
" -qeo:<0/1> Allows/disallowes QUIC encryption offload. (def:0)\n"
#ifndef _KERNEL_MODE
" -io:<mode> Configures a requested network IO model to be used.\n"
" - {iocp, rio, xdp, qtip, wsk, epoll, kqueue}\n"
#ifndef _KERNEL_MODE
" -cpu:<cpu_index> Specify the processor(s) to use.\n"
" -cipher:<value> Decimal value of 1 or more QUIC_ALLOWED_CIPHER_SUITE_FLAGS.\n"
" -highpri:<0/1> Configures MsQuic to run threads at high priority. (def:0)\n"
#endif // _KERNEL_MODE
"\n",
PERF_DEFAULT_PORT,
@ -137,7 +139,7 @@ QuicMainStart(
uint8_t RawConfig[QUIC_EXECUTION_CONFIG_MIN_SIZE + 256 * sizeof(uint16_t)] = {0};
QUIC_EXECUTION_CONFIG* Config = (QUIC_EXECUTION_CONFIG*)RawConfig;
Config->PollingIdleTimeoutUs = UINT32_MAX; // Default to no sleep.
Config->PollingIdleTimeoutUs = 0; // Default to no polling.
bool SetConfig = false;
#ifndef _KERNEL_MODE
@ -173,6 +175,12 @@ QuicMainStart(
} while (*CpuStr && Config->ProcessorCount < 256);
}
}
TryGetValue(argc, argv, "highpri", &PerfDefaultHighPriority);
if (PerfDefaultHighPriority) {
Config->Flags |= QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY;
SetConfig = true;
}
#endif // _KERNEL_MODE
if (TryGetValue(argc, argv, "pollidle", &Config->PollingIdleTimeoutUs)) {

Просмотреть файл

@ -211,7 +211,11 @@ bool TcpWorker::Initialize(TcpEngine* _Engine, uint16_t PartitionIndex)
}
#endif
CXPLAT_THREAD_CONFIG Config = { 0, PartitionIndex, "TcpPerfWorker", WorkerThread, this };
uint16_t ThreadFlags = CXPLAT_THREAD_FLAG_SET_IDEAL_PROC;
if (PerfDefaultHighPriority) {
ThreadFlags |= CXPLAT_THREAD_FLAG_HIGH_PRIORITY;
}
CXPLAT_THREAD_CONFIG Config = { ThreadFlags, PartitionIndex, "TcpPerfWorker", WorkerThread, this };
if (QUIC_FAILED(
CxPlatThreadCreate(
&Config,

Просмотреть файл

@ -176,10 +176,11 @@ CxPlatProcessorInfoInit(
if (Proc >= CxPlatProcessorGroupInfo[Group].Offset &&
Proc < CxPlatProcessorGroupInfo[Group].Offset + Info->Group.GroupInfo[Group].ActiveProcessorCount) {
CxPlatProcessorInfo[Proc].Group = Group;
CxPlatProcessorInfo[Proc].Index = (Proc - CxPlatProcessorGroupInfo[Group].Offset);
CXPLAT_DBG_ASSERT(Proc - CxPlatProcessorGroupInfo[Group].Offset <= UINT8_MAX);
CxPlatProcessorInfo[Proc].Index = (uint8_t)(Proc - CxPlatProcessorGroupInfo[Group].Offset);
QuicTraceLogInfo(
ProcessorInfoV2,
"[ dll] Proc[%u] Group[%hu] Index[%u] Active=%hhu",
ProcessorInfoV3,
"[ dll] Proc[%u] Group[%hu] Index[%hhu] Active=%hhu",
Proc,
(uint16_t)Group,
CxPlatProcessorInfo[Proc].Index,
@ -609,6 +610,118 @@ CxPlatGetAllocFailDenominator(
}
#endif
QUIC_STATUS
CxPlatThreadCreate(
_In_ CXPLAT_THREAD_CONFIG* Config,
_Out_ CXPLAT_THREAD* Thread
)
{
#ifdef CXPLAT_USE_CUSTOM_THREAD_CONTEXT
CXPLAT_THREAD_CUSTOM_CONTEXT* CustomContext =
CXPLAT_ALLOC_NONPAGED(sizeof(CXPLAT_THREAD_CUSTOM_CONTEXT), QUIC_POOL_CUSTOM_THREAD);
if (CustomContext == NULL) {
QuicTraceEvent(
AllocFailure,
"Allocation of '%s' failed. (%llu bytes)",
"Custom thread context",
sizeof(CXPLAT_THREAD_CUSTOM_CONTEXT));
return QUIC_STATUS_OUT_OF_MEMORY;
}
CustomContext->Callback = Config->Callback;
CustomContext->Context = Config->Context;
*Thread =
CreateThread(
NULL,
0,
CxPlatThreadCustomStart,
CustomContext,
0,
NULL);
if (*Thread == NULL) {
CXPLAT_FREE(CustomContext, QUIC_POOL_CUSTOM_THREAD);
DWORD Error = GetLastError();
QuicTraceEvent(
LibraryErrorStatus,
"[ lib] ERROR, %u, %s.",
Error,
"CreateThread");
return Error;
}
#else // CXPLAT_USE_CUSTOM_THREAD_CONTEXT
*Thread =
CreateThread(
NULL,
0,
Config->Callback,
Config->Context,
0,
NULL);
if (*Thread == NULL) {
DWORD Error = GetLastError();
QuicTraceEvent(
LibraryErrorStatus,
"[ lib] ERROR, %u, %s.",
Error,
"CreateThread");
return Error;
}
#endif // CXPLAT_USE_CUSTOM_THREAD_CONTEXT
CXPLAT_DBG_ASSERT(Config->IdealProcessor < CxPlatProcCount());
const CXPLAT_PROCESSOR_INFO* ProcInfo = &CxPlatProcessorInfo[Config->IdealProcessor];
GROUP_AFFINITY Group = {0};
if (Config->Flags & CXPLAT_THREAD_FLAG_SET_AFFINITIZE) {
Group.Mask = (KAFFINITY)(1ull << ProcInfo->Index); // Fixed processor
} else {
Group.Mask = CxPlatProcessorGroupInfo[ProcInfo->Group].Mask;
}
Group.Group = ProcInfo->Group;
if (!SetThreadGroupAffinity(*Thread, &Group, NULL)) {
QuicTraceEvent(
LibraryErrorStatus,
"[ lib] ERROR, %u, %s.",
GetLastError(),
"SetThreadGroupAffinity");
}
if (Config->Flags & CXPLAT_THREAD_FLAG_SET_IDEAL_PROC &&
!SetThreadIdealProcessorEx(*Thread, (PROCESSOR_NUMBER*)ProcInfo, NULL)) {
QuicTraceEvent(
LibraryErrorStatus,
"[ lib] ERROR, %u, %s.",
GetLastError(),
"SetThreadIdealProcessorEx");
}
if (Config->Flags & CXPLAT_THREAD_FLAG_HIGH_PRIORITY &&
!SetThreadPriority(*Thread, THREAD_PRIORITY_HIGHEST)) {
QuicTraceEvent(
LibraryErrorStatus,
"[ lib] ERROR, %u, %s.",
GetLastError(),
"SetThreadPriority");
}
if (Config->Name) {
WCHAR WideName[64] = L"";
size_t WideNameLength;
mbstowcs_s(
&WideNameLength,
WideName,
ARRAYSIZE(WideName) - 1,
Config->Name,
_TRUNCATE);
#if defined(QUIC_RESTRICTED_BUILD)
SetThreadDescription(*Thread, WideName);
#else
THREAD_NAME_INFORMATION_PRIVATE ThreadNameInfo;
RtlInitUnicodeString(&ThreadNameInfo.ThreadName, WideName);
NtSetInformationThread(
*Thread,
ThreadNameInformationPrivate,
&ThreadNameInfo,
sizeof(ThreadNameInfo));
#endif
}
return QUIC_STATUS_SUCCESS;
}
#ifdef QUIC_EVENTS_MANIFEST_ETW
_IRQL_requires_max_(PASSIVE_LEVEL)
_IRQL_requires_same_

Просмотреть файл

@ -154,12 +154,18 @@ CxPlatWorkerPoolLazyStart(
goto Error;
}
BOOLEAN SetIdealProc = !Config || !(Config->Flags & QUIC_EXECUTION_CONFIG_FLAG_NO_IDEAL_PROC);
uint16_t ThreadFlags = CXPLAT_THREAD_FLAG_SET_IDEAL_PROC;
if (Config) {
if (Config->Flags & QUIC_EXECUTION_CONFIG_FLAG_NO_IDEAL_PROC) {
ThreadFlags &= ~CXPLAT_THREAD_FLAG_SET_IDEAL_PROC; // Remove the flag
}
if (Config->Flags & QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY) {
ThreadFlags |= CXPLAT_THREAD_FLAG_HIGH_PRIORITY;
}
}
CXPLAT_THREAD_CONFIG ThreadConfig = {
SetIdealProc ?
CXPLAT_THREAD_FLAG_SET_IDEAL_PROC :
CXPLAT_THREAD_FLAG_NONE,
ThreadFlags,
0,
"cxplat_worker",
CxPlatWorkerThread,