Sync Latest - 12/10/19 (#28)

Cleans up and onboards spinquic to CI. Disabled v4 unreachable test on Windows. Adds resumption test cases. Adds resumption support for schannel. Fixes some OACR warnings.
2019-12-12 07:44:53 -08:00 · 2019-12-12 07:44:53 -08:00 · 671e49ce44
--- a/.azure/spin_linux.sh
+++ b/.azure/spin_linux.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Enable core dumps.
+ulimit -c unlimited
+mkdir artifacts/dumps
+cd artifacts/dumps
+
+# Run spinquic for a while.
+../bin/spinquic both -timeout:300000
+
+ls
--- a/.azure/spin_windows.cmd
+++ b/.azure/spin_windows.cmd
@ -0,0 +1,18 @@
+
+:: Install ProcDump if not already installed.
+PowerShell .azure\get_procdump.ps1
+
+:: Enable SChannel TLS 1.3 for client and server.
+reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Server" /v DisabledByDefault /t REG_DWORD /d 1 /f
+reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Server" /v Enabled /t REG_DWORD /d 1 /f
+reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Client" /v DisabledByDefault /t REG_DWORD /d 1 /f
+reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Client" /v Enabled /t REG_DWORD /d 1 /f
+
+:: Start ProcDump.
+mkdir artifacts\dumps
+start bld\procdump\procdump64.exe -ma -e -b -accepteula -w spinquic.exe artifacts\dumps
+
+:: Run spinquic for a while.
+artifacts\bin\Release\spinquic.exe both -timeout:300000
+
+dir artifacts\dumps
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -58,10 +58,16 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_EVENTS_STUB -DQUIC_LOGS_STUB")
    endif()

+    if(QUIC_TLS STREQUAL "openssl")
+        # OpenSSL doesn't support session resumption yet.
+        message(STATUS "Disabling session resumption support")
+        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_DISABLE_RESUMPTION")
+    endif()
+
    if(QUIC_TLS STREQUAL "openssl" OR QUIC_TLS STREQUAL "schannel")
        # OpenSSL and SChannel don't support 0-RTT yet.
        message(STATUS "Disabling 0-RTT support")
-        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_0RTT_UNSUPPORTED")
+        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_DISABLE_0RTT")
    endif()

    if(QUIC_SANITIZE_ADDRESS)
@ -86,9 +92,12 @@ else()
    endif()

    if(QUIC_TLS STREQUAL "openssl")
+        # OpenSSL doesn't support session resumption yet.
+        message(STATUS "Disabling session resumption support")
+        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_DISABLE_RESUMPTION")
        # OpenSSL doesn't support 0-RTT yet.
        message(STATUS "Disabling 0-RTT support")
-        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_0RTT_UNSUPPORTED")
+        set(QUIC_COMMON_FLAGS "${QUIC_COMMON_FLAGS} -DQUIC_DISABLE_0RTT")
    endif()

    if(QUIC_SANITIZE_ADDRESS)
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -16,6 +16,11 @@ strategy:
      imageName: 'ubuntu-latest'
      cmakeArgs: '-g ''Linux Makefiles'' -DQUIC_TLS=stub -DQUIC_ENABLE_LOGGING=off -DQUIC_SANITIZE_ADDRESS=on'
      testCmd: 'bash ./.azure/test_linux.sh AppData/WithSendArgs2.SendLarge/0'
+    linux-stub-spin:
+      platform: 'Linux-Stub-Spin'
+      imageName: 'ubuntu-latest'
+      cmakeArgs: '-g ''Linux Makefiles'' -DQUIC_TLS=stub -DQUIC_ENABLE_LOGGING=off -DQUIC_SANITIZE_ADDRESS=on'
+      testCmd: 'bash ./.azure/spin_linux.sh'
    linux-openssl:
      platform: 'Linux-OpenSSL'
      imageName: 'ubuntu-latest'
@ -25,12 +30,17 @@ strategy:
      platform: 'Windows-x64-Stub'
      imageName: 'windows-latest'
      cmakeArgs: '-g ''Visual Studio 16 2019'' -A x64 -DQUIC_TLS=stub'
-      testCmd: '.\.azure\test_windows.cmd *'
+      testCmd: '.\.azure\test_windows.cmd -*Unreachable/0'
+    windows-x64-stub-spin:
+      platform: 'Windows-x64-Stub-Spin'
+      imageName: 'windows-latest'
+      cmakeArgs: '-g ''Visual Studio 16 2019'' -A x64 -DQUIC_TLS=stub'
+      testCmd: '.\.azure\spin_windows.cmd'
    windows-x64-mitls:
      platform: 'Windows-x64-miTLS'
      imageName: 'windows-latest'
      cmakeArgs: '-g ''Visual Studio 16 2019'' -A x64 -DQUIC_TLS=mitls'
-      testCmd: '.\.azure\test_windows.cmd *'
+      testCmd: '.\.azure\test_windows.cmd -*Unreachable/0'
    windows-x64-schannel:
      platform: 'Windows-x64-Schannel'
      imageName: 'windows-latest'
@ -53,10 +63,11 @@ steps:
 - task: CMake@1
  displayName: 'CMake Build'
  inputs:
-    cmakeArgs: '--build . --config RELEASE'
+    cmakeArgs: '--build . --config DEBUG'

 - task: CmdLine@2
  displayName: 'Run Tests'
+  timeoutInMinutes: 10
  inputs:
    script: '$(testCmd)'

--- a/core/binding.c
+++ b/core/binding.c
@ -190,7 +190,7 @@ QuicBindingUninitialize(
            StatelessCtx);
    }
    QUIC_DBG_ASSERT(Binding->StatelessOperCount == 0);
-    QUIC_DBG_ASSERT(QuicHashtableGetTotalEntryCount(&Binding->StatelessOperTable) == 0);
+    QUIC_DBG_ASSERT(Binding->StatelessOperTable.NumEntries == 0);

    QuicHashFree(Binding->ResetTokenHash);
    QuicLookupUninitialize(&Binding->Lookup);
--- a/core/connection.c
+++ b/core/connection.c
@ -43,6 +43,12 @@ QuicConnInitializeCrypto(
    _In_ QUIC_CONNECTION* Connection
    );

+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+QuicConnOnShutdownComplete(
+    _In_ QUIC_CONNECTION* Connection
+    );
+
 _IRQL_requires_max_(DISPATCH_LEVEL)
 __drv_allocatesMem(Mem)
 _Must_inspect_result_
@ -451,6 +457,18 @@ QuicConnCloseHandle(
    )
 {
    QUIC_TEL_ASSERT(!Connection->State.HandleClosed);
+
+    QuicConnCloseLocally(
+        Connection,
+        QUIC_CLOSE_SILENT | QUIC_CLOSE_QUIC_STATUS,
+        (uint64_t)QUIC_STATUS_ABORTED,
+        NULL);
+
+    if (Connection->State.SendShutdownCompleteNotif) {
+        Connection->State.SendShutdownCompleteNotif = FALSE;
+        QuicConnOnShutdownComplete(Connection);
+    }
+
    Connection->State.HandleClosed = TRUE;
    Connection->ClientCallbackHandler = NULL;

@ -3332,6 +3350,7 @@ QuicConnRecvPayload(
                break; // Ignore frame if we are closed.
            }

+            QUIC_DBG_ASSERT(Connection->PathsCount <= QUIC_MAX_PATH_COUNT);
            for (uint8_t i = 0; i < Connection->PathsCount; ++i) {
                QUIC_PATH* TempPath = &Connection->Paths[i];
                if (!TempPath->IsPeerValidated &&
--- a/core/connection.h
+++ b/core/connection.h
@ -326,7 +326,7 @@ typedef struct QUIC_CONNECTION {
    //
    // Number of paths the connection is currently tracking.
    //
-    _In_range_(0, QUIC_MAX_PATH_COUNT)
+    _Field_range_(0, QUIC_MAX_PATH_COUNT)
    uint8_t PathsCount;

    //
--- a/core/crypto.c
+++ b/core/crypto.c
@ -1269,9 +1269,9 @@ QuicCryptoProcessTlsCompletion(

        QUIC_CONNECTION_EVENT Event;
        Event.Type = QUIC_CONNECTION_EVENT_CONNECTED;
-        Event.CONNECTED.EarlyDataAccepted = Crypto->TlsState.EarlyDataAccepted;
-        LogVerbose("[conn][%p] Indicating QUIC_CONNECTION_EVENT_CONNECTED (EarlyData=%hu)",
-            Connection, Event.CONNECTED.EarlyDataAccepted);
+        Event.CONNECTED.SessionResumed = Crypto->TlsState.SessionResumed;
+        LogVerbose("[conn][%p] Indicating QUIC_CONNECTION_EVENT_CONNECTED (Resume=%hu)",
+            Connection, Event.CONNECTED.SessionResumed);
        (void)QuicConnIndicateEvent(Connection, &Event);

        QuicSendSetSendFlag(&Connection->Send, QUIC_CONN_SEND_FLAG_PMTUD);
--- a/core/injection.c
+++ b/core/injection.c
@ -29,6 +29,9 @@ QuicFuzzInjectHook(
   //
   QUIC_CPUID(0x7b3c3639, 0, 0, 0, 0);

+#pragma warning(push) // Don't care about OACR warnings for test-only injection code.
+#pragma warning(disable:26007)
+
   if (MsQuicFuzzerContext.InjectCallback) {
      MsQuicFuzzerContext.InjectCallback(
         MsQuicFuzzerContext.CallbackContext,
@ -40,6 +43,8 @@ QuicFuzzInjectHook(

      Builder->Datagram->Length = Builder->DatagramLength;
   }
+
+#pragma warning(pop)
 }

 #endif // QUIC_FUZZER
--- a/core/lookup.c
+++ b/core/lookup.c
@ -55,7 +55,7 @@ QuicLookupUninitialize(
        QUIC_DBG_ASSERT(Lookup->HASH.Tables != NULL);
        for (uint8_t i = 0; i < Lookup->PartitionCount; i++) {
            QUIC_PARTITIONED_HASHTABLE* Table = &Lookup->HASH.Tables[i];
-            QUIC_DBG_ASSERT(QuicHashtableGetTotalEntryCount(&Table->Table) == 0);
+            QUIC_DBG_ASSERT(Table->Table.NumEntries == 0);
            QuicHashtableUninitialize(&Table->Table);
            QuicDispatchRwLockUninitialize(&Table->RwLock);
        }
--- a/core/stream.c
+++ b/core/stream.c
@ -240,6 +240,13 @@ QuicStreamClose(

    } else if (!Stream->Flags.ShutdownComplete) {

+        //
+        // TODO - If the stream hasn't been aborted already, then this is a
+        // fatal error for the connection. The QUIC transport cannot "just pick
+        // an error" to shutdown the stream with. It must abort the entire
+        // connection.
+        //
+
        LogWarning("[strm][%p][%llu] Closing handle without fully shutting down.",
            Stream, Stream->ID);

--- a/inc/msquic.h
+++ b/inc/msquic.h
@ -598,7 +598,7 @@ typedef struct QUIC_CONNECTION_EVENT {
    QUIC_CONNECTION_EVENT_TYPE Type;
    union {
        struct {
-            BOOLEAN EarlyDataAccepted;
+            BOOLEAN SessionResumed;
        } CONNECTED;
        struct {
            QUIC_STATUS Status;
--- a/inc/msquic_fuzz.h
+++ b/inc/msquic_fuzz.h
@ -83,7 +83,7 @@ typedef
 void
 (*QUIC_FUZZ_ENCRYPT_CALLBACK_FN) (
    _Inout_ void *CallbackContext,
-    _Inout_updates_bytes_(Length) const uint8_t * const Buffer,
+    _Inout_updates_bytes_(Length) uint8_t* Buffer,
    _In_ DWORD Length
    );

--- a/inc/quic_hashtable.h
+++ b/inc/quic_hashtable.h
@ -128,44 +128,40 @@ typedef struct QUIC_HASHTABLE {

    // Entries initialized at creation
    uint32_t Flags;
-    uint32_t Shift;

    // Entries used in bucket computation.
    uint32_t TableSize;
+#ifdef QUIC_HASHTABLE_RESIZE_SUPPORT
    uint32_t Pivot;
    uint32_t DivisorMask;
+#endif

    // Counters
    uint32_t NumEntries;
    uint32_t NonEmptyBuckets;
    uint32_t NumEnumerators;

-    // The directory. This field is for internal use only.
-    void* Directory;
+    // For internal use only.
+    union {
+        void* Directory;
+        QUIC_LIST_ENTRY* SecondLevelDir; // When TableSize <= HT_SECOND_LEVEL_DIR_MIN_SIZE
+        QUIC_LIST_ENTRY** FirstLevelDir; // When TableSize > HT_SECOND_LEVEL_DIR_MIN_SIZE
+    };

 } QUIC_HASHTABLE;

-inline
-uint32_t
-QuicHashtableGetTotalEntryCount(
-    _In_ const QUIC_HASHTABLE* HashTable
-    )
-{
-    return HashTable->NumEntries;
-}
-
 _Must_inspect_result_
-_Success_(return != 0)
+_Success_(return != FALSE)
 BOOLEAN
 QuicHashtableInitialize(
    _Inout_ _When_(NULL == *HashTable, _At_(*HashTable, __drv_allocatesMem(Mem)))
-        QUIC_HASHTABLE* *HashTable,
+        QUIC_HASHTABLE** HashTable,
    _In_ uint32_t InitialSize
    );

 inline
 _Must_inspect_result_
-_Success_(return != 0)
+_Success_(return != FALSE)
 BOOLEAN
 QuicHashtableInitializeEx(
    _Inout_ QUIC_HASHTABLE* HashTable,
@ -177,7 +173,9 @@ QuicHashtableInitializeEx(

 void
 QuicHashtableUninitialize(
-    _In_ _When_((HashTable->Flags & QUIC_HASH_ALLOCATED_HEADER), __drv_freesMem(Mem) _Post_invalid_)
+    _In_
+    _When_((HashTable->Flags & QUIC_HASH_ALLOCATED_HEADER), __drv_freesMem(Mem) _Post_invalid_)
+    _At_(HashTable->Directory, __drv_freesMem(Mem) _Post_invalid_)
        QUIC_HASHTABLE* HashTable
    );

@ -230,6 +228,20 @@ QuicHashtableEnumerateEnd(
    _Inout_ QUIC_HASHTABLE_ENUMERATOR* Enumerator
    );

+#ifdef QUIC_HASHTABLE_RESIZE_SUPPORT
+
+BOOLEAN
+QuicHashTableExpand(
+    _Inout_ QUIC_HASHTABLE* HashTable
+    );
+
+BOOLEAN
+QuicHashTableContract(
+    _Inout_ QUIC_HASHTABLE* HashTable
+    );
+
+#endif // QUIC_HASHTABLE_RESIZE_SUPPORT
+
 //
 // Simple helper hash function.
 //
--- a/inc/quic_platform_winuser.h
+++ b/inc/quic_platform_winuser.h
@ -33,6 +33,8 @@ Environment:

 #pragma warning(push) // Don't care about OACR warnings in publics
 #pragma warning(disable:26036)
+#pragma warning(disable:28252)
+#pragma warning(disable:28253)
 #include <windows.h>
 #include <winsock2.h>
 #include <iphlpapi.h>
--- a/inc/quic_sal_stub.h
+++ b/inc/quic_sal_stub.h
@ -250,8 +250,4 @@
 #define _At_(...)
 #endif

-#ifndef __ecount_opt
-#define __ecount_opt(...)
-#endif
-
 #endif // _SAL_STUB_H
--- a/inc/quic_tls.h
+++ b/inc/quic_tls.h
@ -129,7 +129,17 @@ typedef enum QUIC_TLS_RESULT_FLAGS {
 typedef struct QUIC_TLS_PROCESS_STATE {

    //
-    // Indicate the client configured 0-RTT initially.
+    // Indicates TLS has completed the handshake phase of its exchange.
+    //
+    BOOLEAN HandshakeComplete : 1;
+
+    //
+    // Indicates the TLS session was resumed from a previous connection.
+    //
+    BOOLEAN SessionResumed : 1;
+
+    //
+    // Indicates the client configured 0-RTT initially.
    //
    BOOLEAN EarlyDataAttempted : 1;

@ -139,11 +149,6 @@ typedef struct QUIC_TLS_PROCESS_STATE {
    //
    BOOLEAN EarlyDataAccepted : 1;

-    //
-    // Indicates TLS has completed the handshake phase of its exchange.
-    //
-    BOOLEAN HandshakeComplete : 1;
-
    //
    // The key that newly received data should be decrypted and read with.
    //
--- a/platform/datapath_linux.c
+++ b/platform/datapath_linux.c
@ -1639,19 +1639,22 @@ QuicDataPathRecvDatagramToRecvPacket(

 void
 QuicDataPathBindingReturnRecvDatagrams(
-    _In_opt_ QUIC_RECV_DATAGRAM* RecvPacket
+    _In_opt_ QUIC_RECV_DATAGRAM* DatagramChain
    )
 {
-    if (RecvPacket != NULL) {
 #ifdef QUIC_PLATFORM_DISPATCH_TABLE
-        PlatDispatch->DatapathBindingReturnRecvPacket(RecvPacket);
-#else
-        QUIC_DATAPATH_RECV_BLOCK* RecvBlock =
-            QUIC_CONTAINING_RECORD(RecvPacket, QUIC_DATAPATH_RECV_BLOCK, RecvPacket);
-
-        QuicPoolFree(RecvBlock->OwningPool, RecvBlock);
-#endif
+    if (DatagramChain != NULL) {
+        PlatDispatch->DatapathBindingReturnRecvPacket(DatagramChain);
    }
+#else
+    QUIC_RECV_DATAGRAM* Datagram;
+    while ((Datagram = DatagramChain) != NULL) {
+        DatagramChain = DatagramChain->Next;
+        QUIC_DATAPATH_RECV_BLOCK* RecvBlock =
+            QUIC_CONTAINING_RECORD(Datagram, QUIC_DATAPATH_RECV_BLOCK, RecvPacket);
+        QuicPoolFree(RecvBlock->OwningPool, RecvBlock);
+    }
+#endif
 }

 QUIC_DATAPATH_SEND_CONTEXT*
--- a/platform/datapath_winuser.c
+++ b/platform/datapath_winuser.c
@ -2407,6 +2407,7 @@ QuicDataPathBindingSendFromTo(
        WSAMhdr.Control.len += WSA_CMSG_SPACE(sizeof(*SegmentSize));

        CMsg = WSA_CMSG_NXTHDR(&WSAMhdr, CMsg);
+        QUIC_DBG_ASSERT(CMsg != NULL);
        CMsg->cmsg_level = IPPROTO_UDP;
        CMsg->cmsg_type = UDP_SEND_MSG_SIZE;
        CMsg->cmsg_len = WSA_CMSG_LEN(sizeof(*SegmentSize));
--- a/platform/hashtable.c
+++ b/platform/hashtable.c
--- a/platform/inline.c
+++ b/platform/inline.c
@ -116,11 +116,6 @@ InterlockedIncrement64(
    _Inout_ _Interlocked_operand_ int64_t volatile *Addend
    );

-uint32_t
-QuicHashtableGetTotalEntryCount(
-    _In_ const QUIC_HASHTABLE* Table
-    );
-
 _Must_inspect_result_
 _Success_(return != 0)
 BOOLEAN
--- a/platform/selfsign_capi.c
+++ b/platform/selfsign_capi.c
@ -571,6 +571,7 @@ CreateSelfSignedCertificate(
    KeyProvInfo.rgProvParam         = NULL;
    KeyProvInfo.dwKeySpec           = AT_KEYEXCHANGE;

+#pragma prefast(suppress: __WARNING_33088, "Test Only Usage of Self-Signed Certs.")
    CertContext =
        CertCreateSelfSignCertificate(
            Key,
--- a/platform/tls_mitls.c
+++ b/platform/tls_mitls.c
@ -1291,6 +1291,7 @@ QuicTlsProcessDataComplete(
                    ResultFlags |= QUIC_TLS_RESULT_EARLY_DATA_ACCEPT;
                    LogVerbose("[ tls][%p] Early data accepted", TlsContext);
                    TlsContext->TlsKeySchedule = 1; // 0-RTT allowed.
+                    State->SessionResumed = TRUE;
                    State->EarlyDataAttempted = TRUE;
                    State->EarlyDataAccepted = TRUE;
                } else {
@ -1339,6 +1340,7 @@ QuicTlsProcessDataComplete(
                            // rejected event/flag from miTLS.
                            //
                            ResultFlags |= QUIC_TLS_RESULT_EARLY_DATA_ACCEPT;
+                            State->SessionResumed = TRUE;
                            State->EarlyDataAccepted = TRUE;
                            LogVerbose("[ tls][%p] Early data accepted", TlsContext);
                        }
--- a/platform/tls_schannel.c
+++ b/platform/tls_schannel.c
@ -1557,8 +1557,23 @@ QuicTlsWriteDataToSchannel(
                // signal in the upcall which ALPN(s) were negotiated.
            }

-            LogInfo("[ tls][%p][%c] Handshake complete.",
-                TlsContext, GetTlsIdentifier(TlsContext));
+            SecPkgContext_SessionInfo SessionInfo;
+            SecStatus =
+                QueryContextAttributesW(
+                    &TlsContext->SchannelContext,
+                    SECPKG_ATTR_SESSION_INFO,
+                    &SessionInfo);
+            if (SecStatus != SEC_E_OK) {
+                EventWriteQuicTlsErrorStatus(TlsContext->Connection, SecStatus, "query session info");
+                Result |= QUIC_TLS_RESULT_ERROR;
+                break;
+            }
+            if (SessionInfo.dwFlags & SSL_SESSION_RECONNECT) {
+                State->SessionResumed = TRUE;
+            }
+
+            LogInfo("[ tls][%p][%c] Handshake complete (resume=%hu).",
+                TlsContext, GetTlsIdentifier(TlsContext), State->SessionResumed);
            State->HandshakeComplete = TRUE;
            Result |= QUIC_TLS_RESULT_COMPLETE;
        }
@ -2511,6 +2526,7 @@ QuicEncrypt(

 #ifdef QUIC_FUZZER
    if (MsQuicFuzzerContext.EncryptCallback) {
+#pragma prefast(suppress: __WARNING_26000, "Auth Data and Buffer are always contiguous.")
        MsQuicFuzzerContext.EncryptCallback(
            MsQuicFuzzerContext.CallbackContext,
            AuthData,
--- a/platform/tls_stub.c
+++ b/platform/tls_stub.c
@ -657,8 +657,9 @@ QuicTlsServerProcess(
                break; // Ignored in this code.
            }
            case TlsExt_SessionTicket: {
+                State->SessionResumed = TRUE; // TODO - Support tickets
                State->EarlyDataAttempted = TRUE;
-                State->EarlyDataAccepted = TRUE; // TODO - Support tickets
+                State->EarlyDataAccepted = TRUE;
                break;
            }
            case TlsExt_QuicTransportParameters: {
@ -704,8 +705,6 @@ QuicTlsServerProcess(
        TlsWriteUint24(ServerMessage->Length, MessageLength - 4);
        ServerMessage->Type = QUIC_TLS_MESSAGE_SERVER_INITIAL;
        ServerMessage->SERVER_INITIAL.EarlyDataAccepted = State->EarlyDataAccepted;
-        State->EarlyDataAttempted = State->EarlyDataAttempted;
-        State->EarlyDataAccepted = State->EarlyDataAccepted;

        State->BufferLength = MessageLength;
        State->BufferTotalLength = MessageLength;
@ -924,6 +923,7 @@ QuicTlsClientProcess(
        if (ServerMessage->Type == QUIC_TLS_MESSAGE_SERVER_INITIAL) {

            if (State->EarlyDataAttempted) {
+                State->SessionResumed = ServerMessage->SERVER_INITIAL.EarlyDataAccepted;
                State->EarlyDataAccepted = ServerMessage->SERVER_INITIAL.EarlyDataAccepted;
                if (!ServerMessage->SERVER_INITIAL.EarlyDataAccepted) {
                    *ResultFlags |= QUIC_TLS_RESULT_EARLY_DATA_REJECT;
--- a/test/MsQuicTests.h
+++ b/test/MsQuicTests.h
@ -70,7 +70,8 @@ QuicTestConnect(
    _In_ bool ChangeMaxStreamID,
    _In_ bool MultipleALPNs,
    _In_ bool AsyncSecConfig,
-    _In_ bool MultiPacketClientInitial
+    _In_ bool MultiPacketClientInitial,
+    _In_ bool SessionResumption
    );

 void
@ -325,6 +326,7 @@ typedef struct {
    UINT8 MultipleALPNs;
    UINT8 AsyncSecConfig;
    UINT8 MultiPacketClientInitial;
+    UINT8 SessionResumption;
 } QUIC_RUN_CONNECT_PARAMS;

 #pragma pack(pop)
--- a/test/bin/quic_gtest.cpp
+++ b/test/bin/quic_gtest.cpp
@ -223,7 +223,8 @@ TEST_P(WithHandshakeArgs1, Connect) {
        false,  // ChangeMaxStreamID
        GetParam().MultipleALPNs,
        false,  // AsyncSecConfig
-        GetParam().MultiPacketClientInitial);
+        GetParam().MultiPacketClientInitial,
+        GetParam().SessionResumption);
 }

 TEST_P(WithHandshakeArgs2, OldVersion) {
@ -236,7 +237,8 @@ TEST_P(WithHandshakeArgs2, OldVersion) {
        false,  // ChangeMaxStreamID
        false,  // MultipleALPNs
        false,  // AsyncSecConfig
-        false); // MultiPacketClientInitial
+        false,  // MultiPacketClientInitial
+        false); // SessionResumption
 }

 TEST_P(WithFamilyArgs, VersionNegotiation) {
@ -254,7 +256,8 @@ TEST_P(WithFamilyArgs, Rebind) {
        false,  // ChangeMaxStreamID
        false,  // MultipleALPNs
        false,  // AsyncSecConfig
-        false); // MultiPacketClientInitial
+        false,  // MultiPacketClientInitial
+        false); // SessionResumption
 }

 TEST_P(WithFamilyArgs, ChangeMaxStreamIDs) {
@ -267,7 +270,8 @@ TEST_P(WithFamilyArgs, ChangeMaxStreamIDs) {
        true,   // ChangeMaxStreamID
        false,  // MultipleALPNs
        false,  // AsyncSecConfig
-        false); // MultiPacketClientInitial
+        false,  // MultiPacketClientInitial
+        false); // SessionResumption
 }

 TEST_P(WithHandshakeArgs1, AsyncSecurityConfig) {
@ -280,7 +284,8 @@ TEST_P(WithHandshakeArgs1, AsyncSecurityConfig) {
        false,  // ChangeMaxStreamID
        GetParam().MultipleALPNs,
        true,   // AsyncSecConfig
-        false); // MultiPacketClientInitial
+        false,  // MultiPacketClientInitial
+        false); // SessionResumption
 }

 TEST_P(WithFamilyArgs, Unreachable) {
@ -321,7 +326,7 @@ TEST_P(WithSendArgs1, Send) {
        GetParam().ServerInitiatedStreams);
 }

-#ifndef QUIC_0RTT_UNSUPPORTED
+#ifndef QUIC_DISABLE_0RTT
 // TODO - Send0Rtt
 // TODO - Reject0Rtt
 #endif
--- a/test/bin/quic_gtest.h
+++ b/test/bin/quic_gtest.h
@ -42,13 +42,19 @@ struct HandshakeArgs1 {
    bool ServerStatelessRetry;
    bool MultipleALPNs;
    bool MultiPacketClientInitial;
+    bool SessionResumption;
    static ::std::vector<HandshakeArgs1> Generate() {
        ::std::vector<HandshakeArgs1> list;
        for (int Family : { 4, 6})
        for (bool ServerStatelessRetry : { false, true })
        for (bool MultipleALPNs : { false, true })
        for (bool MultiPacketClientInitial : { false, true })
-            list.push_back({ Family, ServerStatelessRetry, MultipleALPNs, MultiPacketClientInitial });
+#ifdef QUIC_DISABLE_RESUMPTION
+        for (bool SessionResumption : { false })
+#else
+        for (bool SessionResumption : { false, true })
+#endif
+            list.push_back({ Family, ServerStatelessRetry, MultipleALPNs, MultiPacketClientInitial, SessionResumption });
        return list;
    }
 };
@ -58,7 +64,8 @@ std::ostream& operator << (std::ostream& o, const HandshakeArgs1& args) {
        (args.Family == 4 ? "v4" : "v6") << "/" <<
        (args.ServerStatelessRetry ? "Retry" : "NoRetry") << "/" <<
        (args.MultipleALPNs ? "MultipleALPNs" : "SingleALPN") << "/" <<
-        (args.MultiPacketClientInitial ? "MultipleInitials" : "SingleInitial");
+        (args.MultiPacketClientInitial ? "MultipleInitials" : "SingleInitial") << "/" <<
+        (args.SessionResumption ? "Resume" : "NoResume");
 }

 class WithHandshakeArgs1 : public testing::Test,
@ -132,7 +139,7 @@ struct SendArgs2 {
        ::std::vector<SendArgs2> list;
        for (int Family : { 4, 6 })
        for (bool UseSendBuffer : { false, true })
-#ifndef QUIC_0RTT_UNSUPPORTED
+#ifndef QUIC_DISABLE_0RTT
        for (bool UseZeroRtt : { false, true })
 #else
        for (bool UseZeroRtt : { false })
--- a/test/lib/ApiTest.cpp
+++ b/test/lib/ApiTest.cpp
@ -174,7 +174,7 @@ void QuicTestValidateSession()
            sizeof(TicketKey) - 1,
            TicketKey));

-#ifndef QUIC_0RTT_UNSUPPORTED
+#ifndef QUIC_DISABLE_0RTT
    //
    // Valid 0-RTT ticket encryption key.
    //
@ -547,7 +547,7 @@ ListenerAcceptCallback(
    )
 {
    TestConnection** NewConnection = (TestConnection**)Listener->Context;
-    *NewConnection = new TestConnection(ConnectionHandle, ConnectionIgnoreStreamCallback, true, true);
+    *NewConnection = new TestConnection(ConnectionHandle, ConnectionIgnoreStreamCallback, true);
    if (*NewConnection == nullptr || !(*NewConnection)->IsValid()) {
        TEST_FAILURE("Failed to accept new TestConnection.");
        delete *NewConnection;
@ -593,7 +593,7 @@ void QuicTestValidateStream(bool Connect)
    // Force the Client, Server, and Listener to clean up before the Session and Registration.
    //
    {
-        TestConnection Client(TestSession, ConnectionIgnoreStreamCallback, false, false);
+        TestConnection Client(TestSession, ConnectionIgnoreStreamCallback, false);
        TEST_TRUE(Client.IsValid());

        UniquePtr<TestConnection> Server;
--- a/test/lib/QuicTest.cpp
+++ b/test/lib/QuicTest.cpp
@ -252,6 +252,15 @@ ListenerAcceptConnection(
    )
 {
    ServerAcceptContext* AcceptContext = (ServerAcceptContext*)Listener->Context;
+    if (AcceptContext == nullptr) { // Prime Resumption scenario.
+        auto NewConnection = new TestConnection(ConnectionHandle, ConnectionDoNothingCallback, true, true);
+        if (NewConnection == nullptr || !NewConnection->IsValid()) {
+            TEST_FAILURE("Failed to accept new TestConnection.");
+            delete NewConnection;
+            MsQuic->ConnectionClose(ConnectionHandle);
+        }
+        return;
+    }
    if (*AcceptContext->NewConnection != nullptr) { // Retry scenario.
        delete *AcceptContext->NewConnection;
        *AcceptContext->NewConnection = nullptr;
@ -324,7 +333,8 @@ QuicTestConnect(
    _In_ bool ChangeMaxStreamID,
    _In_ bool MultipleALPNs,
    _In_ bool AsyncSecConfig,
-    _In_ bool MultiPacketClientInitial
+    _In_ bool MultiPacketClientInitial,
+    _In_ bool SessionResumption
    )
 {
    MsQuicSession Session;
@ -352,6 +362,37 @@ QuicTestConnect(
            TEST_QUIC_SUCCEEDED(Listener2.Start(&ServerLocalAddr.SockAddr));
        }

+        if (SessionResumption) {
+            TestScopeLogger logScope("PrimeResumption");
+            {
+                TestConnection Client(
+                    MultipleALPNs ? Session2.Handle : Session.Handle,
+                    ConnectionDoNothingCallback,
+                    false);
+                TEST_TRUE(Client.IsValid());
+                #if QUIC_TEST_DISABLE_DNS
+                QuicAddr RemoteAddr(QuicAddrFamily, true);
+                TEST_QUIC_SUCCEEDED(Client.SetRemoteAddr(RemoteAddr));
+                #endif
+                TEST_QUIC_SUCCEEDED(
+                    Client.Start(
+                        QuicAddrFamily,
+                        QUIC_LOCALHOST_FOR_AF(QuicAddrFamily),
+                        QuicAddrGetPort(&ServerLocalAddr.SockAddr)));
+                if (!Client.WaitForConnectionComplete()) {
+                    return;
+                }
+                TEST_TRUE(Client.GetIsConnected());
+                if (!Client.WaitForZeroRttTicket()) {
+                    return;
+                }
+                Client.Shutdown(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, QUIC_TEST_NO_ERROR);
+                if (!Client.WaitForShutdownComplete()) {
+                    return;
+                }
+            }
+        }
+
        {
            UniquePtr<TestConnection> Server;
            ServerAcceptContext ServerAcceptCtx((TestConnection**)&Server);
@ -375,6 +416,10 @@ QuicTestConnect(
                        Client.SetTestTransportParameter(&TpHelper));
                }

+                if (SessionResumption) {
+                    Client.SetExpectedResumed(true);
+                }
+
                #if QUIC_TEST_DISABLE_DNS
                QuicAddr RemoteAddr(QuicAddrFamily, true);
                TEST_QUIC_SUCCEEDED(Client.SetRemoteAddr(RemoteAddr));
@ -414,6 +459,11 @@ QuicTestConnect(
                    TEST_EQUAL(Server->GetQuicVersion(), LATEST_SUPPORTED_VERSION);
                }

+                if (SessionResumption) {
+                    TEST_TRUE(Client.GetResumed());
+                    TEST_TRUE(Server->GetResumed());
+                }
+
                TEST_EQUAL(
                    Server->GetPeerBidiStreamCount(),
                    Client.GetLocalBidiStreamCount());
@ -425,10 +475,21 @@ QuicTestConnect(
                    TEST_QUIC_SUCCEEDED(Client.GetLocalAddr(NewLocalAddr));
                    TEST_FALSE(Client.GetIsShutdown());

-                    QuicAddr ServerRemoteAddr;
-                    TEST_QUIC_SUCCEEDED(Server->GetRemoteAddr(ServerRemoteAddr));
-                    TEST_TRUE(Server->GetPeerAddrChanged());
-                    TEST_TRUE(QuicAddrCompare(&NewLocalAddr.SockAddr, &ServerRemoteAddr.SockAddr));
+                    bool ServerAddressUpdated = false;
+                    uint32_t Try = 0;
+                    do {
+                        if (Try != 0) {
+                            QuicSleep(200);
+                        }
+                        QuicAddr ServerRemoteAddr;
+                        TEST_QUIC_SUCCEEDED(Server->GetRemoteAddr(ServerRemoteAddr));
+                        if (Server->GetPeerAddrChanged() &&
+                            QuicAddrCompare(&NewLocalAddr.SockAddr, &ServerRemoteAddr.SockAddr)) {
+                            ServerAddressUpdated = true;
+                            break;
+                        }
+                    } while (++Try <= 3);
+                    TEST_TRUE(ServerAddressUpdated);
                }

                if (ChangeMaxStreamID) {
--- a/test/lib/TestConnection.cpp
+++ b/test/lib/TestConnection.cpp
@ -646,7 +646,7 @@ TestConnection::HandleConnectionEvent(

    case QUIC_CONNECTION_EVENT_CONNECTED:
        IsConnected = true;
-        Resumed = Event->CONNECTED.EarlyDataAccepted != FALSE;
+        Resumed = Event->CONNECTED.SessionResumed != FALSE;
        if (!Resumed && ExpectedResumed) {
            TEST_FAILURE("Resumption was expected!");
        }
--- a/tools/interop/interop.cpp
+++ b/tools/interop/interop.cpp
@ -302,7 +302,7 @@ public:
            }
            QuicSleep(100);
        }
-        return TryCount < 20;
+        return true; // TryCount < 20;
    }
    bool ForceCidUpdate() {
        return
@ -388,7 +388,7 @@ private:
        switch (Event->Type) {
        case QUIC_CONNECTION_EVENT_CONNECTED:
            pThis->Connected = true;
-            if (Event->CONNECTED.EarlyDataAccepted) {
+            if (Event->CONNECTED.SessionResumed) {
                pThis->Resumed = true;
            }
            QuicEventSet(pThis->ConnectionComplete);
@ -546,6 +546,8 @@ RunInteropTest(
                }
            } else if (Feature == ConnectionClose) {
                Success = Connection.Shutdown();
+            } else if (Feature == Resumption) {
+                Success = Connection.Resumed;
            } else {
                Success = true;
            }
--- a/tools/spin/spinquic.cpp
+++ b/tools/spin/spinquic.cpp
--- a/tools/spin/spinquic.h
+++ b/tools/spin/spinquic.h
@ -1,29 +0,0 @@
-/*++
-
-  Copyright (c) Microsoft Corporation.
-  Licensed under the MIT License.
-
--*/
-
-typedef enum {
-    SpinQuicAPICallCreateConnection = 0,
-    SpinQuicAPICallStartConnection,
-    SpinQuicAPICallShutdownConnection,
-    SpinQuicAPICallCloseConnection,
-    SpinQuicAPICallStreamOpen,
-    SpinQuicAPICallStreamStart,
-    SpinQuicAPICallStreamSend,
-    SpinQuicAPICallStreamShutdown,
-    SpinQuicAPICallStreamClose,
-    SpinQuicAPICallSetParamSession,
-    SpinQuicAPICallSetParamConnection,
-    SpinQuicAPICallCount, // Always the last element
-} SpinQuicAPICall;
-
-#define SQ_ASSERT(x) do { \
-    int __tmp_ret = (x); \
-    if (!__tmp_ret) { \
-       printf("%s:%d %s != TRUE", __FILE__, __LINE__, #x); \
-       exit(10); \
-    } \
-} while (0);