Bug 939318 - Find and close HTTP connections without traffic after network change. r=mcmanus

SPDY/http2 connections get a ping and be allowed N seconds to respond.

Active HTTP connections will be allowed N seconds to get traffic, if they
don't afer N seconds they get closed to avoid risking stalled transfers.

N is 5 by default: pref is "network.http.network-changed.timeout"

modules/libpref/init/all.js

@@ -1166,6 +1166,11 @@ pref("network.http.connection-retry-timeout", 250);
 // to give up if the OS does not give up first
 pref("network.http.connection-timeout", 90);
 
+// The number of seconds to allow active connections to prove that they have
+// traffic before considered stalled, after a network change has been detected
+// and signalled.
+pref("network.http.network-changed.timeout", 5);
+
 // The maximum number of current global half open sockets allowable
 // when starting a new speculative connection.
 pref("network.http.speculative-parallel-limit", 6);

netwerk/protocol/http/ASpdySession.h

@@ -39,6 +39,8 @@ public:
     return true;
   }
 
+  virtual void SendPing() = 0;
+
   const static uint32_t kSendingChunkSize = 4095;
   const static uint32_t kTCPSendBufferSize = 131072;
 

netwerk/protocol/http/Http2Session.cpp

@@ -98,6 +98,7 @@ Http2Session::Http2Session(nsISocketTransport *aSocketTransport)
   , mOutputQueueSent(0)
   , mLastReadEpoch(PR_IntervalNow())
   , mPingSentEpoch(0)
+  , mPreviousUsed(false)
   , mWaitingForSettingsAck(false)
   , mGoAwayOnPush(false)
 {
@@ -293,8 +294,14 @@ Http2Session::ReadTimeoutTick(PRIntervalTime now)
 
   if ((now - mLastReadEpoch) < mPingThreshold) {
     // recent activity means ping is not an issue
-    if (mPingSentEpoch)
+    if (mPingSentEpoch) {
       mPingSentEpoch = 0;
+      if (mPreviousUsed) {
+        // restore the former value
+        mPingThreshold = mPreviousPingThreshold;
+        mPreviousUsed = false;
+      }
+    }
 
     return PR_IntervalToSeconds(mPingThreshold) -
       PR_IntervalToSeconds(now - mLastReadEpoch);
@@ -314,8 +321,9 @@ Http2Session::ReadTimeoutTick(PRIntervalTime now)
   LOG3(("Http2Session::ReadTimeoutTick %p generating ping\n", this));
 
   mPingSentEpoch = PR_IntervalNow();
-  if (!mPingSentEpoch)
+  if (!mPingSentEpoch) {
     mPingSentEpoch = 1; // avoid the 0 sentinel value
+  }
   GeneratePing(false);
   ResumeRecv(); // read the ping reply
 
@@ -3293,5 +3301,29 @@ Http2Session::PushBack(const char *buf, uint32_t len)
   return mConnection->PushBack(buf, len);
 }
 
+void
+Http2Session::SendPing()
+{
+  MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
+
+  if (mPreviousUsed) {
+    // alredy in progress, get out
+    return;
+  }
+
+  mPingSentEpoch = PR_IntervalNow();
+  if (!mPingSentEpoch) {
+    mPingSentEpoch = 1; // avoid the 0 sentinel value
+  }
+  if (!mPingThreshold ||
+      (mPingThreshold > gHttpHandler->NetworkChangedTimeout())) {
+    mPreviousPingThreshold = mPingThreshold;
+    mPreviousUsed = true;
+    mPingThreshold = gHttpHandler->NetworkChangedTimeout();
+  }
+  GeneratePing(false);
+  ResumeRecv();
+}
+
 } // namespace mozilla::net
 } // namespace mozilla

netwerk/protocol/http/Http2Session.h

@@ -212,6 +212,8 @@ public:
   int64_t ServerSessionWindow() { return mServerSessionWindow; }
   void DecrementServerSessionWindow (uint32_t bytes) { mServerSessionWindow -= bytes; }
 
+  void SendPing() MOZ_OVERRIDE;
+
 private:
 
   // These internal states do not correspond to the states of the HTTP/2 specification
@@ -441,6 +443,9 @@ private:
   PRIntervalTime       mLastDataReadEpoch; // used for IdleTime()
   PRIntervalTime       mPingSentEpoch;
 
+  PRIntervalTime       mPreviousPingThreshold; // backup for the former value
+  bool                 mPreviousUsed;          // true when backup is used
+
   // used as a temporary buffer while enumerating the stream hash during GoAway
   nsDeque  mGoAwayStreamsToRestart;
 

netwerk/protocol/http/SpdySession3.cpp

@@ -73,6 +73,7 @@ SpdySession3::SpdySession3(nsISocketTransport *aSocketTransport)
   , mLastReadEpoch(PR_IntervalNow())
   , mPingSentEpoch(0)
   , mNextPingID(1)
+  , mPreviousUsed(false)
 {
   MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
 
@@ -224,8 +225,14 @@ SpdySession3::ReadTimeoutTick(PRIntervalTime now)
 
     if ((now - mLastReadEpoch) < mPingThreshold) {
       // recent activity means ping is not an issue
-      if (mPingSentEpoch)
+      if (mPingSentEpoch) {
         mPingSentEpoch = 0;
+        if (mPreviousUsed) {
+          // restore the former value
+          mPingThreshold = mPreviousPingThreshold;
+          mPreviousUsed = false;
+        }
+      }
 
       return PR_IntervalToSeconds(mPingThreshold) -
         PR_IntervalToSeconds(now - mLastReadEpoch);
@@ -2813,5 +2820,34 @@ SpdySession3::PushBack(const char *buf, uint32_t len)
   return mConnection->PushBack(buf, len);
 }
 
+void
+SpdySession3::SendPing()
+{
+  MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
+
+  if (mPreviousUsed) {
+    // alredy in progress, get out
+    return;
+  }
+
+  mPingSentEpoch = PR_IntervalNow();
+  if (!mPingSentEpoch) {
+    mPingSentEpoch = 1; // avoid the 0 sentinel value
+  }
+  if (!mPingThreshold ||
+      (mPingThreshold > gHttpHandler->NetworkChangedTimeout())) {
+    mPreviousPingThreshold = mPingThreshold;
+    mPreviousUsed = true;
+    mPingThreshold = gHttpHandler->NetworkChangedTimeout();
+  }
+
+  GeneratePing(mNextPingID);
+  mNextPingID += 2;
+  ResumeRecv();
+
+  gHttpHandler->ConnMgr()->ActivateTimeoutTick();
+}
+
+
 } // namespace mozilla::net
 } // namespace mozilla

netwerk/protocol/http/SpdySession3.h

@@ -189,6 +189,8 @@ public:
   z_stream *UpstreamZlib() { return &mUpstreamZlib; }
   nsISocketTransport *SocketTransport() { return mSocketTransport; }
 
+  void SendPing() MOZ_OVERRIDE;
+
 private:
 
   enum stateType {
@@ -378,6 +380,9 @@ private:
   PRIntervalTime       mPingSentEpoch;
   uint32_t             mNextPingID;
 
+  PRIntervalTime       mPreviousPingThreshold; // backup for the former value
+  bool                 mPreviousUsed;          // true when backup is used
+
   // used as a temporary buffer while enumerating the stream hash during GoAway
   nsDeque  mGoAwayStreamsToRestart;
 

netwerk/protocol/http/SpdySession31.cpp

@@ -75,6 +75,7 @@ SpdySession31::SpdySession31(nsISocketTransport *aSocketTransport)
   , mLastReadEpoch(PR_IntervalNow())
   , mPingSentEpoch(0)
   , mNextPingID(1)
+  , mPreviousUsed(false)
 {
   MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
 
@@ -227,8 +228,14 @@ SpdySession31::ReadTimeoutTick(PRIntervalTime now)
 
   if ((now - mLastReadEpoch) < mPingThreshold) {
     // recent activity means ping is not an issue
-    if (mPingSentEpoch)
+    if (mPingSentEpoch) {
       mPingSentEpoch = 0;
+      if (mPreviousUsed) {
+        // restore the former value
+        mPingThreshold = mPreviousPingThreshold;
+        mPreviousUsed = false;
+      }
+    }
 
     return PR_IntervalToSeconds(mPingThreshold) -
       PR_IntervalToSeconds(now - mLastReadEpoch);
@@ -2957,5 +2964,33 @@ SpdySession31::PushBack(const char *buf, uint32_t len)
   return mConnection->PushBack(buf, len);
 }
 
+void
+SpdySession31::SendPing()
+{
+  MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
+
+  if (mPreviousUsed) {
+    // alredy in progress, get out
+    return;
+  }
+
+  mPingSentEpoch = PR_IntervalNow();
+  if (!mPingSentEpoch) {
+    mPingSentEpoch = 1; // avoid the 0 sentinel value
+  }
+  if (!mPingThreshold ||
+      (mPingThreshold >  gHttpHandler->NetworkChangedTimeout())) {
+    mPreviousPingThreshold = mPingThreshold;
+    mPreviousUsed = true;
+    mPingThreshold = gHttpHandler->NetworkChangedTimeout();
+  }
+
+  GeneratePing(mNextPingID);
+  mNextPingID += 2;
+  ResumeRecv();
+
+  gHttpHandler->ConnMgr()->ActivateTimeoutTick();
+}
+
 } // namespace mozilla::net
 } // namespace mozilla

netwerk/protocol/http/SpdySession31.h

@@ -193,6 +193,8 @@ public:
   int64_t RemoteSessionWindow() { return mRemoteSessionWindow; }
   void DecrementRemoteSessionWindow (uint32_t bytes) { mRemoteSessionWindow -= bytes; }
 
+  void SendPing() MOZ_OVERRIDE;
+
 private:
 
   enum stateType {
@@ -397,6 +399,9 @@ private:
   PRIntervalTime       mPingSentEpoch;
   uint32_t             mNextPingID;
 
+  PRIntervalTime       mPreviousPingThreshold; // backup for the former value
+  bool                 mPreviousUsed;          // true when backup is used
+
   // used as a temporary buffer while enumerating the stream hash during GoAway
   nsDeque  mGoAwayStreamsToRestart;
 

netwerk/protocol/http/nsHttpConnection.cpp

@@ -69,6 +69,7 @@ nsHttpConnection::nsHttpConnection()
     , mExperienced(false)
     , mInSpdyTunnel(false)
     , mForcePlainText(false)
+    , mTrafficStamp(false)
     , mHttp1xTransactionCount(0)
     , mRemainingConnectionUses(0xffffffff)
     , mClassification(nsAHttpTransaction::CLASS_GENERAL)
@@ -2080,5 +2081,23 @@ nsHttpConnection::GetInterface(const nsIID &iid, void **result)
     return NS_ERROR_NO_INTERFACE;
 }
 
+void
+nsHttpConnection::CheckForTraffic(bool check)
+{
+    if (check) {
+        if (mSpdySession) {
+            // Send a ping to verify it is still alive
+            mSpdySession->SendPing();
+        } else {
+            // If not SPDY, Store snapshot amount of data right now
+            mTrafficCount = mTotalBytesWritten + mTotalBytesRead;
+            mTrafficStamp = true;
+        }
+    } else {
+        // mark it as not checked
+        mTrafficStamp = false;
+    }
+}
+
 } // namespace mozilla::net
 } // namespace mozilla

netwerk/protocol/http/nsHttpConnection.h

@@ -200,6 +200,20 @@ public:
     void    SetupSecondaryTLS();
     void    SetInSpdyTunnel(bool arg);
 
+    // Returns true if the socket peer has a private (RFC1918-like) address.
+    bool    PeerHasPrivateIP();
+    // Check active connections for traffic (or not). SPDY connections send a
+    // ping, ordinary HTTP connections get some time to get traffic to be
+    // considered alive.
+    void CheckForTraffic(bool check);
+
+    // NoTraffic() returns true if there's been no traffic on the (non-spdy)
+    // connection since CheckForTraffic() was called.
+    bool NoTraffic() {
+        return mTrafficStamp &&
+            (mTrafficCount == (mTotalBytesWritten + mTotalBytesRead));
+    }
+
 private:
     // Value (set in mTCPKeepaliveConfig) indicates which set of prefs to use.
     enum TCPKeepaliveConfig {
@@ -292,6 +306,10 @@ private:
     bool                            mInSpdyTunnel;
     bool                            mForcePlainText;
 
+    // A snapshot of current number of transfered bytes
+    int64_t                         mTrafficCount;
+    bool                            mTrafficStamp; // true then the above is set
+
     // The number of <= HTTP/1.1 transactions performed on this connection. This
     // excludes spdy transactions.
     uint32_t                        mHttp1xTransactionCount;

netwerk/protocol/http/nsHttpConnectionMgr.cpp

@@ -85,6 +85,7 @@ nsHttpConnectionMgr::nsHttpConnectionMgr()
     , mNumSpdyActiveConns(0)
     , mNumHalfOpenConns(0)
     , mTimeOfNextWakeUp(UINT64_MAX)
+    , mPruningNoTraffic(false)
     , mTimeoutTickArmed(false)
     , mTimeoutTickNext(1)
 {
@@ -271,6 +272,8 @@ nsHttpConnectionMgr::Observe(nsISupports *subject,
         }
         else if (timer == mTimeoutTick) {
             TimeoutTick();
+        } else if (timer == mTrafficTimer) {
+            PruneNoTraffic();
         }
         else {
             MOZ_ASSERT(false, "unexpected timer-callback");
@@ -328,6 +331,25 @@ nsHttpConnectionMgr::PruneDeadConnections()
     return PostEvent(&nsHttpConnectionMgr::OnMsgPruneDeadConnections);
 }
 
+//
+// Called after a timeout. Check for active connections that have had no
+// traffic since they were "marked" and nuke them.
+nsresult
+nsHttpConnectionMgr::PruneNoTraffic()
+{
+    LOG(("nsHttpConnectionMgr::PruneNoTraffic\n"));
+    mPruningNoTraffic = true;
+    return PostEvent(&nsHttpConnectionMgr::OnMsgPruneNoTraffic);
+}
+
+nsresult
+nsHttpConnectionMgr::VerifyTraffic()
+{
+    LOG(("nsHttpConnectionMgr::VerifyTraffic\n"));
+    return PostEvent(&nsHttpConnectionMgr::OnMsgVerifyTraffic);
+}
+
+
 nsresult
 nsHttpConnectionMgr::DoShiftReloadConnectionCleanup(nsHttpConnectionInfo *aCI)
 {
@@ -1010,6 +1032,53 @@ nsHttpConnectionMgr::PruneDeadConnectionsCB(const nsACString &key,
     return PL_DHASH_NEXT;
 }
 
+PLDHashOperator
+nsHttpConnectionMgr::VerifyTrafficCB(const nsACString &key,
+                                     nsAutoPtr<nsConnectionEntry> &ent,
+                                     void *closure)
+{
+    // Iterate over all active connections and check them
+    for (uint32_t index = 0; index < ent->mActiveConns.Length(); ++index) {
+        nsHttpConnection *conn = ent->mActiveConns[index];
+        conn->CheckForTraffic(true);
+    }
+    // Iterate the idle connections and unmark them for traffic checks
+    for (uint32_t index = 0; index < ent->mIdleConns.Length(); ++index) {
+        nsHttpConnection *conn = ent->mIdleConns[index];
+        conn->CheckForTraffic(false);
+    }
+
+    return PL_DHASH_NEXT;
+}
+
+PLDHashOperator
+nsHttpConnectionMgr::PruneNoTrafficCB(const nsACString &key,
+                                      nsAutoPtr<nsConnectionEntry> &ent,
+                                      void *closure)
+{
+    // Close the connections with no registered traffic
+    nsHttpConnectionMgr *self = (nsHttpConnectionMgr *) closure;
+
+    LOG(("  pruning no traffic [ci=%s]\n", ent->mConnInfo->HashKey().get()));
+
+    uint32_t numConns = ent->mActiveConns.Length();
+    if (numConns) {
+        // walk the list backwards to allow us to remove entries easily
+        for (int index = numConns-1; index >= 0; index--) {
+            if (ent->mActiveConns[index]->NoTraffic()) {
+              nsRefPtr<nsHttpConnection> conn = dont_AddRef(ent->mActiveConns[index]);
+              ent->mActiveConns.RemoveElementAt(index);
+              self->DecrementActiveConnCount(conn);
+              conn->Close(NS_ERROR_ABORT);
+              LOG(("  closed active connection due to no traffic [conn=%p]\n",
+                   conn.get()));
+            }
+        }
+    }
+
+    return PL_DHASH_NEXT;
+}
+
 PLDHashOperator
 nsHttpConnectionMgr::ShutdownPassCB(const nsACString &key,
                                     nsAutoPtr<nsConnectionEntry> &ent,
@@ -2214,6 +2283,10 @@ nsHttpConnectionMgr::OnMsgShutdown(int32_t, void *param)
       mTimer->Cancel();
       mTimer = nullptr;
     }
+    if (mTrafficTimer) {
+      mTrafficTimer->Cancel();
+      mTrafficTimer = nullptr;
+    }
 
     // signal shutdown complete
     nsRefPtr<nsIRunnable> runnable =
@@ -2402,6 +2475,50 @@ nsHttpConnectionMgr::OnMsgPruneDeadConnections(int32_t, void *)
         mCT.Enumerate(PruneDeadConnectionsCB, this);
 }
 
+void
+nsHttpConnectionMgr::OnMsgPruneNoTraffic(int32_t, void *)
+{
+    MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
+    LOG(("nsHttpConnectionMgr::OnMsgPruneNoTraffic\n"));
+
+    // Prune connections without traffic
+    mCT.Enumerate(PruneNoTrafficCB, this);
+
+    mPruningNoTraffic = false; // not pruning anymore
+}
+
+void
+nsHttpConnectionMgr::OnMsgVerifyTraffic(int32_t, void *)
+{
+    MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
+    LOG(("nsHttpConnectionMgr::OnMsgVerifyTraffic\n"));
+
+    if (mPruningNoTraffic) {
+      // Called in the time gap when the timeout to prune notraffic
+      // connections has triggered but the pruning hasn't happened yet.
+      return;
+    }
+
+    // Mark connections for traffic verification
+    mCT.Enumerate(VerifyTrafficCB, this);
+
+    // If the timer is already there. we just re-init it
+    if(!mTrafficTimer) {
+        mTrafficTimer = do_CreateInstance("@mozilla.org/timer;1");
+    }
+
+    // failure to create a timer is not a fatal error, but dead
+    // connections will not be cleaned up as nicely
+    if (mTrafficTimer) {
+        // Give active connections time to get more traffic before killing
+        // them off. Default: 5000 milliseconds
+        mTrafficTimer->Init(this, gHttpHandler->NetworkChangedTimeout(),
+                            nsITimer::TYPE_ONE_SHOT);
+    } else {
+        NS_WARNING("failed to create timer for VerifyTraffic!");
+    }
+}
+
 void
 nsHttpConnectionMgr::OnMsgDoShiftReloadConnectionCleanup(int32_t, void *param)
 {

netwerk/protocol/http/nsHttpConnectionMgr.h

@@ -90,6 +90,14 @@ public:
     // connections.
     nsresult PruneDeadConnections();
 
+    // called to close active connections with no registered "traffic"
+    nsresult PruneNoTraffic();
+
+    // "VerifyTraffic" means marking connections now, and then check again in
+    // N seconds to see if there's been any traffic and if not, kill
+    // that connection.
+    nsresult VerifyTraffic();
+
     // Close all idle persistent connections and prevent any active connections
     // from being reused. Optional connection info resets CI specific
     // information such as Happy Eyeballs history.
@@ -243,6 +251,9 @@ public:
 
     uint16_t MaxRequestDelay() { return mMaxRequestDelay; }
 
+    // public, so that the SPDY/http2 seesions can activate
+    void ActivateTimeoutTick();
+
 private:
     virtual ~nsHttpConnectionMgr();
 
@@ -525,6 +536,8 @@ private:
     static PLDHashOperator PurgeExcessIdleConnectionsCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
     static PLDHashOperator PurgeExcessSpdyConnectionsCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
     static PLDHashOperator ClosePersistentConnectionsCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
+    static PLDHashOperator VerifyTrafficCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
+    static PLDHashOperator PruneNoTrafficCB(const nsACString &, nsAutoPtr<nsConnectionEntry> &, void *);
     bool     ProcessPendingQForEntry(nsConnectionEntry *, bool considerAll);
     bool     IsUnderPressure(nsConnectionEntry *ent,
                              nsHttpTransaction::Classifier classification);
@@ -643,6 +656,8 @@ private:
     void OnMsgProcessFeedback      (int32_t, void *);
     void OnMsgProcessAllSpdyPendingQ (int32_t, void *);
     void OnMsgUpdateRequestTokenBucket (int32_t, void *);
+    void OnMsgVerifyTraffic (int32_t, void *);
+    void OnMsgPruneNoTraffic (int32_t, void *);
 
     // Total number of active connections in all of the ConnectionEntry objects
     // that are accessed from mCT connection table.
@@ -660,6 +675,9 @@ private:
     uint64_t mTimeOfNextWakeUp;
     // Timer for next pruning of dead connections.
     nsCOMPtr<nsITimer> mTimer;
+    // Timer for pruning stalled connections after changed network.
+    nsCOMPtr<nsITimer> mTrafficTimer;
+    bool mPruningNoTraffic;
 
     // A 1s tick to call nsHttpConnection::ReadTimeoutTick on
     // active http/1 connections and check for orphaned half opens.
@@ -685,7 +703,6 @@ private:
         void *aArg);
 
     // Read Timeout Tick handlers
-    void ActivateTimeoutTick();
     void TimeoutTick();
     static PLDHashOperator TimeoutTickCB(const nsACString &key,
                                          nsAutoPtr<nsConnectionEntry> &ent,

netwerk/protocol/http/nsHttpHandler.cpp

@@ -144,6 +144,7 @@ nsHttpHandler::nsHttpHandler()
     , mSpdyTimeout(PR_SecondsToInterval(180))
     , mResponseTimeout(PR_SecondsToInterval(300))
     , mResponseTimeoutEnabled(false)
+    , mNetworkChangedTimeout(5000)
     , mMaxRequestAttempts(10)
     , mMaxRequestDelay(10)
     , mIdleSynTimeout(250)
@@ -859,6 +860,12 @@ nsHttpHandler::PrefsChanged(nsIPrefBranch *prefs, const char *pref)
             mResponseTimeout = PR_SecondsToInterval(clamped(val, 0, 0xffff));
     }
 
+    if (PREF_CHANGED(HTTP_PREF("network-changed.timeout"))) {
+        rv = prefs->GetIntPref(HTTP_PREF("network-changed.timeout"), &val);
+        if (NS_SUCCEEDED(rv))
+            mNetworkChangedTimeout = clamped(val, 1, 600) * 1000;
+    }
+
     if (PREF_CHANGED(HTTP_PREF("max-connections"))) {
         rv = prefs->GetIntPref(HTTP_PREF("max-connections"), &val);
         if (NS_SUCCEEDED(rv)) {
@@ -1832,6 +1839,7 @@ nsHttpHandler::Observe(nsISupports *subject,
         if (strcmp(state, NS_NETWORK_LINK_DATA_CHANGED) == 0) {
             if (mConnMgr) {
                 mConnMgr->PruneDeadConnections();
+                mConnMgr->VerifyTraffic();
             }
         }
     }

netwerk/protocol/http/nsHttpHandler.h

@@ -79,6 +79,7 @@ public:
       return mResponseTimeoutEnabled ? mResponseTimeout : 0;
     }
     PRIntervalTime ResponseTimeoutEnabled()  { return mResponseTimeoutEnabled; }
+    uint32_t       NetworkChangedTimeout()   { return mNetworkChangedTimeout; }
     uint16_t       MaxRequestAttempts()      { return mMaxRequestAttempts; }
     const char    *DefaultSocketType()       { return mDefaultSocketType.get(); /* ok to return null */ }
     uint32_t       PhishyUserPassLength()    { return mPhishyUserPassLength; }
@@ -357,7 +358,7 @@ private:
     PRIntervalTime mSpdyTimeout;
     PRIntervalTime mResponseTimeout;
     bool mResponseTimeoutEnabled;
-
+    uint32_t mNetworkChangedTimeout; // milliseconds
     uint16_t mMaxRequestAttempts;
     uint16_t mMaxRequestDelay;
     uint16_t mIdleSynTimeout;