From 2a5b7140ea1c133f6aa2abd5933b18842c7db2c5 Mon Sep 17 00:00:00 2001 From: Patrick McManus Date: Fri, 7 Nov 2008 18:00:26 -0500 Subject: [PATCH] Bug 453403. Add DNS prefetching, similar to what Google chrome does. r+sr=bzbarsky, a=beltzner --- content/base/src/nsContentSink.cpp | 23 ++ content/base/src/nsContentSink.h | 4 + content/base/src/nsDocument.cpp | 1 + content/base/src/nsGkAtomList.h | 1 + content/html/content/src/Makefile.in | 1 + .../html/content/src/nsHTMLAnchorElement.cpp | 18 ++ .../html/document/src/nsHTMLContentSink.cpp | 7 + content/xml/document/src/nsXMLContentSink.cpp | 12 + layout/build/nsLayoutStatics.cpp | 8 + netwerk/base/public/nsNetError.h | 8 + netwerk/base/src/Makefile.in | 1 + netwerk/build/nsNetModule.cpp | 6 +- netwerk/dns/public/nsIDNSService.idl | 9 +- netwerk/dns/src/nsDNSService2.cpp | 18 +- netwerk/dns/src/nsDNSService2.h | 1 + netwerk/dns/src/nsHostResolver.cpp | 296 ++++++++++++++---- netwerk/dns/src/nsHostResolver.h | 44 ++- netwerk/protocol/http/src/nsHttpChannel.cpp | 8 + 18 files changed, 393 insertions(+), 73 deletions(-) diff --git a/content/base/src/nsContentSink.cpp b/content/base/src/nsContentSink.cpp index 99dceb0b5d8d..e4009f12f13e 100644 --- a/content/base/src/nsContentSink.cpp +++ b/content/base/src/nsContentSink.cpp @@ -102,6 +102,8 @@ #include "nsIDocumentLoader.h" #include "nsICachingChannel.h" #include "nsICacheEntryDescriptor.h" +#include "nsGenericHTMLElement.h" +#include "nsHTMLDNSPrefetch.h" PRLogModuleInfo* gContentSinkLogModuleInfo; @@ -722,6 +724,10 @@ nsContentSink::ProcessLink(nsIContent* aElement, PrefetchHref(aHref, aElement, hasPrefetch); } + if ((!aHref.IsEmpty()) && linkTypes.IndexOf(NS_LITERAL_STRING("dns-prefetch")) != -1) { + PrefetchDNS(aHref); + } + // is it a stylesheet link? if (linkTypes.IndexOf(NS_LITERAL_STRING("stylesheet")) == -1) { return NS_OK; @@ -853,6 +859,23 @@ nsContentSink::PrefetchHref(const nsAString &aHref, } } +void +nsContentSink::PrefetchDNS(const nsAString &aHref) +{ + nsAutoString hostname; + + if (StringBeginsWith(aHref, NS_LITERAL_STRING("//"))) { + hostname = Substring(aHref, 2); + } + else + nsGenericHTMLElement::GetHostnameFromHrefString(aHref, hostname); + + nsRefPtr prefetch = new nsHTMLDNSPrefetch(hostname, mDocument); + if (prefetch) { + prefetch->PrefetchLow(); + } +} + nsresult nsContentSink::GetChannelCacheKey(nsIChannel* aChannel, nsACString& aCacheKey) { diff --git a/content/base/src/nsContentSink.h b/content/base/src/nsContentSink.h index 9b7ca1ca642c..b640edefabbb 100644 --- a/content/base/src/nsContentSink.h +++ b/content/base/src/nsContentSink.h @@ -195,6 +195,10 @@ protected: void PrefetchHref(const nsAString &aHref, nsIContent *aSource, PRBool aExplicit); + // aHref can either be the usual URI format or of the form "//www.hostname.com" + // without a scheme. + void PrefetchDNS(const nsAString &aHref); + // Gets the cache key (used to identify items in a cache) of the channel. nsresult GetChannelCacheKey(nsIChannel* aChannel, nsACString& aCacheKey); diff --git a/content/base/src/nsDocument.cpp b/content/base/src/nsDocument.cpp index 0cae16446df4..72586ac47386 100644 --- a/content/base/src/nsDocument.cpp +++ b/content/base/src/nsDocument.cpp @@ -6572,6 +6572,7 @@ nsDocument::RetrieveRelevantHeaders(nsIChannel *aChannel) "content-language", "content-disposition", "refresh", + "x-dns-prefetch-control", // add more http headers if you need // XXXbz don't add content-location support without reading bug // 238654 and its dependencies/dups first. diff --git a/content/base/src/nsGkAtomList.h b/content/base/src/nsGkAtomList.h index d4a2e3bc12e1..54e0fa137619 100755 --- a/content/base/src/nsGkAtomList.h +++ b/content/base/src/nsGkAtomList.h @@ -987,6 +987,7 @@ GK_ATOM(headerWindowTarget, "window-target") GK_ATOM(withParam, "with-param") GK_ATOM(wizard, "wizard") GK_ATOM(wrap, "wrap") +GK_ATOM(headerDNSPrefetchControl,"x-dns-prefetch-control") GK_ATOM(xml, "xml") GK_ATOM(xmlns, "xmlns") GK_ATOM(xmp, "xmp") diff --git a/content/html/content/src/Makefile.in b/content/html/content/src/Makefile.in index 962a240e57f2..f9562ccc0c1a 100644 --- a/content/html/content/src/Makefile.in +++ b/content/html/content/src/Makefile.in @@ -83,6 +83,7 @@ EXPORTS = \ CPPSRCS = \ nsClientRect.cpp \ + nsHTMLDNSPrefetch.cpp \ nsGenericHTMLElement.cpp \ nsFormSubmission.cpp \ nsImageMapUtils.cpp \ diff --git a/content/html/content/src/nsHTMLAnchorElement.cpp b/content/html/content/src/nsHTMLAnchorElement.cpp index e373bbc9a9c5..1887aaf79cf1 100644 --- a/content/html/content/src/nsHTMLAnchorElement.cpp +++ b/content/html/content/src/nsHTMLAnchorElement.cpp @@ -64,6 +64,8 @@ #include "nsIPresShell.h" #include "nsIDocument.h" +#include "nsHTMLDNSPrefetch.h" + nsresult NS_NewContentIterator(nsIContentIterator** aInstancePtrResult); class nsHTMLAnchorElement : public nsGenericHTMLElement, @@ -135,11 +137,26 @@ public: protected: // The cached visited state nsLinkState mLinkState; + + void PrefetchDNS(); }; NS_IMPL_NS_NEW_HTML_ELEMENT(Anchor) +void +nsHTMLAnchorElement::PrefetchDNS() +{ + nsCOMPtr hrefURI; + GetHrefURI(getter_AddRefs(hrefURI)); + + if (hrefURI) { + nsRefPtr prefetch = + new nsHTMLDNSPrefetch(hrefURI, GetOwnerDoc()); + if (prefetch) + prefetch->PrefetchLow(); + } +} nsHTMLAnchorElement::nsHTMLAnchorElement(nsINodeInfo *aNodeInfo) : nsGenericHTMLElement(aNodeInfo), @@ -212,6 +229,7 @@ nsHTMLAnchorElement::BindToTree(nsIDocument* aDocument, nsIContent* aParent, RegUnRegAccessKey(PR_TRUE); } + PrefetchDNS(); return rv; } diff --git a/content/html/document/src/nsHTMLContentSink.cpp b/content/html/document/src/nsHTMLContentSink.cpp index 9c9be73c2a54..0ba96e520113 100644 --- a/content/html/document/src/nsHTMLContentSink.cpp +++ b/content/html/document/src/nsHTMLContentSink.cpp @@ -2950,6 +2950,13 @@ HTMLContentSink::ProcessLINKTag(const nsIParserNode& aNode) PrefetchHref(hrefVal, element, hasPrefetch); } } + if (linkTypes.IndexOf(NS_LITERAL_STRING("dns-prefetch")) != -1) { + nsAutoString hrefVal; + element->GetAttr(kNameSpaceID_None, nsGkAtoms::href, hrefVal); + if (!hrefVal.IsEmpty()) { + PrefetchDNS(hrefVal); + } + } } } } diff --git a/content/xml/document/src/nsXMLContentSink.cpp b/content/xml/document/src/nsXMLContentSink.cpp index 7c5c5000fa18..56f1ddb7e1dc 100644 --- a/content/xml/document/src/nsXMLContentSink.cpp +++ b/content/xml/document/src/nsXMLContentSink.cpp @@ -657,6 +657,18 @@ nsXMLContentSink::CloseElement(nsIContent* aContent) mScriptLoader->AddExecuteBlocker(); } } + // Look for + if (nodeInfo->Equals(nsGkAtoms::link, kNameSpaceID_XHTML)) { + nsAutoString relVal; + aContent->GetAttr(kNameSpaceID_None, nsGkAtoms::rel, relVal); + if (relVal.EqualsLiteral("dns-prefetch")) { + nsAutoString hrefVal; + aContent->GetAttr(kNameSpaceID_None, nsGkAtoms::href, hrefVal); + if (!hrefVal.IsEmpty()) { + PrefetchDNS(hrefVal); + } + } + } } return rv; diff --git a/layout/build/nsLayoutStatics.cpp b/layout/build/nsLayoutStatics.cpp index e1a1df2bef35..c6fe44f49869 100644 --- a/layout/build/nsLayoutStatics.cpp +++ b/layout/build/nsLayoutStatics.cpp @@ -83,6 +83,7 @@ #include "nsXMLHttpRequest.h" #include "nsIFocusEventSuppressor.h" #include "nsDOMThreadService.h" +#include "nsHTMLDNSPrefetch.h" #ifdef MOZ_XUL #include "nsXULPopupManager.h" @@ -185,6 +186,12 @@ nsLayoutStatics::Initialize() return rv; } + rv = nsHTMLDNSPrefetch::Initialize(); + if (NS_FAILED(rv)) { + NS_ERROR("Could not initialize HTML DNS prefetch"); + return rv; + } + #ifdef MOZ_XUL rv = nsXULContentUtils::Init(); if (NS_FAILED(rv)) { @@ -280,6 +287,7 @@ nsLayoutStatics::Shutdown() CSSLoaderImpl::Shutdown(); nsCSSRuleProcessor::FreeSystemMetrics(); nsTextFrameTextRunCache::Shutdown(); + nsHTMLDNSPrefetch::Shutdown(); nsCSSRendering::Shutdown(); #ifdef DEBUG nsFrame::DisplayReflowShutdown(); diff --git a/netwerk/base/public/nsNetError.h b/netwerk/base/public/nsNetError.h index f3f8bda3d487..b9bac3de8f8e 100644 --- a/netwerk/base/public/nsNetError.h +++ b/netwerk/base/public/nsNetError.h @@ -272,6 +272,14 @@ #define NS_ERROR_UNKNOWN_HOST \ NS_ERROR_GENERATE_FAILURE(NS_ERROR_MODULE_NETWORK, 30) +/** + * A low or medium priority DNS lookup failed because the pending + * queue was already full. High priorty (the default) always + * makes room + */ +#define NS_ERROR_DNS_LOOKUP_QUEUE_FULL \ + NS_ERROR_GENERATE_FAILURE(NS_ERROR_MODULE_NETWORK, 33) + /** * The lookup of a proxy hostname failed. * diff --git a/netwerk/base/src/Makefile.in b/netwerk/base/src/Makefile.in index 4c2dea5de2dd..2058b7a39f1f 100644 --- a/netwerk/base/src/Makefile.in +++ b/netwerk/base/src/Makefile.in @@ -93,6 +93,7 @@ CPPSRCS = \ nsNetStrings.cpp \ nsBase64Encoder.cpp \ nsSerializationHelper.cpp \ + nsDNSPrefetch.cpp \ $(NULL) ifeq ($(MOZ_WIDGET_TOOLKIT),os2) diff --git a/netwerk/build/nsNetModule.cpp b/netwerk/build/nsNetModule.cpp index 6e894aea381f..6c573c6d6b77 100644 --- a/netwerk/build/nsNetModule.cpp +++ b/netwerk/build/nsNetModule.cpp @@ -59,6 +59,7 @@ #include "nsDiskCacheDeviceSQL.h" #include "nsMimeTypes.h" #include "nsNetStrings.h" +#include "nsDNSPrefetch.h" #include "nsNetCID.h" @@ -619,10 +620,13 @@ static void nsNetShutdown(nsIModule *neckoModule) #ifdef XP_MACOSX net_ShutdownURLHelperOSX(); #endif - + // Release necko strings delete gNetStrings; gNetStrings = nsnull; + + // Release DNS service reference. + nsDNSPrefetch::Shutdown(); } static const nsModuleComponentInfo gNetModuleInfo[] = { diff --git a/netwerk/dns/public/nsIDNSService.idl b/netwerk/dns/public/nsIDNSService.idl index d7d5e135c2b2..522361a1c997 100644 --- a/netwerk/dns/public/nsIDNSService.idl +++ b/netwerk/dns/public/nsIDNSService.idl @@ -46,7 +46,7 @@ interface nsIDNSListener; /** * nsIDNSService */ -[scriptable, uuid(3ac9e611-e6b6-44b5-b312-c040e65b2929)] +[scriptable, uuid(ee4d9f1d-4f99-4384-b547-29da735f8b6e)] interface nsIDNSService : nsISupports { /** @@ -107,4 +107,11 @@ interface nsIDNSService : nsISupports * if set, the canonical name of the specified host will be queried. */ const unsigned long RESOLVE_CANONICAL_NAME = (1 << 1); + + /** + * if set, the query is given lower priority. Medium takes precedence + * if both are used. + */ + const unsigned long RESOLVE_PRIORITY_MEDIUM = (1 << 2); + const unsigned long RESOLVE_PRIORITY_LOW = (1 << 3); }; diff --git a/netwerk/dns/src/nsDNSService2.cpp b/netwerk/dns/src/nsDNSService2.cpp index a6b8a75175e2..ed4eecb3af08 100644 --- a/netwerk/dns/src/nsDNSService2.cpp +++ b/netwerk/dns/src/nsDNSService2.cpp @@ -50,6 +50,7 @@ #include "nsAutoPtr.h" #include "nsNetCID.h" #include "nsNetError.h" +#include "nsDNSPrefetch.h" #include "prsystem.h" #include "prnetdb.h" #include "prmon.h" @@ -61,6 +62,7 @@ static const char kPrefDnsCacheExpiration[] = "network.dnsCacheExpiration"; static const char kPrefEnableIDN[] = "network.enableIDN"; static const char kPrefIPv4OnlyDomains[] = "network.dns.ipv4OnlyDomains"; static const char kPrefDisableIPv6[] = "network.dns.disableIPv6"; +static const char kPrefDisablePrefetch[] = "network.dns.disablePrefetch"; //----------------------------------------------------------------------------- @@ -321,10 +323,12 @@ nsDNSService::Init() PRBool firstTime = (mLock == nsnull); // prefs - PRUint32 maxCacheEntries = 20; - PRUint32 maxCacheLifetime = 1; // minutes + PRUint32 maxCacheEntries = 400; + PRUint32 maxCacheLifetime = 3; // minutes PRBool enableIDN = PR_TRUE; PRBool disableIPv6 = PR_FALSE; + PRBool disablePrefetch = PR_FALSE; + nsAdoptingCString ipv4OnlyDomains; // read prefs @@ -340,6 +344,7 @@ nsDNSService::Init() prefs->GetBoolPref(kPrefEnableIDN, &enableIDN); prefs->GetBoolPref(kPrefDisableIPv6, &disableIPv6); prefs->GetCharPref(kPrefIPv4OnlyDomains, getter_Copies(ipv4OnlyDomains)); + prefs->GetBoolPref(kPrefDisablePrefetch, &disablePrefetch); } if (firstTime) { @@ -354,6 +359,7 @@ nsDNSService::Init() prefs->AddObserver(kPrefEnableIDN, this, PR_FALSE); prefs->AddObserver(kPrefIPv4OnlyDomains, this, PR_FALSE); prefs->AddObserver(kPrefDisableIPv6, this, PR_FALSE); + prefs->AddObserver(kPrefDisablePrefetch, this, PR_FALSE); } } @@ -374,8 +380,10 @@ nsDNSService::Init() mIDN = idn; mIPv4OnlyDomains = ipv4OnlyDomains; // exchanges buffer ownership mDisableIPv6 = disableIPv6; + mDisablePrefetch = disablePrefetch; } - + + nsDNSPrefetch::Initialize(this); return rv; } @@ -406,6 +414,10 @@ nsDNSService::AsyncResolve(const nsACString &hostname, nsCOMPtr idn; { nsAutoLock lock(mLock); + + if (mDisablePrefetch && (flags & (RESOLVE_PRIORITY_LOW | RESOLVE_PRIORITY_MEDIUM))) + return NS_ERROR_DNS_LOOKUP_QUEUE_FULL; + res = mResolver; idn = mIDN; } diff --git a/netwerk/dns/src/nsDNSService2.h b/netwerk/dns/src/nsDNSService2.h index e0e0b34285a9..9487d7836a38 100644 --- a/netwerk/dns/src/nsDNSService2.h +++ b/netwerk/dns/src/nsDNSService2.h @@ -68,4 +68,5 @@ private: // a per-domain basis and work around broken DNS servers. See bug 68796. nsAdoptingCString mIPv4OnlyDomains; PRBool mDisableIPv6; + PRBool mDisablePrefetch; }; diff --git a/netwerk/dns/src/nsHostResolver.cpp b/netwerk/dns/src/nsHostResolver.cpp index 5b5629cd6e54..a6cdafd748a8 100644 --- a/netwerk/dns/src/nsHostResolver.cpp +++ b/netwerk/dns/src/nsHostResolver.cpp @@ -68,8 +68,28 @@ //---------------------------------------------------------------------------- -#define MAX_THREADS 8 -#define IDLE_TIMEOUT PR_SecondsToInterval(60) +// Use a persistent thread pool in order to avoid spinning up new threads all the time. +// In particular, thread creation results in a res_init() call from libc which is +// quite expensive. +// +// The pool dynamically grows between 0 and MAX_RESOLVER_THREADS in size. New requests +// go first to an idle thread. If that cannot be found and there are fewer than MAX_RESOLVER_THREADS +// currently in the pool a new thread is created for high priority requests. If +// the new request is at a lower priority a new thread will only be created if +// there are fewer than HighThreadThreshold currently outstanding. If a thread cannot be +// created or an idle thread located for the request it is queued. +// +// When the pool is greater than HighThreadThreshold in size a thread will be destroyed after +// ShortIdleTimeoutSeconds of idle time. Smaller pools use LongIdleTimeoutSeconds for a +// timeout period. + +#define MAX_NON_PRIORITY_REQUESTS 150 + +#define HighThreadThreshold MAX_RESOLVER_THREADS_FOR_ANY_PRIORITY +#define LongIdleTimeoutSeconds 300 // for threads 1 -> HighThreadThreshold +#define ShortIdleTimeoutSeconds 60 // for threads HighThreadThreshold+1 -> MAX_RESOLVER_THREADS + +PR_STATIC_ASSERT (HighThreadThreshold <= MAX_RESOLVER_THREADS); //---------------------------------------------------------------------------- @@ -184,6 +204,7 @@ nsHostRecord::Create(const nsHostKey *key, nsHostRecord **result) rec->addr = nsnull; rec->expiration = NowInMinutes(); rec->resolving = PR_FALSE; + rec->onQueue = PR_FALSE; PR_INIT_CLIST(rec); PR_INIT_CLIST(&rec->callbacks); rec->negative = PR_FALSE; @@ -308,14 +329,26 @@ nsHostResolver::nsHostResolver(PRUint32 maxCacheEntries, , mMaxCacheLifetime(maxCacheLifetime) , mLock(nsnull) , mIdleThreadCV(nsnull) - , mHaveIdleThread(PR_FALSE) + , mNumIdleThreads(0) , mThreadCount(0) + , mAnyPriorityThreadCount(0) , mEvictionQSize(0) + , mPendingCount(0) , mShutdown(PR_TRUE) { mCreationTime = PR_Now(); - PR_INIT_CLIST(&mPendingQ); + PR_INIT_CLIST(&mHighQ); + PR_INIT_CLIST(&mMediumQ); + PR_INIT_CLIST(&mLowQ); PR_INIT_CLIST(&mEvictionQ); + + mHighPriorityInfo.self = this; + mHighPriorityInfo.onlyHighPriority = PR_TRUE; + mAnyPriorityInfo.self = this; + mAnyPriorityInfo.onlyHighPriority = PR_FALSE; + + mLongIdleTimeout = PR_SecondsToInterval(LongIdleTimeoutSeconds); + mShortIdleTimeout = PR_SecondsToInterval(ShortIdleTimeoutSeconds); } nsHostResolver::~nsHostResolver() @@ -359,13 +392,29 @@ nsHostResolver::Init() return NS_OK; } +void +nsHostResolver::ClearPendingQueue(PRCList *aPendingQ) +{ + // loop through pending queue, erroring out pending lookups. + if (!PR_CLIST_IS_EMPTY(aPendingQ)) { + PRCList *node = aPendingQ->next; + while (node != aPendingQ) { + nsHostRecord *rec = static_cast(node); + node = node->next; + OnLookupComplete(rec, NS_ERROR_ABORT, nsnull); + } + } +} + void nsHostResolver::Shutdown() { LOG(("nsHostResolver::Shutdown\n")); - PRCList pendingQ, evictionQ; - PR_INIT_CLIST(&pendingQ); + PRCList pendingQHigh, pendingQMed, pendingQLow, evictionQ; + PR_INIT_CLIST(&pendingQHigh); + PR_INIT_CLIST(&pendingQMed); + PR_INIT_CLIST(&pendingQLow); PR_INIT_CLIST(&evictionQ); { @@ -373,26 +422,23 @@ nsHostResolver::Shutdown() mShutdown = PR_TRUE; - MoveCList(mPendingQ, pendingQ); + MoveCList(mHighQ, pendingQHigh); + MoveCList(mMediumQ, pendingQMed); + MoveCList(mLowQ, pendingQLow); MoveCList(mEvictionQ, evictionQ); mEvictionQSize = 0; - - if (mHaveIdleThread) - PR_NotifyCondVar(mIdleThreadCV); + mPendingCount = 0; + + if (mNumIdleThreads) + PR_NotifyAllCondVar(mIdleThreadCV); // empty host database PL_DHashTableEnumerate(&mDB, HostDB_RemoveEntry, nsnull); } - - // loop through pending queue, erroring out pending lookups. - if (!PR_CLIST_IS_EMPTY(&pendingQ)) { - PRCList *node = pendingQ.next; - while (node != &pendingQ) { - nsHostRecord *rec = static_cast(node); - node = node->next; - OnLookupComplete(rec, NS_ERROR_ABORT, nsnull); - } - } + + ClearPendingQueue(&pendingQHigh); + ClearPendingQueue(&pendingQMed); + ClearPendingQueue(&pendingQLow); if (!PR_CLIST_IS_EMPTY(&evictionQ)) { PRCList *node = evictionQ.next; @@ -403,6 +449,47 @@ nsHostResolver::Shutdown() } } +#ifdef NS_BUILD_REFCNT_LOGGING + + // Logically join the outstanding worker threads with a timeout. + // Use this approach instead of PR_JoinThread() because that does + // not allow a timeout which may be necessary for a semi-responsive + // shutdown if the thread is blocked on a very slow DNS resolution. + // mThreadCount is read outside of mLock, but the worst case + // scenario for that race is one extra 25ms sleep. + + PRIntervalTime delay = PR_MillisecondsToInterval(25); + PRIntervalTime stopTime = PR_IntervalNow() + PR_SecondsToInterval(20); + while (mThreadCount && PR_IntervalNow() < stopTime) + PR_Sleep(delay); +#endif +} + +static inline PRBool +IsHighPriority(PRUint16 flags) +{ + return !(flags & (nsHostResolver::RES_PRIORITY_LOW | nsHostResolver::RES_PRIORITY_MEDIUM)); +} + +static inline PRBool +IsMediumPriority(PRUint16 flags) +{ + return flags & nsHostResolver::RES_PRIORITY_MEDIUM; +} + +static inline PRBool +IsLowPriority(PRUint16 flags) +{ + return flags & nsHostResolver::RES_PRIORITY_LOW; +} + +void +nsHostResolver::MoveQueue(nsHostRecord *aRec, PRCList &aDestQ) +{ + NS_ASSERTION(aRec->onQueue, "Moving Host Record Not Currently Queued"); + + PR_REMOVE_LINK(aRec); + PR_APPEND_LINK(aRec, &aDestQ); } nsresult @@ -484,9 +571,15 @@ nsHostResolver::ResolveHost(const char *host, // put reference to host record on stack... result = he->rec; } + else if (mPendingCount >= MAX_NON_PRIORITY_REQUESTS && + !IsHighPriority(flags) && + !he->rec->resolving) { + // This is a lower priority request and we are swamped, so refuse it. + rv = NS_ERROR_DNS_LOOKUP_QUEUE_FULL; + } // otherwise, hit the resolver... else { - // add callback to the list of pending callbacks + // Add callback to the list of pending callbacks. PR_APPEND_LINK(callback, &he->rec->callbacks); if (!he->rec->resolving) { @@ -495,6 +588,22 @@ nsHostResolver::ResolveHost(const char *host, if (NS_FAILED(rv)) PR_REMOVE_AND_INIT_LINK(callback); } + else if (he->rec->onQueue) { + // Consider the case where we are on a pending queue of + // lower priority than the request is being made at. + // In that case we should upgrade to the higher queue. + + if (IsHighPriority(flags) && !IsHighPriority(he->rec->flags)) { + // Move from (low|med) to high. + MoveQueue(he->rec, mHighQ); + he->rec->flags = flags; + ConditionallyCreateThread(he->rec); + } else if (IsMediumPriority(flags) && IsLowPriority(he->rec->flags)) { + // Move from low to med. + MoveQueue(he->rec, mMediumQ); + he->rec->flags = flags; + } + } } } } @@ -539,39 +648,38 @@ nsHostResolver::DetachCallback(const char *host, } nsresult -nsHostResolver::IssueLookup(nsHostRecord *rec) +nsHostResolver::ConditionallyCreateThread(nsHostRecord *rec) { - NS_ASSERTION(!rec->resolving, "record is already being resolved"); - - // add rec to mPendingQ, possibly removing it from mEvictionQ. - // if rec is on mEvictionQ, then we can just move the owning - // reference over to mPendingQ. - if (rec->next == rec) - NS_ADDREF(rec); - else { - PR_REMOVE_LINK(rec); - mEvictionQSize--; - } - PR_APPEND_LINK(rec, &mPendingQ); - rec->resolving = PR_TRUE; - - if (mHaveIdleThread) { + if (mNumIdleThreads) { // wake up idle thread to process this lookup PR_NotifyCondVar(mIdleThreadCV); } - else if (mThreadCount < MAX_THREADS) { + else if ((mThreadCount < HighThreadThreshold) || + (IsHighPriority(rec->flags) && mThreadCount < MAX_RESOLVER_THREADS)) { // dispatch new worker thread NS_ADDREF_THIS(); // owning reference passed to thread + + struct nsHostResolverThreadInfo *info; + + if (mAnyPriorityThreadCount < HighThreadThreshold) { + info = &mAnyPriorityInfo; + mAnyPriorityThreadCount++; + } + else + info = &mHighPriorityInfo; + mThreadCount++; PRThread *thr = PR_CreateThread(PR_SYSTEM_THREAD, ThreadFunc, - this, + info, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, 0); if (!thr) { mThreadCount--; + if (info == &mAnyPriorityInfo) + mAnyPriorityThreadCount--; NS_RELEASE_THIS(); return NS_ERROR_OUT_OF_MEMORY; } @@ -580,29 +688,96 @@ nsHostResolver::IssueLookup(nsHostRecord *rec) else LOG(("lookup waiting for thread - %s ...\n", rec->host)); #endif - return NS_OK; } +nsresult +nsHostResolver::IssueLookup(nsHostRecord *rec) +{ + nsresult rv = NS_OK; + NS_ASSERTION(!rec->resolving, "record is already being resolved"); + + // Add rec to one of the pending queues, possibly removing it from mEvictionQ. + // If rec is on mEvictionQ, then we can just move the owning + // reference over to the new active queue. + if (rec->next == rec) + NS_ADDREF(rec); + else { + PR_REMOVE_LINK(rec); + mEvictionQSize--; + } + + if (IsHighPriority(rec->flags)) + PR_APPEND_LINK(rec, &mHighQ); + else if (IsMediumPriority(rec->flags)) + PR_APPEND_LINK(rec, &mMediumQ); + else + PR_APPEND_LINK(rec, &mLowQ); + mPendingCount++; + + rec->resolving = PR_TRUE; + rec->onQueue = PR_TRUE; + + rv = ConditionallyCreateThread(rec); + + LOG (("DNS Thread Counters: total=%d any=%d high=%d idle=%d pending=%d\n", + mThreadCount, + mAnyPriorityThreadCount, + mThreadCount - mAnyPriorityThreadCount, + mNumIdleThreads, + mPendingCount)); + + return rv; +} + +void +nsHostResolver::DeQueue(PRCList &aQ, nsHostRecord **aResult) +{ + *aResult = static_cast(aQ.next); + PR_REMOVE_AND_INIT_LINK(*aResult); + mPendingCount--; + (*aResult)->onQueue = PR_FALSE; +} + PRBool -nsHostResolver::GetHostToLookup(nsHostRecord **result) +nsHostResolver::GetHostToLookup(nsHostRecord **result, struct nsHostResolverThreadInfo *aID) { nsAutoLock lock(mLock); - PRIntervalTime start = PR_IntervalNow(), timeout = IDLE_TIMEOUT; - // - // wait for one or more of the following to occur: - // (1) the pending queue has a host record to process - // (2) the shutdown flag has been set - // (3) the thread has been idle for too long - // - // PR_WaitCondVar will return when any of these conditions is true. - // - while (PR_CLIST_IS_EMPTY(&mPendingQ) && !mHaveIdleThread && !mShutdown) { + PRIntervalTime start = PR_IntervalNow(), timeout; + + while (!mShutdown) { + // remove next record from Q; hand over owning reference. Check high, then med, then low + + if (!PR_CLIST_IS_EMPTY(&mHighQ)) { + DeQueue (mHighQ, result); + return PR_TRUE; + } + + if (! aID->onlyHighPriority) { + if (!PR_CLIST_IS_EMPTY(&mMediumQ)) { + DeQueue (mMediumQ, result); + return PR_TRUE; + } + + if (!PR_CLIST_IS_EMPTY(&mLowQ)) { + DeQueue (mLowQ, result); + return PR_TRUE; + } + } + + timeout = (mNumIdleThreads >= HighThreadThreshold) ? mShortIdleTimeout : mLongIdleTimeout; + // wait for one or more of the following to occur: + // (1) the pending queue has a host record to process + // (2) the shutdown flag has been set + // (3) the thread has been idle for too long + // + // PR_WaitCondVar will return when any of these conditions is true. // become the idle thread and wait for a lookup - mHaveIdleThread = PR_TRUE; + + mNumIdleThreads++; PR_WaitCondVar(mIdleThreadCV, timeout); - mHaveIdleThread = PR_FALSE; + mNumIdleThreads--; PRIntervalTime delta = PR_IntervalNow() - start; if (delta >= timeout) @@ -611,15 +786,10 @@ nsHostResolver::GetHostToLookup(nsHostRecord **result) start += delta; } - if (!PR_CLIST_IS_EMPTY(&mPendingQ)) { - // remove next record from mPendingQ; hand over owning reference. - *result = static_cast(mPendingQ.next); - PR_REMOVE_AND_INIT_LINK(*result); - return PR_TRUE; - } - // tell thread to exit... mThreadCount--; + if (!aID->onlyHighPriority) + mAnyPriorityThreadCount--; return PR_FALSE; } @@ -698,11 +868,11 @@ nsHostResolver::ThreadFunc(void *arg) #if defined(RES_RETRY_ON_FAILURE) nsResState rs; #endif - - nsHostResolver *resolver = (nsHostResolver *) arg; + struct nsHostResolverThreadInfo *info = (struct nsHostResolverThreadInfo *) arg; + nsHostResolver *resolver = info->self; nsHostRecord *rec; PRAddrInfo *ai; - while (resolver->GetHostToLookup(&rec)) { + while (resolver->GetHostToLookup(&rec, info)) { LOG(("resolving %s ...\n", rec->host)); PRIntn flags = PR_AI_ADDRCONFIG; diff --git a/netwerk/dns/src/nsHostResolver.h b/netwerk/dns/src/nsHostResolver.h index cd6c43523a87..054e317c41c6 100644 --- a/netwerk/dns/src/nsHostResolver.h +++ b/netwerk/dns/src/nsHostResolver.h @@ -68,6 +68,11 @@ class nsResolveHostCallback; return n; \ } +#define MAX_RESOLVER_THREADS_FOR_ANY_PRIORITY 3 +#define MAX_RESOLVER_THREADS_FOR_HIGH_PRIORITY 5 +#define MAX_RESOLVER_THREADS (MAX_RESOLVER_THREADS_FOR_ANY_PRIORITY + \ + MAX_RESOLVER_THREADS_FOR_HIGH_PRIORITY) + struct nsHostKey { const char *host; @@ -124,6 +129,9 @@ private: PRBool resolving; /* true if this record is being resolved, which means * that it is either on the pending queue or owned by * one of the worker threads. */ + + PRBool onQueue; /* true if pending and on the queue (not yet given to getaddrinfo())*/ + ~nsHostRecord(); }; @@ -157,6 +165,16 @@ public: nsresult status) = 0; }; +/** + * nsHostResolverThreadInfo structures are passed to the resolver + * thread. + */ +struct nsHostResolverThreadInfo +{ + nsHostResolver *self; + PRBool onlyHighPriority; +}; + /** * nsHostResolver - an asynchronous host name resolver. */ @@ -214,32 +232,48 @@ public: */ enum { RES_BYPASS_CACHE = 1 << 0, - RES_CANON_NAME = 1 << 1 + RES_CANON_NAME = 1 << 1, + RES_PRIORITY_MEDIUM = 1 << 2, + RES_PRIORITY_LOW = 1 << 3 }; private: nsHostResolver(PRUint32 maxCacheEntries=50, PRUint32 maxCacheLifetime=1); ~nsHostResolver(); + // nsHostResolverThreadInfo * is passed to the ThreadFunc + struct nsHostResolverThreadInfo mHighPriorityInfo, mAnyPriorityInfo; + nsresult Init(); nsresult IssueLookup(nsHostRecord *); - PRBool GetHostToLookup(nsHostRecord **); + PRBool GetHostToLookup(nsHostRecord **m, struct nsHostResolverThreadInfo *aID); void OnLookupComplete(nsHostRecord *, nsresult, PRAddrInfo *); - + void DeQueue(PRCList &aQ, nsHostRecord **aResult); + void ClearPendingQueue(PRCList *aPendingQueue); + nsresult ConditionallyCreateThread(nsHostRecord *rec); + + static void MoveQueue(nsHostRecord *aRec, PRCList &aDestQ); + static void ThreadFunc(void *); PRUint32 mMaxCacheEntries; PRUint32 mMaxCacheLifetime; PRLock *mLock; PRCondVar *mIdleThreadCV; // non-null if idle thread - PRBool mHaveIdleThread; + PRUint32 mNumIdleThreads; PRUint32 mThreadCount; + PRUint32 mAnyPriorityThreadCount; PLDHashTable mDB; - PRCList mPendingQ; + PRCList mHighQ; + PRCList mMediumQ; + PRCList mLowQ; PRCList mEvictionQ; PRUint32 mEvictionQSize; + PRUint32 mPendingCount; PRTime mCreationTime; PRBool mShutdown; + PRIntervalTime mLongIdleTimeout; + PRIntervalTime mShortIdleTimeout; }; #endif // nsHostResolver_h__ diff --git a/netwerk/protocol/http/src/nsHttpChannel.cpp b/netwerk/protocol/http/src/nsHttpChannel.cpp index bc7e0aaa279a..808b856500f0 100644 --- a/netwerk/protocol/http/src/nsHttpChannel.cpp +++ b/netwerk/protocol/http/src/nsHttpChannel.cpp @@ -82,6 +82,7 @@ #include "nsIOService.h" #include "nsAuthInformationHolder.h" #include "nsICacheService.h" +#include "nsDNSPrefetch.h" // True if the local cache should be bypassed when processing a request. #define BYPASS_LOCAL_CACHE(loadFlags) \ @@ -4009,6 +4010,13 @@ nsHttpChannel::AsyncOpen(nsIStreamListener *listener, nsISupports *context) if (NS_FAILED(rv)) return rv; + // Start a DNS lookup very early in case the real open is queued the DNS can + // happen in parallel. + nsRefPtr prefetch = new nsDNSPrefetch(mURI); + if (prefetch) { + prefetch->PrefetchHigh(); + } + // Remember the cookie header that was set, if any const char *cookieHeader = mRequestHead.PeekHeader(nsHttp::Cookie); if (cookieHeader)