diff --git a/layout/style/FontFaceSet.cpp b/layout/style/FontFaceSet.cpp
index dd57ef11dc70..3cddf3fa94db 100644
--- a/layout/style/FontFaceSet.cpp
+++ b/layout/style/FontFaceSet.cpp
@@ -581,6 +581,7 @@ nsresult FontFaceSet::StartLoad(gfxUserFontEntry* aUserFontEntry,
 
   RefPtr<nsFontFaceLoader> fontLoader = new nsFontFaceLoader(
       aUserFontEntry, aFontFaceSrc->mURI->get(), this, channel);
+  mLoaders.PutEntry(fontLoader);
 
   if (LOG_ENABLED()) {
     LOG(
@@ -638,7 +639,6 @@ nsresult FontFaceSet::StartLoad(gfxUserFontEntry* aUserFontEntry,
   }
 
   if (NS_SUCCEEDED(rv)) {
-    mLoaders.PutEntry(fontLoader);
     fontLoader->StartedLoading(streamLoader);
     // let the font entry remember the loader, in case we need to cancel it
     aUserFontEntry->SetLoader(fontLoader);
diff --git a/layout/style/nsFontFaceLoader.h b/layout/style/nsFontFaceLoader.h
index 157c21bf843f..1ee86a07fecc 100644
--- a/layout/style/nsFontFaceLoader.h
+++ b/layout/style/nsFontFaceLoader.h
@@ -55,7 +55,8 @@ class nsFontFaceLoader final : public nsIStreamLoaderObserver,
  private:
   RefPtr<gfxUserFontEntry> mUserFontEntry;
   nsCOMPtr<nsIURI> mFontURI;
-  RefPtr<mozilla::dom::FontFaceSet> mFontFaceSet;
+  // Cleared in FontFaceSet::~FontFaceSet, and on cancelation and such too.
+  mozilla::dom::FontFaceSet* MOZ_NON_OWNING_REF mFontFaceSet;
   nsCOMPtr<nsIChannel> mChannel;
   nsCOMPtr<nsITimer> mLoadTimer;
   mozilla::TimeStamp mStartTime;