2013-01-18 04:45:11 +04:00
|
|
|
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
|
|
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
|
2012-05-21 15:12:37 +04:00
|
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
#include "prio.h"
|
2015-09-16 06:49:53 +03:00
|
|
|
#include "PLDHashTable.h"
|
2014-04-09 08:57:52 +04:00
|
|
|
#include "mozilla/IOInterposer.h"
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
#include "mozilla/AutoMemMap.h"
|
|
|
|
#include "mozilla/IOBuffers.h"
|
2013-06-23 16:03:39 +04:00
|
|
|
#include "mozilla/MemoryReporting.h"
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
#include "mozilla/MemUtils.h"
|
|
|
|
#include "mozilla/ResultExtensions.h"
|
2010-08-12 23:37:44 +04:00
|
|
|
#include "mozilla/scache/StartupCache.h"
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
#include "mozilla/ScopeExit.h"
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
#include "nsAutoPtr.h"
|
|
|
|
#include "nsClassHashtable.h"
|
|
|
|
#include "nsComponentManagerUtils.h"
|
2019-03-05 19:52:57 +03:00
|
|
|
#include "nsCRT.h"
|
2010-08-12 23:37:44 +04:00
|
|
|
#include "nsDirectoryServiceUtils.h"
|
|
|
|
#include "nsIClassInfo.h"
|
|
|
|
#include "nsIFile.h"
|
|
|
|
#include "nsIObserver.h"
|
|
|
|
#include "nsIObserverService.h"
|
|
|
|
#include "nsIOutputStream.h"
|
|
|
|
#include "nsIStorageStream.h"
|
|
|
|
#include "nsIStreamBufferAccess.h"
|
|
|
|
#include "nsIStringStream.h"
|
|
|
|
#include "nsISupports.h"
|
|
|
|
#include "nsITimer.h"
|
|
|
|
#include "nsIZipWriter.h"
|
|
|
|
#include "nsIZipReader.h"
|
|
|
|
#include "nsZipArchive.h"
|
2011-01-07 22:04:33 +03:00
|
|
|
#include "mozilla/Omnijar.h"
|
|
|
|
#include "prenv.h"
|
2012-02-22 23:03:52 +04:00
|
|
|
#include "mozilla/Telemetry.h"
|
2011-04-01 09:24:16 +04:00
|
|
|
#include "nsThreadUtils.h"
|
|
|
|
#include "nsXULAppAPI.h"
|
2011-07-01 01:58:03 +04:00
|
|
|
#include "nsIProtocolHandler.h"
|
2017-01-05 18:34:26 +03:00
|
|
|
#include "GeckoProfiler.h"
|
2011-04-01 09:24:16 +04:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
#ifdef IS_BIG_ENDIAN
|
|
|
|
# define SC_ENDIAN "big"
|
|
|
|
#else
|
|
|
|
# define SC_ENDIAN "little"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if PR_BYTES_PER_WORD == 4
|
|
|
|
# define SC_WORDSIZE "4"
|
|
|
|
#else
|
|
|
|
# define SC_WORDSIZE "8"
|
|
|
|
#endif
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
using namespace mozilla::Compression;
|
|
|
|
|
2011-12-19 04:20:36 +04:00
|
|
|
namespace mozilla {
|
|
|
|
namespace scache {
|
|
|
|
|
2013-12-08 09:39:47 +04:00
|
|
|
MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf)
|
|
|
|
|
2013-11-07 09:35:30 +04:00
|
|
|
NS_IMETHODIMP
|
|
|
|
StartupCache::CollectReports(nsIHandleReportCallback* aHandleReport,
|
2014-05-21 10:06:54 +04:00
|
|
|
nsISupports* aData, bool aAnonymize) {
|
2016-08-24 08:23:45 +03:00
|
|
|
MOZ_COLLECT_REPORT(
|
|
|
|
"explicit/startup-cache/mapping", KIND_NONHEAP, UNITS_BYTES,
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mCacheData.nonHeapSizeOfExcludingThis(),
|
2016-08-24 08:23:45 +03:00
|
|
|
"Memory used to hold the mapping of the startup cache from file. "
|
|
|
|
"This memory is likely to be swapped out shortly after start-up.");
|
|
|
|
|
|
|
|
MOZ_COLLECT_REPORT("explicit/startup-cache/data", KIND_HEAP, UNITS_BYTES,
|
|
|
|
HeapSizeOfIncludingThis(StartupCacheMallocSizeOf),
|
|
|
|
"Memory used by the startup cache for things other than "
|
|
|
|
"the file mapping.");
|
2011-12-19 04:20:36 +04:00
|
|
|
|
2013-11-07 09:35:30 +04:00
|
|
|
return NS_OK;
|
2014-04-15 11:28:21 +04:00
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
static const uint8_t MAGIC[] = "startupcache0002";
|
|
|
|
// This is a heuristic value for how much to reserve for mTable to avoid
|
|
|
|
// rehashing. This is not a hard limit in release builds, but it is in
|
|
|
|
// debug builds as it should be stable. If we exceed this number we should
|
|
|
|
// just increase it.
|
2019-10-04 23:45:18 +03:00
|
|
|
static const size_t STARTUP_CACHE_RESERVE_CAPACITY = 450;
|
|
|
|
// This is a hard limit which we will assert on, to ensure that we don't
|
|
|
|
// have some bug causing runaway cache growth.
|
|
|
|
static const size_t STARTUP_CACHE_MAX_CAPACITY = 5000;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
|
2015-05-16 11:07:10 +03:00
|
|
|
#define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN
|
2013-11-02 03:21:33 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
static inline Result<Ok, nsresult> Write(PRFileDesc* fd, const void* data,
|
|
|
|
int32_t len) {
|
|
|
|
if (PR_Write(fd, data, len) != len) {
|
|
|
|
return Err(NS_ERROR_FAILURE);
|
|
|
|
}
|
|
|
|
return Ok();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline Result<Ok, nsresult> Seek(PRFileDesc* fd, int32_t offset) {
|
|
|
|
if (PR_Seek(fd, offset, PR_SEEK_SET) == -1) {
|
|
|
|
return Err(NS_ERROR_FAILURE);
|
|
|
|
}
|
|
|
|
return Ok();
|
|
|
|
}
|
|
|
|
|
|
|
|
static nsresult MapLZ4ErrorToNsresult(size_t aError) {
|
|
|
|
return NS_ERROR_FAILURE;
|
|
|
|
}
|
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
StartupCache* StartupCache::GetSingleton() {
|
2012-10-27 02:27:53 +04:00
|
|
|
if (!gStartupCache) {
|
2015-07-04 04:29:00 +03:00
|
|
|
if (!XRE_IsParentProcess()) {
|
2012-10-27 02:27:53 +04:00
|
|
|
return nullptr;
|
|
|
|
}
|
2014-12-17 05:18:21 +03:00
|
|
|
#ifdef MOZ_DISABLE_STARTUPCACHE
|
2014-04-21 21:20:49 +04:00
|
|
|
return nullptr;
|
2016-03-19 10:10:16 +03:00
|
|
|
#else
|
2010-08-12 23:37:44 +04:00
|
|
|
StartupCache::InitSingleton();
|
2016-03-19 10:10:16 +03:00
|
|
|
#endif
|
2012-10-27 02:27:53 +04:00
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
return StartupCache::gStartupCache;
|
|
|
|
}
|
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
void StartupCache::DeleteSingleton() { StartupCache::gStartupCache = nullptr; }
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
nsresult StartupCache::InitSingleton() {
|
2010-08-12 23:37:44 +04:00
|
|
|
nsresult rv;
|
|
|
|
StartupCache::gStartupCache = new StartupCache();
|
|
|
|
|
|
|
|
rv = StartupCache::gStartupCache->Init();
|
|
|
|
if (NS_FAILED(rv)) {
|
2012-07-30 18:20:58 +04:00
|
|
|
StartupCache::gStartupCache = nullptr;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
return rv;
|
|
|
|
}
|
|
|
|
|
2013-11-26 03:57:40 +04:00
|
|
|
StaticRefPtr<StartupCache> StartupCache::gStartupCache;
|
2011-09-29 10:19:26 +04:00
|
|
|
bool StartupCache::gShutdownInitiated;
|
2012-10-11 12:17:15 +04:00
|
|
|
bool StartupCache::gIgnoreDiskCache;
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2014-04-27 11:06:00 +04:00
|
|
|
NS_IMPL_ISUPPORTS(StartupCache, nsIMemoryReporter)
|
2013-11-26 03:57:40 +04:00
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
StartupCache::StartupCache()
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
: mDirty(false),
|
|
|
|
mStartupWriteInitiated(false),
|
|
|
|
mWrittenOnce(false),
|
|
|
|
mCurTableReferenced(false),
|
2019-10-04 23:45:18 +03:00
|
|
|
mRequestedCount(0),
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mCacheEntriesBaseOffset(0),
|
2019-10-04 23:45:32 +03:00
|
|
|
mWriteThread(nullptr),
|
|
|
|
mPrefetchThread(nullptr) {}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
StartupCache::~StartupCache() {
|
2010-09-24 01:45:26 +04:00
|
|
|
if (mTimer) {
|
|
|
|
mTimer->Cancel();
|
|
|
|
}
|
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
// Generally, the in-memory table should be empty here,
|
2013-01-18 04:45:11 +04:00
|
|
|
// but an early shutdown means either mTimer didn't run
|
2011-04-01 09:24:16 +04:00
|
|
|
// or the write thread is still running.
|
|
|
|
WaitOnWriteThread();
|
2019-10-04 23:45:32 +03:00
|
|
|
WaitOnPrefetchThread();
|
2012-12-05 23:09:30 +04:00
|
|
|
|
|
|
|
// If we shutdown quickly timer wont have fired. Instead of writing
|
|
|
|
// it on the main thread and block the shutdown we simply wont update
|
|
|
|
// the startup cache. Always do this if the file doesn't exist since
|
2014-01-11 04:16:25 +04:00
|
|
|
// we use it part of the package step.
|
2019-10-04 23:45:18 +03:00
|
|
|
if (!mCacheData.initialized() || ShouldCompactCache()) {
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mDirty = true;
|
|
|
|
auto result = WriteToDisk();
|
|
|
|
Unused << NS_WARN_IF(result.isErr());
|
2012-12-05 23:09:30 +04:00
|
|
|
}
|
|
|
|
|
2013-11-07 09:35:30 +04:00
|
|
|
UnregisterWeakMemoryReporter(this);
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
nsresult StartupCache::Init() {
|
2011-07-01 01:58:03 +04:00
|
|
|
// workaround for bug 653936
|
|
|
|
nsCOMPtr<nsIProtocolHandler> jarInitializer(
|
|
|
|
do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "jar"));
|
2013-01-18 04:45:11 +04:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
nsresult rv;
|
|
|
|
|
2011-01-07 22:04:33 +03:00
|
|
|
// This allows to override the startup cache filename
|
|
|
|
// which is useful from xpcshell, when there is no ProfLDS directory to keep
|
|
|
|
// cache in.
|
|
|
|
char* env = PR_GetEnv("MOZ_STARTUP_CACHE");
|
2016-11-11 19:57:08 +03:00
|
|
|
if (env && *env) {
|
2011-10-17 18:59:28 +04:00
|
|
|
rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false,
|
|
|
|
getter_AddRefs(mFile));
|
2011-01-07 22:04:33 +03:00
|
|
|
} else {
|
|
|
|
nsCOMPtr<nsIFile> file;
|
|
|
|
rv = NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(file));
|
|
|
|
if (NS_FAILED(rv)) {
|
|
|
|
// return silently, this will fail in mochitests's xpcshell process.
|
|
|
|
return rv;
|
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2011-01-07 22:04:33 +03:00
|
|
|
rv = file->AppendNative(NS_LITERAL_CSTRING("startupCache"));
|
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2011-01-07 22:04:33 +03:00
|
|
|
// Try to create the directory if it's not there yet
|
|
|
|
rv = file->Create(nsIFile::DIRECTORY_TYPE, 0777);
|
|
|
|
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) return rv;
|
|
|
|
|
2015-05-16 11:07:10 +03:00
|
|
|
rv = file->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME));
|
2011-01-07 22:04:33 +03:00
|
|
|
|
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
2013-01-18 04:45:11 +04:00
|
|
|
|
2018-10-02 00:38:01 +03:00
|
|
|
mFile = file;
|
2011-01-07 22:04:33 +03:00
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
NS_ENSURE_TRUE(mFile, NS_ERROR_UNEXPECTED);
|
|
|
|
|
|
|
|
mObserverService = do_GetService("@mozilla.org/observer-service;1");
|
2013-01-18 04:45:11 +04:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
if (!mObserverService) {
|
|
|
|
NS_WARNING("Could not get observerService.");
|
|
|
|
return NS_ERROR_UNEXPECTED;
|
|
|
|
}
|
2013-01-18 04:45:11 +04:00
|
|
|
|
|
|
|
mListener = new StartupCacheListener();
|
2010-08-12 23:37:44 +04:00
|
|
|
rv = mObserverService->AddObserver(mListener, NS_XPCOM_SHUTDOWN_OBSERVER_ID,
|
2011-10-17 18:59:28 +04:00
|
|
|
false);
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
2010-09-15 04:39:07 +04:00
|
|
|
rv = mObserverService->AddObserver(mListener, "startupcache-invalidate",
|
2011-10-17 18:59:28 +04:00
|
|
|
false);
|
2010-09-15 04:39:07 +04:00
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
2012-10-11 12:17:15 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
auto result = LoadArchive();
|
|
|
|
rv = result.isErr() ? result.unwrapErr() : NS_OK;
|
2013-01-18 04:45:11 +04:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
// Sometimes we don't have a cache yet, that's ok.
|
|
|
|
// If it's corrupted, just remove it and start over.
|
2012-10-11 12:17:15 +04:00
|
|
|
if (gIgnoreDiskCache || (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND)) {
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_WARNING("Failed to load startupcache file correctly, removing!");
|
|
|
|
InvalidateCache();
|
|
|
|
}
|
2011-10-25 04:50:47 +04:00
|
|
|
|
2013-11-07 09:35:30 +04:00
|
|
|
RegisterWeakMemoryReporter(this);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mDecompressionContext = MakeUnique<LZ4FrameDecompressionContext>(true);
|
2011-10-25 04:50:47 +04:00
|
|
|
|
2011-01-07 21:55:14 +03:00
|
|
|
return NS_OK;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:32 +03:00
|
|
|
void StartupCache::StartPrefetchMemoryThread() {
|
|
|
|
// XXX: It would be great for this to not create its own thread, unfortunately
|
|
|
|
// there doesn't seem to be an existing thread that makes sense for this, so
|
|
|
|
// barring a coordinated global scheduling system this is the best we get.
|
|
|
|
mPrefetchThread = PR_CreateThread(
|
|
|
|
PR_USER_THREAD, StartupCache::ThreadedPrefetch, this, PR_PRIORITY_NORMAL,
|
2019-10-04 23:45:41 +03:00
|
|
|
PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 256 * 1024);
|
2019-10-04 23:45:32 +03:00
|
|
|
}
|
|
|
|
|
2013-01-18 04:45:11 +04:00
|
|
|
/**
|
2011-04-01 09:24:16 +04:00
|
|
|
* LoadArchive can be called from the main thread or while reloading cache on
|
|
|
|
* write thread.
|
|
|
|
*/
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
Result<Ok, nsresult> StartupCache::LoadArchive() {
|
|
|
|
if (gIgnoreDiskCache) return Err(NS_ERROR_FAILURE);
|
2012-10-11 12:17:15 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
MOZ_TRY(mCacheData.init(mFile));
|
|
|
|
auto size = mCacheData.size();
|
2019-10-04 23:45:32 +03:00
|
|
|
if (CanPrefetchMemory()) {
|
|
|
|
StartPrefetchMemoryThread();
|
|
|
|
}
|
2019-05-21 20:32:45 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
uint32_t headerSize;
|
|
|
|
if (size < sizeof(MAGIC) + sizeof(headerSize)) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
}
|
2011-12-08 14:03:36 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
auto data = mCacheData.get<uint8_t>();
|
|
|
|
auto end = data + size;
|
2011-12-08 14:03:36 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (memcmp(MAGIC, data.get(), sizeof(MAGIC))) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
|
|
|
data += sizeof(MAGIC);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
headerSize = LittleEndian::readUint32(data.get());
|
|
|
|
data += sizeof(headerSize);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
|
2019-10-22 06:15:18 +03:00
|
|
|
if (headerSize > end - data) {
|
|
|
|
MOZ_ASSERT(false, "StartupCache file is corrupt.");
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
Range<uint8_t> header(data, data + headerSize);
|
|
|
|
data += headerSize;
|
|
|
|
|
|
|
|
mCacheEntriesBaseOffset = sizeof(MAGIC) + sizeof(headerSize) + headerSize;
|
|
|
|
{
|
2019-10-04 23:45:18 +03:00
|
|
|
if (!mTable.reserve(STARTUP_CACHE_RESERVE_CAPACITY)) {
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
|
|
|
auto cleanup = MakeScopeExit([&]() {
|
|
|
|
mTable.clear();
|
|
|
|
mCacheData.reset();
|
|
|
|
});
|
|
|
|
loader::InputBuffer buf(header);
|
|
|
|
|
|
|
|
uint32_t currentOffset = 0;
|
|
|
|
while (!buf.finished()) {
|
|
|
|
uint32_t offset = 0;
|
|
|
|
uint32_t compressedSize = 0;
|
|
|
|
uint32_t uncompressedSize = 0;
|
|
|
|
nsCString key;
|
|
|
|
buf.codeUint32(offset);
|
|
|
|
buf.codeUint32(compressedSize);
|
|
|
|
buf.codeUint32(uncompressedSize);
|
|
|
|
buf.codeString(key);
|
|
|
|
|
2019-10-22 06:15:18 +03:00
|
|
|
if (offset + compressedSize > end - data) {
|
|
|
|
MOZ_ASSERT(false, "StartupCache file is corrupt.");
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure offsets match what we'd expect based on script ordering and
|
|
|
|
// size, as a basic sanity check.
|
|
|
|
if (offset != currentOffset) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
|
|
|
currentOffset += compressedSize;
|
|
|
|
|
|
|
|
if (!mTable.putNew(key, StartupCacheEntry(offset, compressedSize,
|
|
|
|
uncompressedSize))) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (buf.error()) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
|
|
|
}
|
2011-12-08 14:03:36 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
cleanup.release();
|
|
|
|
}
|
|
|
|
|
|
|
|
return Ok();
|
|
|
|
}
|
2019-09-28 01:15:01 +03:00
|
|
|
|
2019-10-04 23:44:32 +03:00
|
|
|
bool StartupCache::HasEntry(const char* id) {
|
|
|
|
AUTO_PROFILER_LABEL("StartupCache::HasEntry", OTHER);
|
|
|
|
|
|
|
|
MOZ_ASSERT(NS_IsMainThread(), "Startup cache only available on main thread");
|
|
|
|
WaitOnWriteThread();
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return mTable.has(nsDependentCString(id));
|
2019-10-04 23:44:32 +03:00
|
|
|
}
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
nsresult StartupCache::GetBuffer(const char* id, const char** outbuf,
|
2017-07-06 15:00:35 +03:00
|
|
|
uint32_t* length) {
|
Bug 1375392 - Tweak the PROFILER_LABEL* macros. r=mstange.
This patch makes the following changes to the macros.
- Removes PROFILER_LABEL_FUNC. It's only suitable for use in functions outside
classes, due to PROFILER_FUNCTION_NAME not getting class names, and it was
mostly misused.
- Removes PROFILER_FUNCTION_NAME. It's no longer used, and __func__ is
universally available now anyway.
- Combines the first two string literal arguments of PROFILER_LABEL and
PROFILER_LABEL_DYNAMIC into a single argument. There was no good reason for
them to be separate, and it forced a '::' in the label, which isn't always
appropriate. Also, the meaning of the "name_space" argument was interpreted
in an interesting variety of ways.
- Adds an "AUTO_" prefix to PROFILER_LABEL and PROFILER_LABEL_DYNAMIC, to make
it clearer they construct RAII objects rather than just being function calls.
(I myself have screwed up the scoping because of this in the past.)
- Fills in the 'js::ProfileEntry::Category::' qualifier within the macro, so
the caller doesn't need to. This makes a *lot* more of the uses fit onto a
single line.
The patch also makes the following changes to the macro uses (beyond those
required by the changes described above).
- Fixes a bunch of labels that had gotten out of sync with the name of the
class and/or function that encloses them.
- Removes a useless PROFILER_LABEL use within a trivial scope in
EventStateManager::DispatchMouseOrPointerEvent(). It clearly wasn't serving
any useful purpose. It also serves as extra evidence that the AUTO_ prefix is
a good idea.
- Tweaks DecodePool::SyncRunIf{Preferred,Possible} so that the labelling is
done within them, instead of at their callsites, because that's a more
standard way of doing things.
--HG--
extra : rebase_source : 318d1bc6fc1425a94aacbf489dd46e4f83211de4
2017-06-22 10:08:53 +03:00
|
|
|
AUTO_PROFILER_LABEL("StartupCache::GetBuffer", OTHER);
|
2016-01-15 13:38:03 +03:00
|
|
|
|
2011-04-01 09:24:16 +04:00
|
|
|
NS_ASSERTION(NS_IsMainThread(),
|
|
|
|
"Startup cache only available on main thread");
|
2016-01-15 13:38:03 +03:00
|
|
|
|
2011-01-21 00:40:45 +03:00
|
|
|
WaitOnWriteThread();
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
Telemetry::LABELS_STARTUP_CACHE_REQUESTS label =
|
|
|
|
Telemetry::LABELS_STARTUP_CACHE_REQUESTS::Miss;
|
|
|
|
auto telemetry =
|
|
|
|
MakeScopeExit([&label] { Telemetry::AccumulateCategorical(label); });
|
|
|
|
|
|
|
|
decltype(mTable)::Ptr p = mTable.lookup(nsDependentCString(id));
|
|
|
|
if (!p) {
|
|
|
|
return NS_ERROR_NOT_AVAILABLE;
|
|
|
|
}
|
|
|
|
|
|
|
|
auto& value = p->value();
|
|
|
|
if (value.mData) {
|
|
|
|
label = Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitMemory;
|
|
|
|
} else {
|
|
|
|
if (!mCacheData.initialized()) {
|
|
|
|
return NS_ERROR_NOT_AVAILABLE;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t totalRead = 0;
|
|
|
|
size_t totalWritten = 0;
|
|
|
|
Span<const char> compressed = MakeSpan(
|
|
|
|
mCacheData.get<char>().get() + mCacheEntriesBaseOffset + value.mOffset,
|
|
|
|
value.mCompressedSize);
|
|
|
|
value.mData = MakeUnique<char[]>(value.mUncompressedSize);
|
|
|
|
Span<char> uncompressed =
|
|
|
|
MakeSpan(value.mData.get(), value.mUncompressedSize);
|
|
|
|
bool finished = false;
|
|
|
|
while (!finished) {
|
|
|
|
auto result = mDecompressionContext->Decompress(
|
|
|
|
uncompressed.From(totalWritten), compressed.From(totalRead));
|
|
|
|
if (NS_WARN_IF(result.isErr())) {
|
|
|
|
value.mData = nullptr;
|
|
|
|
InvalidateCache();
|
|
|
|
return NS_ERROR_FAILURE;
|
|
|
|
}
|
|
|
|
auto decompressionResult = result.unwrap();
|
|
|
|
totalRead += decompressionResult.mSizeRead;
|
|
|
|
totalWritten += decompressionResult.mSizeWritten;
|
|
|
|
finished = decompressionResult.mFinished;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
|
|
|
|
label = Telemetry::LABELS_STARTUP_CACHE_REQUESTS::HitDisk;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (!value.mRequested) {
|
|
|
|
value.mRequested = true;
|
|
|
|
value.mRequestedOrder = ++mRequestedCount;
|
2019-10-04 23:45:18 +03:00
|
|
|
MOZ_ASSERT(mRequestedCount <= mTable.count(),
|
|
|
|
"Somehow we requested more StartupCache items than exist.");
|
|
|
|
ResetStartupWriteTimerCheckingReadCount();
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// Track that something holds a reference into mTable, so we know to hold
|
|
|
|
// onto it in case the cache is invalidated.
|
|
|
|
mCurTableReferenced = true;
|
2019-10-04 23:45:18 +03:00
|
|
|
*outbuf = value.mData.get();
|
|
|
|
*length = value.mUncompressedSize;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return NS_OK;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// Makes a copy of the buffer, client retains ownership of inbuf.
|
2018-02-17 02:30:47 +03:00
|
|
|
nsresult StartupCache::PutBuffer(const char* id, UniquePtr<char[]>&& inbuf,
|
|
|
|
uint32_t len) {
|
2011-04-01 09:24:16 +04:00
|
|
|
NS_ASSERTION(NS_IsMainThread(),
|
|
|
|
"Startup cache only available on main thread");
|
2011-01-21 00:40:45 +03:00
|
|
|
WaitOnWriteThread();
|
2010-08-12 23:37:44 +04:00
|
|
|
if (StartupCache::gShutdownInitiated) {
|
|
|
|
return NS_ERROR_NOT_AVAILABLE;
|
|
|
|
}
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
bool exists = mTable.has(nsDependentCString(id));
|
2017-07-06 15:00:35 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (exists) {
|
2016-08-17 08:26:40 +03:00
|
|
|
NS_WARNING("Existing entry in StartupCache.");
|
2015-05-22 23:29:51 +03:00
|
|
|
// Double-caching is undesirable but not an error.
|
|
|
|
return NS_OK;
|
|
|
|
}
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
// putNew returns false on alloc failure - in the very unlikely event we hit
|
|
|
|
// that and aren't going to crash elsewhere, there's no reason we need to
|
|
|
|
// crash here.
|
|
|
|
if (mTable.putNew(nsCString(id), StartupCacheEntry(std::move(inbuf), len,
|
|
|
|
++mRequestedCount))) {
|
|
|
|
return ResetStartupWriteTimer();
|
2011-01-07 21:55:14 +03:00
|
|
|
}
|
2019-10-04 23:45:18 +03:00
|
|
|
MOZ_DIAGNOSTIC_ASSERT(mTable.count() < STARTUP_CACHE_MAX_CAPACITY,
|
|
|
|
"Too many StartupCache entries.");
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return NS_OK;
|
2011-10-25 04:50:47 +04:00
|
|
|
}
|
|
|
|
|
2015-07-31 07:19:57 +03:00
|
|
|
size_t StartupCache::HeapSizeOfIncludingThis(
|
|
|
|
mozilla::MallocSizeOf aMallocSizeOf) const {
|
2011-12-19 04:20:36 +04:00
|
|
|
// This function could measure more members, but they haven't been found by
|
|
|
|
// DMD to be significant. They can be added later if necessary.
|
|
|
|
|
2015-07-31 07:19:57 +03:00
|
|
|
size_t n = aMallocSizeOf(this);
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
n += mTable.shallowSizeOfExcludingThis(aMallocSizeOf);
|
2019-10-16 18:41:54 +03:00
|
|
|
for (auto iter = mTable.iter(); !iter.done(); iter.next()) {
|
|
|
|
if (iter.get().value().mData) {
|
|
|
|
n += aMallocSizeOf(iter.get().value().mData.get());
|
|
|
|
}
|
|
|
|
n += iter.get().key().SizeOfExcludingThisIfUnshared(aMallocSizeOf);
|
|
|
|
}
|
2015-07-31 07:19:57 +03:00
|
|
|
|
|
|
|
return n;
|
2011-12-19 04:20:36 +04:00
|
|
|
}
|
|
|
|
|
2017-07-06 15:00:35 +03:00
|
|
|
/**
|
2011-04-01 09:24:16 +04:00
|
|
|
* WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call
|
|
|
|
* WaitOnWriteThread to make sure there isn't a write happening on another
|
|
|
|
* thread
|
|
|
|
*/
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
Result<Ok, nsresult> StartupCache::WriteToDisk() {
|
2011-10-17 18:59:28 +04:00
|
|
|
mStartupWriteInitiated = true;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (!mDirty || mWrittenOnce) {
|
|
|
|
return Ok();
|
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (!mFile) {
|
|
|
|
return Err(NS_ERROR_UNEXPECTED);
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
AutoFDClose fd;
|
|
|
|
MOZ_TRY(mFile->OpenNSPRFileDesc(PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE,
|
|
|
|
0644, &fd.rwget()));
|
|
|
|
|
|
|
|
nsTArray<Pair<const nsCString*, StartupCacheEntry*>> entries;
|
|
|
|
for (auto iter = mTable.iter(); !iter.done(); iter.next()) {
|
2019-10-04 23:45:18 +03:00
|
|
|
if (iter.get().value().mRequested) {
|
|
|
|
entries.AppendElement(MakePair(&iter.get().key(), &iter.get().value()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (entries.IsEmpty()) {
|
|
|
|
return Ok();
|
2019-09-29 01:14:31 +03:00
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
entries.Sort(StartupCacheEntry::Comparator());
|
|
|
|
loader::OutputBuffer buf;
|
|
|
|
for (auto& e : entries) {
|
|
|
|
auto key = e.first();
|
|
|
|
auto value = e.second();
|
|
|
|
auto uncompressedSize = value->mUncompressedSize;
|
|
|
|
// Set the mHeaderOffsetInFile so we can go back and edit the offset.
|
|
|
|
value->mHeaderOffsetInFile = buf.cursor();
|
|
|
|
// Write a 0 offset/compressed size as a placeholder until we get the real
|
|
|
|
// offset after compressing.
|
|
|
|
buf.codeUint32(0);
|
|
|
|
buf.codeUint32(0);
|
|
|
|
buf.codeUint32(uncompressedSize);
|
|
|
|
buf.codeString(*key);
|
2019-09-29 01:14:31 +03:00
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
uint8_t headerSize[4];
|
|
|
|
LittleEndian::writeUint32(headerSize, buf.cursor());
|
|
|
|
|
|
|
|
MOZ_TRY(Write(fd, MAGIC, sizeof(MAGIC)));
|
|
|
|
MOZ_TRY(Write(fd, headerSize, sizeof(headerSize)));
|
|
|
|
size_t headerStart = sizeof(MAGIC) + sizeof(headerSize);
|
|
|
|
size_t dataStart = headerStart + buf.cursor();
|
|
|
|
MOZ_TRY(Seek(fd, dataStart));
|
|
|
|
|
|
|
|
size_t offset = 0;
|
|
|
|
|
|
|
|
const size_t chunkSize = 1024 * 16;
|
|
|
|
LZ4FrameCompressionContext ctx(6, /* aCompressionLevel */
|
|
|
|
chunkSize, /* aReadBufLen */
|
|
|
|
true, /* aChecksum */
|
|
|
|
true); /* aStableSrc */
|
|
|
|
size_t writeBufLen = ctx.GetRequiredWriteBufferLength();
|
|
|
|
auto writeBuffer = MakeUnique<char[]>(writeBufLen);
|
|
|
|
auto writeSpan = MakeSpan(writeBuffer.get(), writeBufLen);
|
|
|
|
|
|
|
|
for (auto& e : entries) {
|
|
|
|
auto value = e.second();
|
|
|
|
value->mOffset = offset;
|
|
|
|
Span<const char> result;
|
|
|
|
MOZ_TRY_VAR(result,
|
|
|
|
ctx.BeginCompressing(writeSpan).mapErr(MapLZ4ErrorToNsresult));
|
|
|
|
MOZ_TRY(Write(fd, result.Elements(), result.Length()));
|
|
|
|
offset += result.Length();
|
|
|
|
|
|
|
|
for (size_t i = 0; i < value->mUncompressedSize; i += chunkSize) {
|
|
|
|
size_t size = std::min(chunkSize, value->mUncompressedSize - i);
|
|
|
|
char* uncompressed = value->mData.get() + i;
|
|
|
|
MOZ_TRY_VAR(result, ctx.ContinueCompressing(MakeSpan(uncompressed, size))
|
|
|
|
.mapErr(MapLZ4ErrorToNsresult));
|
|
|
|
MOZ_TRY(Write(fd, result.Elements(), result.Length()));
|
|
|
|
offset += result.Length();
|
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
MOZ_TRY_VAR(result, ctx.EndCompressing().mapErr(MapLZ4ErrorToNsresult));
|
|
|
|
MOZ_TRY(Write(fd, result.Elements(), result.Length()));
|
|
|
|
offset += result.Length();
|
|
|
|
value->mCompressedSize = offset - value->mOffset;
|
|
|
|
MOZ_TRY(Seek(fd, dataStart + offset));
|
2015-05-22 23:29:51 +03:00
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
for (auto& e : entries) {
|
|
|
|
auto value = e.second();
|
|
|
|
uint8_t* headerEntry = buf.Get() + value->mHeaderOffsetInFile;
|
|
|
|
LittleEndian::writeUint32(headerEntry, value->mOffset);
|
|
|
|
LittleEndian::writeUint32(headerEntry + sizeof(value->mOffset),
|
|
|
|
value->mCompressedSize);
|
|
|
|
}
|
|
|
|
MOZ_TRY(Seek(fd, headerStart));
|
|
|
|
MOZ_TRY(Write(fd, buf.Get(), buf.cursor()));
|
2011-01-21 00:40:45 +03:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mDirty = false;
|
|
|
|
mWrittenOnce = true;
|
2012-10-11 12:17:15 +04:00
|
|
|
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return Ok();
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
2019-03-05 19:52:57 +03:00
|
|
|
void StartupCache::InvalidateCache(bool memoryOnly) {
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
WaitOnWriteThread();
|
|
|
|
mWrittenOnce = false;
|
2019-03-05 19:52:57 +03:00
|
|
|
if (memoryOnly) {
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
auto writeResult = WriteToDisk();
|
|
|
|
if (NS_WARN_IF(writeResult.isErr())) {
|
|
|
|
gIgnoreDiskCache = true;
|
|
|
|
return;
|
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-09-28 01:15:35 +03:00
|
|
|
}
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (mCurTableReferenced) {
|
|
|
|
// There should be no way for this assert to fail other than a user manually
|
|
|
|
// sending startupcache-invalidate messages through the Browser Toolbox.
|
|
|
|
MOZ_DIAGNOSTIC_ASSERT(xpc::IsInAutomation() || mOldTables.Length() < 10,
|
|
|
|
"Startup cache invalidated too many times.");
|
|
|
|
mOldTables.AppendElement(std::move(mTable));
|
|
|
|
mCurTableReferenced = false;
|
2019-10-04 23:45:18 +03:00
|
|
|
} else {
|
|
|
|
mTable.clear();
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
}
|
2019-10-04 23:45:18 +03:00
|
|
|
mRequestedCount = 0;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (!memoryOnly) {
|
|
|
|
mCacheData.reset();
|
|
|
|
nsresult rv = mFile->Remove(false);
|
|
|
|
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST &&
|
|
|
|
rv != NS_ERROR_FILE_NOT_FOUND) {
|
|
|
|
gIgnoreDiskCache = true;
|
|
|
|
return;
|
|
|
|
}
|
2012-10-11 12:17:15 +04:00
|
|
|
}
|
|
|
|
gIgnoreDiskCache = false;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
auto result = LoadArchive();
|
|
|
|
if (NS_WARN_IF(result.isErr())) {
|
|
|
|
gIgnoreDiskCache = true;
|
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
2012-10-11 12:17:15 +04:00
|
|
|
void StartupCache::IgnoreDiskCache() {
|
|
|
|
gIgnoreDiskCache = true;
|
|
|
|
if (gStartupCache) gStartupCache->InvalidateCache();
|
|
|
|
}
|
|
|
|
|
2011-01-21 00:40:45 +03:00
|
|
|
/*
|
|
|
|
* WaitOnWriteThread() is called from a main thread to wait for the worker
|
|
|
|
* thread to finish. However since the same code is used in the worker thread
|
|
|
|
* and main thread, the worker thread can also call WaitOnWriteThread() which is
|
|
|
|
* a no-op.
|
|
|
|
*/
|
|
|
|
void StartupCache::WaitOnWriteThread() {
|
2011-04-01 09:24:16 +04:00
|
|
|
NS_ASSERTION(NS_IsMainThread(),
|
|
|
|
"Startup cache should only wait for io thread on main thread");
|
|
|
|
if (!mWriteThread || mWriteThread == PR_GetCurrentThread()) return;
|
2011-01-21 00:40:45 +03:00
|
|
|
|
2011-04-01 09:24:16 +04:00
|
|
|
PR_JoinThread(mWriteThread);
|
2013-07-31 19:44:43 +04:00
|
|
|
mWriteThread = nullptr;
|
2011-01-21 00:40:45 +03:00
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:32 +03:00
|
|
|
void StartupCache::WaitOnPrefetchThread() {
|
|
|
|
if (!mPrefetchThread || mPrefetchThread == PR_GetCurrentThread()) return;
|
|
|
|
|
|
|
|
PR_JoinThread(mPrefetchThread);
|
|
|
|
mPrefetchThread = nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StartupCache::ThreadedPrefetch(void* aClosure) {
|
|
|
|
AUTO_PROFILER_REGISTER_THREAD("StartupCache");
|
|
|
|
NS_SetCurrentThreadName("StartupCache");
|
|
|
|
mozilla::IOInterposer::RegisterCurrentThread();
|
|
|
|
StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
|
|
|
|
PrefetchMemory(startupCacheObj->mCacheData.get<uint8_t>().get(),
|
|
|
|
startupCacheObj->mCacheData.size());
|
|
|
|
mozilla::IOInterposer::UnregisterCurrentThread();
|
|
|
|
}
|
|
|
|
|
2011-01-21 00:40:45 +03:00
|
|
|
void StartupCache::ThreadedWrite(void* aClosure) {
|
2017-10-04 01:11:18 +03:00
|
|
|
AUTO_PROFILER_REGISTER_THREAD("StartupCache");
|
2017-02-07 13:57:23 +03:00
|
|
|
NS_SetCurrentThreadName("StartupCache");
|
2014-04-09 08:57:52 +04:00
|
|
|
mozilla::IOInterposer::RegisterCurrentThread();
|
2014-01-11 04:16:25 +04:00
|
|
|
/*
|
|
|
|
* It is safe to use the pointer passed in aClosure to reference the
|
|
|
|
* StartupCache object because the thread's lifetime is tightly coupled to
|
|
|
|
* the lifetime of the StartupCache object; this thread is joined in the
|
|
|
|
* StartupCache destructor, guaranteeing that this function runs if and only
|
|
|
|
* if the StartupCache object is valid.
|
|
|
|
*/
|
|
|
|
StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
auto result = startupCacheObj->WriteToDisk();
|
|
|
|
Unused << NS_WARN_IF(result.isErr());
|
2014-04-09 08:57:52 +04:00
|
|
|
mozilla::IOInterposer::UnregisterCurrentThread();
|
2011-01-21 00:40:45 +03:00
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:18 +03:00
|
|
|
bool StartupCache::ShouldCompactCache() {
|
|
|
|
// If we've requested less than 4/5 of the startup cache, then we should
|
|
|
|
// probably compact it down. This can happen quite easily after the first run,
|
|
|
|
// which seems to request quite a few more things than subsequent runs.
|
|
|
|
CheckedInt<uint32_t> threshold = CheckedInt<uint32_t>(mTable.count()) * 4 / 5;
|
|
|
|
MOZ_RELEASE_ASSERT(threshold.isValid(), "Runaway StartupCache size");
|
|
|
|
return mRequestedCount < threshold.value();
|
|
|
|
}
|
|
|
|
|
2011-01-21 00:40:45 +03:00
|
|
|
/*
|
|
|
|
* The write-thread is spawned on a timeout(which is reset with every write).
|
|
|
|
* This can avoid a slow shutdown. After writing out the cache, the zipreader is
|
|
|
|
* reloaded on the worker thread.
|
|
|
|
*/
|
2010-08-12 23:37:44 +04:00
|
|
|
void StartupCache::WriteTimeout(nsITimer* aTimer, void* aClosure) {
|
2014-01-11 04:16:25 +04:00
|
|
|
/*
|
|
|
|
* It is safe to use the pointer passed in aClosure to reference the
|
|
|
|
* StartupCache object because the timer's lifetime is tightly coupled to
|
|
|
|
* the lifetime of the StartupCache object; this timer is canceled in the
|
|
|
|
* StartupCache destructor, guaranteeing that this function runs if and only
|
|
|
|
* if the StartupCache object is valid.
|
|
|
|
*/
|
|
|
|
StartupCache* startupCacheObj = static_cast<StartupCache*>(aClosure);
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
if (startupCacheObj->mWrittenOnce) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:18 +03:00
|
|
|
if (startupCacheObj->mCacheData.initialized() &&
|
|
|
|
!startupCacheObj->ShouldCompactCache()) {
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:32 +03:00
|
|
|
startupCacheObj->WaitOnPrefetchThread();
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
startupCacheObj->mStartupWriteInitiated = false;
|
|
|
|
startupCacheObj->mDirty = true;
|
|
|
|
startupCacheObj->mCacheData.reset();
|
2019-09-29 01:14:31 +03:00
|
|
|
startupCacheObj->mWriteThread = PR_CreateThread(
|
|
|
|
PR_USER_THREAD, StartupCache::ThreadedWrite, startupCacheObj,
|
2019-10-10 08:39:15 +03:00
|
|
|
PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 512 * 1024);
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// We don't want to refcount StartupCache, so we'll just
|
|
|
|
// hold a ref to this and pass it to observerService instead.
|
2014-04-27 11:06:00 +04:00
|
|
|
NS_IMPL_ISUPPORTS(StartupCacheListener, nsIObserver)
|
2010-08-12 23:37:44 +04:00
|
|
|
|
2014-01-04 19:02:17 +04:00
|
|
|
nsresult StartupCacheListener::Observe(nsISupports* subject, const char* topic,
|
|
|
|
const char16_t* data) {
|
2011-01-21 00:40:45 +03:00
|
|
|
StartupCache* sc = StartupCache::GetSingleton();
|
|
|
|
if (!sc) return NS_OK;
|
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
if (strcmp(topic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) {
|
2011-01-21 00:40:45 +03:00
|
|
|
// Do not leave the thread running past xpcom shutdown
|
|
|
|
sc->WaitOnWriteThread();
|
2011-10-17 18:59:28 +04:00
|
|
|
StartupCache::gShutdownInitiated = true;
|
2010-09-15 04:39:07 +04:00
|
|
|
} else if (strcmp(topic, "startupcache-invalidate") == 0) {
|
2019-03-05 19:52:57 +03:00
|
|
|
sc->InvalidateCache(data && nsCRT::strcmp(data, u"memoryOnly") == 0);
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
2011-01-21 00:40:45 +03:00
|
|
|
return NS_OK;
|
2017-07-06 15:00:35 +03:00
|
|
|
}
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
nsresult StartupCache::GetDebugObjectOutputStream(
|
2017-07-06 15:00:35 +03:00
|
|
|
nsIObjectOutputStream* aStream, nsIObjectOutputStream** aOutStream) {
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ENSURE_ARG_POINTER(aStream);
|
|
|
|
#ifdef DEBUG
|
2016-11-23 15:22:34 +03:00
|
|
|
auto* stream = new StartupCacheDebugOutputStream(aStream, &mWriteObjectMap);
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ADDREF(*aOutStream = stream);
|
|
|
|
#else
|
|
|
|
NS_ADDREF(*aOutStream = aStream);
|
|
|
|
#endif
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
return NS_OK;
|
|
|
|
}
|
|
|
|
|
2019-10-04 23:45:18 +03:00
|
|
|
nsresult StartupCache::ResetStartupWriteTimerCheckingReadCount() {
|
|
|
|
nsresult rv = NS_OK;
|
|
|
|
if (!mTimer)
|
|
|
|
mTimer = NS_NewTimer();
|
|
|
|
else
|
|
|
|
rv = mTimer->Cancel();
|
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Wait for 10 seconds, then write out the cache.
|
|
|
|
mTimer->InitWithNamedFuncCallback(StartupCache::WriteTimeout, this, 60000,
|
|
|
|
nsITimer::TYPE_ONE_SHOT,
|
|
|
|
"StartupCache::WriteTimeout");
|
|
|
|
return NS_OK;
|
|
|
|
}
|
|
|
|
|
2011-01-07 21:55:14 +03:00
|
|
|
nsresult StartupCache::ResetStartupWriteTimer() {
|
2011-10-17 18:59:28 +04:00
|
|
|
mStartupWriteInitiated = false;
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
mDirty = true;
|
2017-09-25 05:57:48 +03:00
|
|
|
nsresult rv = NS_OK;
|
2011-01-07 21:55:14 +03:00
|
|
|
if (!mTimer)
|
2017-09-25 05:57:48 +03:00
|
|
|
mTimer = NS_NewTimer();
|
2011-01-07 21:55:14 +03:00
|
|
|
else
|
|
|
|
rv = mTimer->Cancel();
|
|
|
|
NS_ENSURE_SUCCESS(rv, rv);
|
|
|
|
// Wait for 10 seconds, then write out the cache.
|
2017-02-14 02:45:23 +03:00
|
|
|
mTimer->InitWithNamedFuncCallback(StartupCache::WriteTimeout, this, 60000,
|
|
|
|
nsITimer::TYPE_ONE_SHOT,
|
|
|
|
"StartupCache::WriteTimeout");
|
2011-01-07 21:55:14 +03:00
|
|
|
return NS_OK;
|
|
|
|
}
|
|
|
|
|
2016-11-11 19:57:08 +03:00
|
|
|
bool StartupCache::StartupWriteComplete() {
|
|
|
|
WaitOnWriteThread();
|
Bug 1550108 - Change StartupCache format from zip to custom r=froydnj
I am not aware of anything that depends on StartupCache being a
zip file, and since I want to use lz4 compression because inflate
is showing up quite a lot in profiles, it's simplest to just use
a custom format. This loosely mimicks the ScriptPreloader code,
with a few diversions:
- Obviously the contents of the cache are compressed. I used lz4
for this as I hit the same file size as deflate at a compression
level of 1, which is what the StartupCache was using previously,
while decompressing an order of magnitude faster. Seemed like
the most conservative change to make. I think it's worth
investigating what the impact of slower algs with higher ratios
would be, but for right now I settled on this. We'd probably
want to look at zstd next.
- I use streaming compression for this via lz4frame. This is not
strictly necessary, but has the benefit of not requiring as
much memory for large buffers, as well as giving us a built-in
checksum, rather than relying on the much slower CRC that we
were doing with the zip-based approach.
- I coded the serialization of the headers inline, since I had to
jump back to add the offset and compressed size, which would
make the nice Code(...) method for the ScriptPreloader stuff
rather more complex. Open to cleaner solutions, but moving it
out just felt like extra hoops for the reader to jump through
to understand without the benefit of being more concise.
Differential Revision: https://phabricator.services.mozilla.com/D34652
--HG--
extra : moz-landing-system : lando
2019-10-04 23:44:59 +03:00
|
|
|
return mStartupWriteInitiated && !mDirty;
|
2016-11-11 19:57:08 +03:00
|
|
|
}
|
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
// StartupCacheDebugOutputStream implementation
|
|
|
|
#ifdef DEBUG
|
2017-07-06 15:00:35 +03:00
|
|
|
NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream, nsIObjectOutputStream,
|
2014-04-27 11:06:00 +04:00
|
|
|
nsIBinaryOutputStream, nsIOutputStream)
|
2010-08-12 23:37:44 +04:00
|
|
|
|
|
|
|
bool StartupCacheDebugOutputStream::CheckReferences(nsISupports* aObject) {
|
|
|
|
nsresult rv;
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
nsCOMPtr<nsIClassInfo> classInfo = do_QueryInterface(aObject);
|
|
|
|
if (!classInfo) {
|
|
|
|
NS_ERROR("aObject must implement nsIClassInfo");
|
2011-10-17 18:59:28 +04:00
|
|
|
return false;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2012-08-22 19:56:38 +04:00
|
|
|
uint32_t flags;
|
2010-08-12 23:37:44 +04:00
|
|
|
rv = classInfo->GetFlags(&flags);
|
2011-10-17 18:59:28 +04:00
|
|
|
NS_ENSURE_SUCCESS(rv, false);
|
2010-08-12 23:37:44 +04:00
|
|
|
if (flags & nsIClassInfo::SINGLETON) return true;
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
nsISupportsHashKey* key = mObjectMap->GetEntry(aObject);
|
|
|
|
if (key) {
|
2017-07-06 15:00:35 +03:00
|
|
|
NS_ERROR(
|
|
|
|
"non-singleton aObject is referenced multiple times in this"
|
2010-08-12 23:37:44 +04:00
|
|
|
"serialization, we don't support that.");
|
2011-10-17 18:59:28 +04:00
|
|
|
return false;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
mObjectMap->PutEntry(aObject);
|
2011-10-17 18:59:28 +04:00
|
|
|
return true;
|
2010-08-12 23:37:44 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// nsIObjectOutputStream implementation
|
2011-09-29 10:19:26 +04:00
|
|
|
nsresult StartupCacheDebugOutputStream::WriteObject(nsISupports* aObject,
|
|
|
|
bool aIsStrongRef) {
|
2010-08-12 23:37:44 +04:00
|
|
|
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ASSERTION(rootObject.get() == aObject,
|
|
|
|
"bad call to WriteObject -- call WriteCompoundObject!");
|
2011-09-29 10:19:26 +04:00
|
|
|
bool check = CheckReferences(aObject);
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
|
|
|
|
return mBinaryStream->WriteObject(aObject, aIsStrongRef);
|
|
|
|
}
|
|
|
|
|
|
|
|
nsresult StartupCacheDebugOutputStream::WriteSingleRefObject(
|
|
|
|
nsISupports* aObject) {
|
|
|
|
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ASSERTION(rootObject.get() == aObject,
|
|
|
|
"bad call to WriteSingleRefObject -- call WriteCompoundObject!");
|
2011-09-29 10:19:26 +04:00
|
|
|
bool check = CheckReferences(aObject);
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
|
|
|
|
return mBinaryStream->WriteSingleRefObject(aObject);
|
|
|
|
}
|
|
|
|
|
|
|
|
nsresult StartupCacheDebugOutputStream::WriteCompoundObject(
|
2011-09-29 10:19:26 +04:00
|
|
|
nsISupports* aObject, const nsIID& aIID, bool aIsStrongRef) {
|
2010-08-12 23:37:44 +04:00
|
|
|
nsCOMPtr<nsISupports> rootObject(do_QueryInterface(aObject));
|
2017-07-06 15:00:35 +03:00
|
|
|
|
2010-08-12 23:37:44 +04:00
|
|
|
nsCOMPtr<nsISupports> roundtrip;
|
|
|
|
rootObject->QueryInterface(aIID, getter_AddRefs(roundtrip));
|
|
|
|
NS_ASSERTION(roundtrip.get() == aObject,
|
|
|
|
"bad aggregation or multiple inheritance detected by call to "
|
|
|
|
"WriteCompoundObject!");
|
|
|
|
|
2011-09-29 10:19:26 +04:00
|
|
|
bool check = CheckReferences(aObject);
|
2010-08-12 23:37:44 +04:00
|
|
|
NS_ENSURE_TRUE(check, NS_ERROR_FAILURE);
|
|
|
|
return mBinaryStream->WriteCompoundObject(aObject, aIID, aIsStrongRef);
|
|
|
|
}
|
|
|
|
|
2017-07-06 15:00:35 +03:00
|
|
|
nsresult StartupCacheDebugOutputStream::WriteID(nsID const& aID) {
|
2010-08-12 23:37:44 +04:00
|
|
|
return mBinaryStream->WriteID(aID);
|
|
|
|
}
|
|
|
|
|
2012-08-22 19:56:38 +04:00
|
|
|
char* StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength,
|
|
|
|
uint32_t aAlignMask) {
|
2010-08-12 23:37:44 +04:00
|
|
|
return mBinaryStream->GetBuffer(aLength, aAlignMask);
|
|
|
|
}
|
|
|
|
|
2012-08-22 19:56:38 +04:00
|
|
|
void StartupCacheDebugOutputStream::PutBuffer(char* aBuffer, uint32_t aLength) {
|
2010-08-12 23:37:44 +04:00
|
|
|
mBinaryStream->PutBuffer(aBuffer, aLength);
|
|
|
|
}
|
|
|
|
#endif // DEBUG
|
|
|
|
|
|
|
|
} // namespace scache
|
|
|
|
} // namespace mozilla
|