зеркало из https://github.com/mozilla/gecko-dev.git
Bug 481347: Split uploads into multiple chunks to get around server upload limit and curtail memory usage
This commit is contained in:
Родитель
e041769fd8
Коммит
ddefe825fc
|
@ -40,7 +40,7 @@ const EXPORTED_SYMBOLS = ["WEAVE_VERSION", "STORAGE_VERSION",
|
|||
'MODE_CREATE', 'MODE_APPEND', 'MODE_TRUNCATE',
|
||||
'PERMS_FILE', 'PERMS_PASSFILE', 'PERMS_DIRECTORY',
|
||||
'ONE_BYTE', 'ONE_KILOBYTE', 'ONE_MEGABYTE',
|
||||
'CONNECTION_TIMEOUT', 'KEEP_DELTAS',
|
||||
'CONNECTION_TIMEOUT', 'MAX_UPLOAD_RECORDS',
|
||||
'WEAVE_STATUS_OK', 'WEAVE_STATUS_FAILED',
|
||||
'WEAVE_STATUS_PARTIAL', 'SERVER_LOW_QUOTA',
|
||||
'SERVER_DOWNTIME', 'SERVER_UNREACHABLE',
|
||||
|
@ -79,7 +79,10 @@ const ONE_MEGABYTE = 1024 * ONE_KILOBYTE;
|
|||
|
||||
const CONNECTION_TIMEOUT = 30000;
|
||||
|
||||
const KEEP_DELTAS = 25;
|
||||
// How many records to upload in a single POST
|
||||
// If there are more, multiple POST calls will be made.
|
||||
// Record size limit is currently 10K, so 100 is a bit over 1MB
|
||||
const MAX_UPLOAD_RECORDS = 100;
|
||||
|
||||
// Top-level statuses:
|
||||
const WEAVE_STATUS_OK = "Sync succeeded.";
|
||||
|
|
|
@ -459,14 +459,28 @@ SyncEngine.prototype = {
|
|||
// don't cache the outgoing items, we won't need them later
|
||||
this._store.cache.enabled = false;
|
||||
|
||||
let count = 0;
|
||||
for (let id in this._tracker.changedIDs) {
|
||||
let out = this._createRecord(id);
|
||||
this._log.trace("Outgoing:\n" + out);
|
||||
|
||||
// skip getting siblings of already processed and deleted records
|
||||
if (!out.deleted && !(out.id in meta))
|
||||
this._store.createMetaRecords(out.id, meta);
|
||||
|
||||
out.encrypt(ID.get("WeaveCryptoID"));
|
||||
up.pushData(JSON.parse(out.serialize())); // FIXME: inefficient
|
||||
|
||||
if ((++count % MAX_UPLOAD_RECORDS) == 0) {
|
||||
// partial upload
|
||||
this._log.info("Uploading " + (count - MAX_UPLOAD_RECORDS) + " - " +
|
||||
count + " out of " + outnum + " records");
|
||||
up.post();
|
||||
if (up.data.modified > this.lastSync)
|
||||
this.lastSync = up.data.modified;
|
||||
up.clearRecords();
|
||||
}
|
||||
|
||||
Sync.sleep(0);
|
||||
}
|
||||
|
||||
|
@ -474,22 +488,21 @@ SyncEngine.prototype = {
|
|||
|
||||
// now add short depth-and-index-only records, except the ones we're
|
||||
// sending as full records
|
||||
let count = 0;
|
||||
let metaCount = 0;
|
||||
for each (let obj in meta) {
|
||||
if (!(obj.id in this._tracker.changedIDs)) {
|
||||
up.pushData(obj);
|
||||
count++;
|
||||
}
|
||||
if (!(obj.id in this._tracker.changedIDs)) {
|
||||
up.pushData(obj);
|
||||
metaCount++;
|
||||
}
|
||||
}
|
||||
|
||||
this._log.info("Uploading " + outnum + " records + " + count + " index/depth records)");
|
||||
// do the upload
|
||||
// final upload
|
||||
this._log.info("Uploading " +
|
||||
(count >= MAX_UPLOAD_RECORDS? "last batch of " : "") +
|
||||
count + " records, and " + metaCount + " index/depth records");
|
||||
up.post();
|
||||
|
||||
// save last modified date
|
||||
let mod = up.data.modified;
|
||||
if (mod > this.lastSync)
|
||||
this.lastSync = mod;
|
||||
if (up.data.modified > this.lastSync)
|
||||
this.lastSync = up.data.modified;
|
||||
}
|
||||
this._tracker.clearChangedIDs();
|
||||
},
|
||||
|
|
Загрузка…
Ссылка в новой задаче