diff --git a/services/sync/modules/base_records/collection.js b/services/sync/modules/base_records/collection.js index ff672eafdd41..57d3a286794f 100644 --- a/services/sync/modules/base_records/collection.js +++ b/services/sync/modules/base_records/collection.js @@ -127,63 +127,15 @@ Collection.prototype = { // Save this because onProgress is called with this as the ChannelListener let coll = this; + // Switch to newline separated records for incremental parsing + coll.setHeader("Accept", "application/newlines"); + this._onProgress = function() { - // Save some work by quitting early when there's no records - if (this._data == "[]") - return; - - do { - // Strip off the the starting "[" or separating "," or trailing "]" if - // it wasn't stripped off from a previous progress update - let start = this._data[0]; - if (start == "[" || start == "," || start == "]") - this._data = this._data.slice(1); - - // Track various states of # open braces and ignore for strings - let json = ""; - let braces = 1; - let ignore = false; - let escaped = false; - let length = this._data.length; - - // Skip the first character, the "{", and try to find a json record - for (let i = 1; i < length; i++) { - let char = this._data[i]; - - // Opening a string makes us ignore all characters except close " - if (char == '"') { - if (!ignore) - ignore = true; - // It's a real close " if it's not escaped - else if (!escaped) - ignore = false; - } - - // Track if an end quote might be escaped when processing strings - if (ignore) { - escaped = char == "\\" ? !escaped : false; - - // Don't bother checking other characters when ignoring - continue; - } - - // Increase the brace count on open { - if (char == "{") - braces++; - // Decrement brace count on close } - else if (char == "}" && --braces == 0) { - // Split the json record from the rest of the data - json = this._data.slice(0, i + 1); - this._data = this._data.slice(i + 1); - - // Stop processing for now that we found one record - break; - } - } - - // No valid record json found? - if (json.length == 0) - break; + let newline; + while ((newline = this._data.indexOf("\n")) > 0) { + // Split the json record from the rest of the data + let json = this._data.slice(0, newline); + this._data = this._data.slice(newline + 1); // Deserialize a record from json and give it to the callback let record = new coll._recordObj(); @@ -191,9 +143,7 @@ Collection.prototype = { record.baseURI = coll.uri; record.id = record.data.id; onRecord(record); - - // Keep processing the data until we can't find a json record - } while (true); + } // Aggressively clean up the objects we created above so that the next set // of records have enough memory to decrypt, reconcile, apply, etc. diff --git a/services/sync/tests/unit/test_collection_inc_get.js b/services/sync/tests/unit/test_collection_inc_get.js index 6815c27f8460..0c89f1a6c8be 100644 --- a/services/sync/tests/unit/test_collection_inc_get.js +++ b/services/sync/tests/unit/test_collection_inc_get.js @@ -9,7 +9,7 @@ function run_test() { _("Parse empty string payload as deleted"); called = false; - stream._data = '[{"payload":""}]'; + stream._data = '{"payload":""}\n'; coll.recordHandler = function(rec) { called = true; _("Got record:", JSON.stringify(rec)); @@ -23,7 +23,7 @@ function run_test() { _("Parse record with payload"); called = false; - stream._data = '[{"payload":"{\\"value\\":123}"}]'; + stream._data = '{"payload":"{\\"value\\":123}"}\n'; coll.recordHandler = function(rec) { called = true; _("Got record:", JSON.stringify(rec)); @@ -39,7 +39,7 @@ function run_test() { called = false; recCount = 0; sum = 0; - stream._data = '[{"payload":"{\\"value\\":100}"},{"payload":"{\\"value\\":10}"},{"payload":"{\\"value\\":1}"}]'; + stream._data = '{"payload":"{\\"value\\":100}"}\n{"payload":"{\\"value\\":10}"}\n{"payload":"{\\"value\\":1}"}\n'; coll.recordHandler = function(rec) { called = true; _("Got record:", JSON.stringify(rec)); @@ -76,7 +76,7 @@ function run_test() { called = false; recCount = 0; sum = 0; - stream._data = '[{"payl'; + stream._data = '{"payl'; coll.recordHandler = function(rec) { called = true; do_throw("shouldn't have gotten a record.."); @@ -92,7 +92,7 @@ function run_test() { _("adding more data enough for one record.."); called = false; - stream._data += 'oad":"{\\"value\\":100}"},'; + stream._data += 'oad":"{\\"value\\":100}"}\n'; coll.recordHandler = function(rec) { called = true; _("Got record:", JSON.stringify(rec)); @@ -126,7 +126,7 @@ function run_test() { _("add data for two records.."); called = false; - stream._data += '},{"payload":"{\\"value\\":1}"}'; + stream._data += '}\n{"payload":"{\\"value\\":1}"}\n'; coll.recordHandler = function(rec) { called = true; _("Got record:", JSON.stringify(rec)); @@ -155,9 +155,9 @@ function run_test() { do_check_true(called); _(); - _("add ending array bracket"); + _("add no extra data"); called = false; - stream._data += ']'; + stream._data += ''; coll.recordHandler = function(rec) { called = true; do_throw("shouldn't have gotten a record.."); @@ -166,7 +166,7 @@ function run_test() { _("should still have 3 records with sum 111"); do_check_eq(recCount, 3); do_check_eq(sum, 111); - _("should have consumed the last array bracket"); + _("should have consumed nothing but still have nothing"); do_check_eq(stream._data, ""); do_check_false(called); _("\n");