diff --git a/services/crypto/modules/WeaveCrypto.js b/services/crypto/modules/WeaveCrypto.js index 2252d9f39e4b..273ccd9d631d 100644 --- a/services/crypto/modules/WeaveCrypto.js +++ b/services/crypto/modules/WeaveCrypto.js @@ -165,9 +165,6 @@ WeaveCrypto.prototype = { // security/nss/lib/softoken/secmodt.h#201 // typedef PRUint32 PK11AttrFlags; this.nss_t.PK11AttrFlags = ctypes.unsigned_int; - // security/nss/lib/util/secoidt.h#454 - // typedef enum - this.nss_t.SECOidTag = ctypes.int; // security/nss/lib/util/seccomon.h#83 // typedef struct SECItemStr SECItem; --> SECItemStr defined right below it this.nss_t.SECItem = ctypes.StructType( diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js index 2440f5987e01..f7daf4be9d3a 100644 --- a/services/sync/modules/engines.js +++ b/services/sync/modules/engines.js @@ -717,7 +717,7 @@ SyncEngine.prototype = { try { item.decrypt(); } catch (ex if (Utils.isHMACMismatch(ex) && - this.handleHMACMismatch())) { + this.handleHMACMismatch(item))) { // Let's try handling it. // If the callback returns true, try decrypting again, because // we've got new keys. @@ -1083,7 +1083,7 @@ SyncEngine.prototype = { this._resetClient(); }, - handleHMACMismatch: function handleHMACMismatch() { + handleHMACMismatch: function handleHMACMismatch(item) { return Weave.Service.handleHMACEvent(); } }; diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js index 0b0f59338558..6d2099dcce46 100644 --- a/services/sync/modules/engines/bookmarks.js +++ b/services/sync/modules/engines/bookmarks.js @@ -992,11 +992,12 @@ BookmarksStore.prototype = { } return this.__childGUIDsStm = stmt; }, + _childGUIDsCols: ["item_id", "guid"], _getChildGUIDsForId: function _getChildGUIDsForId(itemid) { let stmt = this._childGUIDsStm; stmt.params.parent = itemid; - let rows = Utils.queryAsync(stmt, ["item_id", "guid"]); + let rows = Utils.queryAsync(stmt, this._childGUIDsCols); return rows.map(function (row) { if (row.guid) { return row.guid; @@ -1144,6 +1145,7 @@ BookmarksStore.prototype = { "WHERE url = :url " + "LIMIT 1"); }, + _frecencyCols: ["frecency"], get _addGUIDAnnotationNameStm() { let stmt = this._getStmt( @@ -1165,6 +1167,7 @@ BookmarksStore.prototype = { stmt.params.anno_name = GUID_ANNO; return stmt; }, + _checkGUIDItemAnnotationCols: ["item_id", "name_id", "anno_id", "anno_date"], get _addItemAnnotationStm() { return this._getStmt( @@ -1214,8 +1217,7 @@ BookmarksStore.prototype = { let stmt = this._checkGUIDItemAnnotationStm; stmt.params.item_id = id; - let result = Utils.queryAsync(stmt, ["item_id", "name_id", "anno_id", - "anno_date"])[0]; + let result = Utils.queryAsync(stmt, this._checkGUIDItemAnnotationCols)[0]; if (!result) { this._log.warn("Couldn't annotate bookmark id " + id); return guid; @@ -1268,6 +1270,7 @@ BookmarksStore.prototype = { return this.__guidForIdStm = stmt; }, + _guidForIdCols: ["guid"], GUIDForId: function GUIDForId(id) { let special = kSpecialIds.specialGUIDForId(id); @@ -1278,7 +1281,7 @@ BookmarksStore.prototype = { stmt.params.item_id = id; // Use the existing GUID if it exists - let result = Utils.queryAsync(stmt, ["guid"])[0]; + let result = Utils.queryAsync(stmt, this._guidForIdCols)[0]; if (result && result.guid) return result.guid; @@ -1318,6 +1321,7 @@ BookmarksStore.prototype = { return this.__idForGUIDStm = stmt; }, + _idForGUIDCols: ["item_id"], // noCreate is provided as an optional argument to prevent the creation of // non-existent special records, such as "mobile". @@ -1329,7 +1333,7 @@ BookmarksStore.prototype = { // guid might be a String object rather than a string. stmt.params.guid = guid.toString(); - let results = Utils.queryAsync(stmt, ["item_id"]); + let results = Utils.queryAsync(stmt, this._idForGUIDCols); this._log.trace("Rows matching GUID " + guid + ": " + results.map(function(x) x.item_id)); @@ -1372,7 +1376,7 @@ BookmarksStore.prototype = { // Add in the bookmark's frecency if we have something if (record.bmkUri != null) { this._frecencyStm.params.url = record.bmkUri; - let result = Utils.queryAsync(this._frecencyStm, ["frecency"]); + let result = Utils.queryAsync(this._frecencyStm, this._frecencyCols); if (result.length) index += result[0].frecency; } diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js index 24748338c41b..fb817292f057 100644 --- a/services/sync/modules/engines/clients.js +++ b/services/sync/modules/engines/clients.js @@ -192,6 +192,20 @@ ClientEngine.prototype = { _wipeClient: function _wipeClient() { SyncEngine.prototype._resetClient.call(this); this._store.wipe(); + }, + + // Override the default behavior to delete bad records from the server. + handleHMACMismatch: function handleHMACMismatch(item) { + this._log.debug("Handling HMAC mismatch for " + item.id); + if (SyncEngine.prototype.handleHMACMismatch.call(this, item)) + return true; + + // It's a bad client record. Save it to be deleted at the end of the sync. + this._log.debug("Bad client record detected. Scheduling for deletion."); + this._deleteId(item.id); + + // Don't try again. + return false; } }; diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js index 0a086e4b7f82..98ceef37e248 100644 --- a/services/sync/modules/engines/forms.js +++ b/services/sync/modules/engines/forms.js @@ -91,7 +91,7 @@ let FormWrapper = { getQuery.params.value = value; // Give the guid if we found one - let item = Utils.queryAsync(getQuery, "guid")[0]; + let item = Utils.queryAsync(getQuery, ["guid"])[0]; if (!item) { // Shouldn't happen, but Bug 597400... @@ -120,9 +120,9 @@ let FormWrapper = { hasGUID: function hasGUID(guid) { let query = this.createStatement( - "SELECT 1 FROM moz_formhistory WHERE guid = :guid"); + "SELECT guid FROM moz_formhistory WHERE guid = :guid LIMIT 1"); query.params.guid = guid; - return Utils.queryAsync(query).length == 1; + return Utils.queryAsync(query, ["guid"]).length == 1; }, replaceGUID: function replaceGUID(oldGUID, newGUID) { diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js index 9da1c39f6bf7..31b37fcccc3a 100644 --- a/services/sync/modules/engines/history.js +++ b/services/sync/modules/engines/history.js @@ -138,12 +138,14 @@ HistoryStore.prototype = { "SELECT name FROM sqlite_temp_master " + "WHERE name IN ('moz_places_temp', 'moz_historyvisits_temp')"); }, + _haveTempTablesCols: ["name"], __haveTempTables: null, get _haveTempTables() { - if (this.__haveTempTables === null) - this.__haveTempTables = !!Utils.queryAsync(this._haveTempTablesStm, - ["name"]).length; + if (this.__haveTempTables === null) { + this.__haveTempTables = !!Utils.queryAsync( + this._haveTempTablesStm, this._haveTempTablesCols).length; + } return this.__haveTempTables; }, @@ -184,6 +186,8 @@ HistoryStore.prototype = { stmt.params.anno_name = GUID_ANNO; return stmt; }, + _checkGUIDPageAnnotationCols: ["place_id", "name_id", "anno_id", + "anno_date"], get _addPageAnnotationStm() { // Gecko <2.0 only @@ -237,8 +241,7 @@ HistoryStore.prototype = { let stmt = this._checkGUIDPageAnnotationStm; stmt.params.page_url = uri; - let result = Utils.queryAsync(stmt, ["place_id", "name_id", "anno_id", - "anno_date"])[0]; + let result = Utils.queryAsync(stmt, this._checkGUIDPageAnnotationCols)[0]; if (!result) { let log = Log4Moz.repository.getLogger("Engine.History"); log.warn("Couldn't annotate URI " + uri); @@ -295,13 +298,14 @@ HistoryStore.prototype = { return this.__guidStmt = stmt; }, + _guidCols: ["guid"], GUIDForUri: function GUIDForUri(uri, create) { let stm = this._guidStm; stm.params.page_url = uri.spec ? uri.spec : uri; // Use the existing GUID if it exists - let result = Utils.queryAsync(stm, ["guid"])[0]; + let result = Utils.queryAsync(stm, this._guidCols)[0]; if (result && result.guid) return result.guid; @@ -332,6 +336,7 @@ HistoryStore.prototype = { "WHERE place_id = (SELECT id FROM moz_places WHERE url = :url) " + "ORDER BY date DESC LIMIT 10"); }, + _visitCols: ["date", "type"], __urlStmt: null, get _urlStm() { @@ -365,6 +370,7 @@ HistoryStore.prototype = { return this.__urlStmt = stmt; }, + _urlCols: ["url", "title", "frecency"], get _allUrlStm() { // Gecko <2.0 @@ -386,17 +392,18 @@ HistoryStore.prototype = { "ORDER BY frecency DESC " + "LIMIT :max_results"); }, + _allUrlCols: ["url"], // See bug 320831 for why we use SQL here _getVisits: function HistStore__getVisits(uri) { this._visitStm.params.url = uri; - return Utils.queryAsync(this._visitStm, ["date", "type"]); + return Utils.queryAsync(this._visitStm, this._visitCols); }, // See bug 468732 for why we use SQL here _findURLByGUID: function HistStore__findURLByGUID(guid) { this._urlStm.params.guid = guid; - return Utils.queryAsync(this._urlStm, ["url", "title", "frecency"])[0]; + return Utils.queryAsync(this._urlStm, this._urlCols)[0]; }, changeItemID: function HStore_changeItemID(oldID, newID) { @@ -409,7 +416,7 @@ HistoryStore.prototype = { this._allUrlStm.params.cutoff_date = (Date.now() - 2592000000) * 1000; this._allUrlStm.params.max_results = MAX_HISTORY_UPLOAD; - let urls = Utils.queryAsync(this._allUrlStm, "url"); + let urls = Utils.queryAsync(this._allUrlStm, this._allUrlCols); let self = this; return urls.reduce(function(ids, item) { ids[self.GUIDForUri(item.url, true)] = item.url; @@ -420,41 +427,42 @@ HistoryStore.prototype = { applyIncomingBatch: function applyIncomingBatch(records) { // Gecko <2.0 if (!this._asyncHistory) { - return Store.prototype.applyIncomingBatch.apply(this, arguments); + return Store.prototype.applyIncomingBatch.call(this, records); } // Gecko 2.0 let failed = []; - // Convert incoming records to mozIPlaceInfo objects. - let placeInfos = records.map(function (record) { + // Convert incoming records to mozIPlaceInfo objects. Some records can be + // ignored or handled directly, so we're rewriting the array in-place. + let i, k; + for (i = 0, k = 0; i < records.length; i++) { + let record = records[k] = records[i]; + let shouldApply; + // This is still synchronous I/O for now. - if (record.deleted) { - try { + try { + if (record.deleted) { // Consider using nsIBrowserHistory::removePages() here. this.remove(record); - } catch (ex) { - this._log.warn("Failed to delete record " + record.id); - failed.push(record.id); + // No further processing needed. Remove it from the list. + shouldApply = false; + } else { + shouldApply = this._recordToPlaceInfo(record); } - return null; - } - try { - return this._recordToPlaceInfo(record); } catch(ex) { failed.push(record.id); - return null; + shouldApply = false; } - }, this); - // Filter out the places that can't be added (they're null) - function identity(obj) { - return obj; + if (shouldApply) { + k += 1; + } } - placeInfos = placeInfos.filter(identity); + records.length = k; // truncate array // Nothing to do. - if (!placeInfos.length) { + if (!records.length) { return failed; } @@ -469,7 +477,7 @@ HistoryStore.prototype = { cb(); }; Svc.Obs.add(TOPIC_UPDATEPLACES_COMPLETE, onComplete); - this._asyncHistory.updatePlaces(placeInfos, onPlace); + this._asyncHistory.updatePlaces(records, onPlace); Utils.waitForSyncCallback(cb); return failed; }, @@ -477,35 +485,45 @@ HistoryStore.prototype = { /** * Converts a Sync history record to a mozIPlaceInfo. * - * Throws if an invalid record is encountered (invalid URI, etc.) - * and returns null if the record is to be ignored (no visits to add, etc.) + * Throws if an invalid record is encountered (invalid URI, etc.), + * returns true if the record is to be applied, false otherwise + * (no visits to add, etc.), */ _recordToPlaceInfo: function _recordToPlaceInfo(record) { // Sort out invalid URIs and ones Places just simply doesn't want. - let uri = Utils.makeURI(record.histUri); - if (!uri) { + record.uri = Utils.makeURI(record.histUri); + if (!record.uri) { this._log.warn("Attempted to process invalid URI, skipping."); throw "Invalid URI in record"; } if (!Utils.checkGUID(record.id)) { this._log.warn("Encountered record with invalid GUID: " + record.id); - return null; + return false; } + record.guid = record.id; - if (!this._hsvc.canAddURI(uri)) { - this._log.trace("Ignoring record " + record.id + - " with URI " + uri.spec + ": can't add this URI."); - return null; + if (!this._hsvc.canAddURI(record.uri)) { + this._log.trace("Ignoring record " + record.id + " with URI " + + record.uri.spec + ": can't add this URI."); + return false; } // We dupe visits by date and type. So an incoming visit that has // the same timestamp and type as a local one won't get applied. - let curVisitsByDate = {}; - for each (let {date, type} in this._getVisits(record.histUri)) { - curVisitsByDate[date] = type; + // To avoid creating new objects, we rewrite the query result so we + // can simply check for containment below. + let curVisits = this._getVisits(record.histUri); + for (let i = 0; i < curVisits.length; i++) { + curVisits[i] = curVisits[i].date + "," + curVisits[i].type; } - let visits = record.visits.filter(function (visit) { + + // Walk through the visits, make sure we have sound data, and eliminate + // dupes. The latter is done by rewriting the array in-place. + let k; + for (i = 0, k = 0; i < record.visits.length; i++) { + let visit = record.visits[k] = record.visits[i]; + if (!visit.date || typeof visit.date != "number") { this._log.warn("Encountered record with invalid visit date: " + visit.date); @@ -520,24 +538,29 @@ HistoryStore.prototype = { } // Dates need to be integers visit.date = Math.round(visit.date); - return curVisitsByDate[visit.date] != visit.type; - }); + + if (curVisits.indexOf(visit.date + "," + visit.type) != -1) { + // Visit is a dupe, don't increment 'k' so the element will be + // overwritten. + continue; + } + visit.visitDate = visit.date; + visit.transitionType = visit.type; + k += 1; + } + record.visits.length = k; // truncate array // No update if there aren't any visits to apply. // mozIAsyncHistory::updatePlaces() wants at least one visit. // In any case, the only thing we could change would be the title // and that shouldn't change without a visit. - if (!visits.length) { - this._log.trace("Ignoring record " + record.id + - " with URI " + uri.spec + ": no visits to add."); - return null; + if (!record.visits.length) { + this._log.trace("Ignoring record " + record.id + " with URI " + + record.uri.spec + ": no visits to add."); + return false; } - return {uri: uri, - guid: record.id, - title: record.title, - visits: [{visitDate: visit.date, transitionType: visit.type} - for each (visit in visits)]}; + return true; }, create: function HistStore_create(record) { @@ -561,19 +584,18 @@ HistoryStore.prototype = { update: function HistStore_update(record) { this._log.trace(" -> processing history entry: " + record.histUri); - let placeInfo = this._recordToPlaceInfo(record); - if (!placeInfo) { + if (!this._recordToPlaceInfo(record)) { return; } - for each (let {visitDate, transitionType} in placeInfo.visits) { - Svc.History.addVisit(placeInfo.uri, visitDate, null, transitionType, + for each (let {visitDate, transitionType} in record.visits) { + Svc.History.addVisit(record.uri, visitDate, null, transitionType, transitionType == 5 || transitionType == 6, 0); } if (record.title) { try { - this._hsvc.setPageTitle(placeInfo.uri, record.title); + this._hsvc.setPageTitle(record.uri, record.title); } catch (ex if ex.result == Cr.NS_ERROR_NOT_AVAILABLE) { // There's no entry for the given URI, either because it's a // URI that Places ignores (e.g. javascript:) or there were no diff --git a/services/sync/modules/jpakeclient.js b/services/sync/modules/jpakeclient.js index eee76b4db218..0576418b9fc3 100644 --- a/services/sync/modules/jpakeclient.js +++ b/services/sync/modules/jpakeclient.js @@ -475,7 +475,8 @@ JPAKEClient.prototype = { } this._crypto_key = aes256Key.value; - this._hmac_key = Utils.makeHMACKey(Utils.safeAtoB(hmac256Key.value)); + let hmac_key = Utils.makeHMACKey(Utils.safeAtoB(hmac256Key.value)); + this._hmac_hasher = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256, hmac_key); callback(); }, @@ -523,7 +524,7 @@ JPAKEClient.prototype = { try { iv = Svc.Crypto.generateRandomIV(); ciphertext = Svc.Crypto.encrypt(this._data, this._crypto_key, iv); - hmac = Utils.sha256HMAC(ciphertext, this._hmac_key); + hmac = Utils.bytesAsHex(Utils.digestUTF8(ciphertext, this._hmac_hasher)); } catch (ex) { this._log.error("Failed to encrypt data."); this.abort(JPAKE_ERROR_INTERNAL); @@ -545,7 +546,8 @@ JPAKEClient.prototype = { } let step3 = this._incoming.payload; try { - let hmac = Utils.sha256HMAC(step3.ciphertext, this._hmac_key); + let hmac = Utils.bytesAsHex( + Utils.digestUTF8(step3.ciphertext, this._hmac_hasher)); if (hmac != step3.hmac) throw "HMAC validation failed!"; } catch (ex) { diff --git a/services/sync/modules/record.js b/services/sync/modules/record.js index d543275a3f4e..4b6d6c16da67 100644 --- a/services/sync/modules/record.js +++ b/services/sync/modules/record.js @@ -190,11 +190,11 @@ CryptoWrapper.prototype = { _logName: "Record.CryptoWrapper", ciphertextHMAC: function ciphertextHMAC(keyBundle) { - let hmacKey = keyBundle.hmacKeyObject; - if (!hmacKey) - throw "Cannot compute HMAC with null key."; - - return Utils.sha256HMAC(this.ciphertext, hmacKey); + let hasher = keyBundle.sha256HMACHasher; + if (!hasher) + throw "Cannot compute HMAC without an HMAC key."; + + return Utils.bytesAsHex(Utils.digestUTF8(this.ciphertext, hasher)); }, /* @@ -207,7 +207,6 @@ CryptoWrapper.prototype = { * Optional key bundle overrides the collection key lookup. */ encrypt: function encrypt(keyBundle) { - keyBundle = keyBundle || CollectionKeys.keyForCollection(this.collection); if (!keyBundle) throw new Error("Key bundle is null for " + this.uri.spec); @@ -221,7 +220,6 @@ CryptoWrapper.prototype = { // Optional key bundle. decrypt: function decrypt(keyBundle) { - if (!this.ciphertext) { throw "No ciphertext: nothing to decrypt?"; } @@ -238,14 +236,14 @@ CryptoWrapper.prototype = { } // Handle invalid data here. Elsewhere we assume that cleartext is an object. - let json_result = JSON.parse(Svc.Crypto.decrypt(this.ciphertext, - keyBundle.encryptionKey, this.IV)); + let cleartext = Svc.Crypto.decrypt(this.ciphertext, + keyBundle.encryptionKey, this.IV); + let json_result = JSON.parse(cleartext); if (json_result && (json_result instanceof Object)) { this.cleartext = json_result; - this.ciphertext = null; - } - else { + this.ciphertext = null; + } else { throw "Decryption failed: result is <" + json_result + ">, not an object."; } @@ -536,15 +534,14 @@ function KeyBundle(realm, collectionName, keyStr) { throw "KeyBundle given non-string key."; Identity.call(this, realm, collectionName, keyStr); - this._hmac = null; - this._encrypt = null; - - // Cache the key object. - this._hmacObj = null; } - KeyBundle.prototype = { __proto__: Identity.prototype, + + _encrypt: null, + _hmac: null, + _hmacObj: null, + _sha256HMACHasher: null, equals: function equals(bundle) { return bundle && @@ -570,12 +567,18 @@ KeyBundle.prototype = { set hmacKey(value) { this._hmac = value; this._hmacObj = value ? Utils.makeHMACKey(value) : null; + this._sha256HMACHasher = value ? Utils.makeHMACHasher( + Ci.nsICryptoHMAC.SHA256, this._hmacObj) : null; }, get hmacKeyObject() { return this._hmacObj; }, -} + + get sha256HMACHasher() { + return this._sha256HMACHasher; + } +}; function BulkKeyBundle(realm, collectionName) { let log = Log4Moz.repository.getLogger("BulkKeyBundle"); @@ -612,8 +615,8 @@ BulkKeyBundle.prototype = { } else { throw "Invalid keypair"; + } } - }, }; function SyncKeyBundle(realm, collectionName, syncKey) { @@ -640,6 +643,7 @@ SyncKeyBundle.prototype = { this._hmac = null; this._hmacObj = null; this._encrypt = null; + this._sha256HMACHasher = null; }, /* @@ -666,7 +670,13 @@ SyncKeyBundle.prototype = { this.generateEntry(); return this._hmacObj; }, - + + get sha256HMACHasher() { + if (!this._sha256HMACHasher) + this.generateEntry(); + return this._sha256HMACHasher; + }, + /* * If we've got a string, hash it into keys and store them. */ @@ -687,6 +697,8 @@ SyncKeyBundle.prototype = { // Individual sets: cheaper than calling parent setter. this._hmac = hmac; this._hmacObj = Utils.makeHMACKey(hmac); + this._sha256HMACHasher = Utils.makeHMACHasher( + Ci.nsICryptoHMAC.SHA256, this._hmacObj); } }; diff --git a/services/sync/modules/service.js b/services/sync/modules/service.js index 9c762e3ba215..529fb5150b02 100644 --- a/services/sync/modules/service.js +++ b/services/sync/modules/service.js @@ -1861,6 +1861,10 @@ WeaveSvc.prototype = { let enabled = [eng.name for each (eng in Engines.getEnabled())]; for (let engineName in meta.payload.engines) { + if (engineName == "clients") { + // Clients is special. + continue; + } let index = enabled.indexOf(engineName); if (index != -1) { // The engine is enabled locally. Nothing to do. diff --git a/services/sync/modules/util.js b/services/sync/modules/util.js index 66bb62d7a126..114f6d0677ee 100644 --- a/services/sync/modules/util.js +++ b/services/sync/modules/util.js @@ -229,32 +229,51 @@ let Utils = { return db.createStatement(query); }, - queryAsync: function(query, names) { - // Allow array of names, single name, and no name - if (!Utils.isArray(names)) - names = names == null ? [] : [names]; + // Prototype for mozIStorageCallback, used in queryAsync below. + // This allows us to define the handle* functions just once rather + // than on every queryAsync invocation. + _storageCallbackPrototype: { + results: null, + // These are set by queryAsync + names: null, + syncCb: null, - // Synchronously asyncExecute fetching all results by name - let execCb = Utils.makeSyncCallback(); - query.executeAsync({ - items: [], - handleResult: function handleResult(results) { - let row; - while ((row = results.getNextRow()) != null) { - this.items.push(names.reduce(function(item, name) { - item[name] = row.getResultByName(name); - return item; - }, {})); - } - }, - handleError: function handleError(error) { - execCb.throw(error); - }, - handleCompletion: function handleCompletion(reason) { - execCb(this.items); + handleResult: function handleResult(results) { + if (!this.names) { + return; } - }); - return Utils.waitForSyncCallback(execCb); + if (!this.results) { + this.results = []; + } + let row; + while ((row = results.getNextRow()) != null) { + let item = {}; + for each (name in this.names) { + item[name] = row.getResultByName(name); + } + this.results.push(item); + } + }, + handleError: function handleError(error) { + this.syncCb.throw(error); + }, + handleCompletion: function handleCompletion(reason) { + // If we were called with column names but didn't find any results, + // the calling code probably still expects an array as a return value. + if (this.names && !this.results) { + this.results = []; + } + this.syncCb(this.results); + } + }, + + queryAsync: function(query, names) { + // Synchronously asyncExecute fetching all results by name + let storageCallback = {names: names, + syncCb: Utils.makeSyncCallback()}; + storageCallback.__proto__ = Utils._storageCallbackPrototype; + query.executeAsync(storageCallback); + return Utils.waitForSyncCallback(storageCallback.syncCb); }, byteArrayToString: function byteArrayToString(bytes) { @@ -588,27 +607,49 @@ let Utils = { throw 'checkStatus failed'; }, - digest: function digest(message, hasher) { - let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]. - createInstance(Ci.nsIScriptableUnicodeConverter); - converter.charset = "UTF-8"; - - let data = converter.convertToByteArray(message, {}); + /** + * UTF8-encode a message and hash it with the given hasher. Returns a + * string containing bytes. The hasher is reset if it's an HMAC hasher. + */ + digestUTF8: function digestUTF8(message, hasher) { + let data = this._utf8Converter.convertToByteArray(message, {}); hasher.update(data, data.length); - return hasher.finish(false); + let result = hasher.finish(false); + if (hasher instanceof Ci.nsICryptoHMAC) { + hasher.reset(); + } + return result; + }, + + /** + * Treat the given message as a bytes string and hash it with the given + * hasher. Returns a string containing bytes. The hasher is reset if it's + * an HMAC hasher. + */ + digestBytes: function digestBytes(message, hasher) { + // No UTF-8 encoding for you, sunshine. + let bytes = [b.charCodeAt() for each (b in message)]; + hasher.update(bytes, bytes.length); + let result = hasher.finish(false); + if (hasher instanceof Ci.nsICryptoHMAC) { + hasher.reset(); + } + return result; }, bytesAsHex: function bytesAsHex(bytes) { - // Convert each hashed byte into 2-hex strings then combine them - return [("0" + byte.charCodeAt().toString(16)).slice(-2) - for each (byte in bytes)].join(""); + let hex = ""; + for (let i = 0; i < bytes.length; i++) { + hex += ("0" + bytes[i].charCodeAt().toString(16)).slice(-2); + } + return hex; }, _sha256: function _sha256(message) { let hasher = Cc["@mozilla.org/security/hash;1"]. createInstance(Ci.nsICryptoHash); hasher.init(hasher.SHA256); - return Utils.digest(message, hasher); + return Utils.digestUTF8(message, hasher); }, sha256: function sha256(message) { @@ -623,7 +664,7 @@ let Utils = { let hasher = Cc["@mozilla.org/security/hash;1"]. createInstance(Ci.nsICryptoHash); hasher.init(hasher.SHA1); - return Utils.digest(message, hasher); + return Utils.digestUTF8(message, hasher); }, sha1: function sha1(message) { @@ -634,6 +675,10 @@ let Utils = { return Utils.encodeBase32(Utils._sha1(message)); }, + sha1Base64: function (message) { + return btoa(Utils._sha1(message)); + }, + /** * Produce an HMAC key object from a key string. */ @@ -642,61 +687,33 @@ let Utils = { }, /** - * Produce an HMAC hasher. + * Produce an HMAC hasher and initialize it with the given HMAC key. */ - makeHMACHasher: function makeHMACHasher() { - return Cc["@mozilla.org/security/hmac;1"] - .createInstance(Ci.nsICryptoHMAC); - }, - - sha1Base64: function (message) { - return btoa(Utils._sha1(message)); + makeHMACHasher: function makeHMACHasher(type, key) { + let hasher = Cc["@mozilla.org/security/hmac;1"] + .createInstance(Ci.nsICryptoHMAC); + hasher.init(type, key); + return hasher; }, /** - * Generate a sha1 HMAC for a message, not UTF-8 encoded, - * and a given nsIKeyObject. - * Optionally provide an existing hasher, which will be - * initialized and reused. + * Some HMAC convenience functions for tests and backwards compatibility: + * + * sha1HMACBytes: hashes byte string, returns bytes string + * sha256HMAC: hashes UTF-8 encoded string, returns hex string + * sha256HMACBytes: hashes byte string, returns bytes string */ - sha1HMACBytes: function sha1HMACBytes(message, key, hasher) { - let h = hasher || this.makeHMACHasher(); - h.init(h.SHA1, key); - - // No UTF-8 encoding for you, sunshine. - let bytes = [b.charCodeAt() for each (b in message)]; - h.update(bytes, bytes.length); - return h.finish(false); + sha1HMACBytes: function sha1HMACBytes(message, key) { + let h = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA1, key); + return Utils.digestBytes(message, h); }, - - /** - * Generate a sha256 HMAC for a string message and a given nsIKeyObject. - * Optionally provide an existing hasher, which will be - * initialized and reused. - * - * Returns hex output. - */ - sha256HMAC: function sha256HMAC(message, key, hasher) { - let h = hasher || this.makeHMACHasher(); - h.init(h.SHA256, key); - return Utils.bytesAsHex(Utils.digest(message, h)); + sha256HMAC: function sha256HMAC(message, key) { + let h = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256, key); + return Utils.bytesAsHex(Utils.digestUTF8(message, h)); }, - - - /** - * Generate a sha256 HMAC for a string message, not UTF-8 encoded, - * and a given nsIKeyObject. - * Optionally provide an existing hasher, which will be - * initialized and reused. - */ - sha256HMACBytes: function sha256HMACBytes(message, key, hasher) { - let h = hasher || this.makeHMACHasher(); - h.init(h.SHA256, key); - - // No UTF-8 encoding for you, sunshine. - let bytes = [b.charCodeAt() for each (b in message)]; - h.update(bytes, bytes.length); - return h.finish(false); + sha256HMACBytes: function sha256HMACBytes(message, key) { + let h = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256, key); + return Utils.digestBytes(message, h); }, /** @@ -704,13 +721,13 @@ let Utils = { */ hkdfExpand: function hkdfExpand(prk, info, len) { const BLOCKSIZE = 256 / 8; - let h = Utils.makeHMACHasher(); + let h = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256, + Utils.makeHMACKey(prk)); let T = ""; let Tn = ""; let iterations = Math.ceil(len/BLOCKSIZE); for (let i = 0; i < iterations; i++) { - Tn = Utils.sha256HMACBytes(Tn + info + String.fromCharCode(i + 1), - Utils.makeHMACKey(prk), h); + Tn = Utils.digestBytes(Tn + info + String.fromCharCode(i + 1), h); T += Tn; } return T.slice(0, len); @@ -736,7 +753,6 @@ let Utils = { * can encode as you wish. */ pbkdf2Generate : function pbkdf2Generate(P, S, c, dkLen) { - // We don't have a default in the algo itself, as NSS does. // Use the constant. if (!dkLen) @@ -745,7 +761,7 @@ let Utils = { /* For HMAC-SHA-1 */ const HLEN = 20; - function F(PK, S, c, i, h) { + function F(S, c, i, h) { function XOR(a, b, isA) { if (a.length != b.length) { @@ -774,9 +790,9 @@ let Utils = { I[2] = String.fromCharCode((i >> 8) & 0xff); I[3] = String.fromCharCode(i & 0xff); - U[0] = Utils.sha1HMACBytes(S + I.join(''), PK, h); + U[0] = Utils.digestBytes(S + I.join(''), h); for (let j = 1; j < c; j++) { - U[j] = Utils.sha1HMACBytes(U[j - 1], PK, h); + U[j] = Utils.digestBytes(U[j - 1], h); } ret = U[0]; @@ -791,12 +807,11 @@ let Utils = { let r = dkLen - ((l - 1) * HLEN); // Reuse the key and the hasher. Remaking them 4096 times is 'spensive. - let PK = Utils.makeHMACKey(P); - let h = Utils.makeHMACHasher(); + let h = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA1, Utils.makeHMACKey(P)); T = []; for (let i = 0; i < l;) { - T[i] = F(PK, S, c, ++i, h); + T[i] = F(S, c, ++i, h); } let ret = ''; @@ -1153,10 +1168,7 @@ let Utils = { let fos = Cc["@mozilla.org/network/safe-file-output-stream;1"] .createInstance(Ci.nsIFileOutputStream); fos.init(file, MODE_WRONLY | MODE_CREATE | MODE_TRUNCATE, PERMS_FILE, 0); - let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] - .createInstance(Ci.nsIScriptableUnicodeConverter); - converter.charset = "UTF-8"; - let is = converter.convertToInputStream(out); + let is = this._utf8Converter.convertToInputStream(out); NetUtil.asyncCopy(is, fos, function (result) { if (typeof callback == "function") { callback.call(that); @@ -1289,11 +1301,8 @@ let Utils = { encodeUTF8: function(str) { try { - var unicodeConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] - .createInstance(Ci.nsIScriptableUnicodeConverter); - unicodeConverter.charset = "UTF-8"; - str = unicodeConverter.ConvertFromUnicode(str); - return str + unicodeConverter.Finish(); + str = this._utf8Converter.ConvertFromUnicode(str); + return str + this._utf8Converter.Finish(); } catch(ex) { return null; } @@ -1301,11 +1310,8 @@ let Utils = { decodeUTF8: function(str) { try { - var unicodeConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] - .createInstance(Ci.nsIScriptableUnicodeConverter); - unicodeConverter.charset = "UTF-8"; - str = unicodeConverter.ConvertToUnicode(str); - return str + unicodeConverter.Finish(); + str = this._utf8Converter.ConvertToUnicode(str); + return str + this._utf8Converter.Finish(); } catch(ex) { return null; } @@ -1645,6 +1651,12 @@ let FakeSvc = { isFake: true } }; +Utils.lazy2(Utils, "_utf8Converter", function() { + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + return converter; +}); /* * Commonly-used services diff --git a/services/sync/tests/unit/head_helpers.js b/services/sync/tests/unit/head_helpers.js index dbbc101b87d4..f3e0d16da2e7 100644 --- a/services/sync/tests/unit/head_helpers.js +++ b/services/sync/tests/unit/head_helpers.js @@ -187,10 +187,13 @@ function FakeCryptoService() { delete Svc.Crypto; // get rid of the getter first Svc.Crypto = this; Utils.sha256HMAC = this.sha256HMAC; + + Cu.import("resource://services-sync/record.js"); + CryptoWrapper.prototype.ciphertextHMAC = this.ciphertextHMAC; } FakeCryptoService.prototype = { - sha256HMAC: function(message, key) { + sha256HMAC: function Utils_sha256HMAC(message, hasher) { message = message.substr(0, 64); while (message.length < 64) { message += " "; @@ -198,6 +201,10 @@ FakeCryptoService.prototype = { return message; }, + ciphertextHMAC: function CryptoWrapper_ciphertextHMAC(keyBundle) { + return Utils.sha256HMAC(this.ciphertext); + }, + encrypt: function(aClearText, aSymmetricKey, aIV) { return aClearText; }, diff --git a/services/sync/tests/unit/head_http_server.js b/services/sync/tests/unit/head_http_server.js index 9dd6419afc64..e6844465750c 100644 --- a/services/sync/tests/unit/head_http_server.js +++ b/services/sync/tests/unit/head_http_server.js @@ -139,6 +139,17 @@ ServerCollection.prototype = { && (!options.newer || (wbo.modified > options.newer)); }, + count: function(options) { + options = options || {}; + let c = 0; + for (let [id, wbo] in Iterator(this.wbos)) { + if (wbo.modified && this._inResultSet(wbo, options)) { + c++; + } + } + return c; + }, + get: function(options) { let result; if (options.full) { diff --git a/services/sync/tests/unit/test_clients_engine.js b/services/sync/tests/unit/test_clients_engine.js index ce1a283f63d4..4f2f237d37f5 100644 --- a/services/sync/tests/unit/test_clients_engine.js +++ b/services/sync/tests/unit/test_clients_engine.js @@ -3,10 +3,86 @@ Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/identity.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-sync/engines/clients.js"); +Cu.import("resource://services-sync/service.js"); const MORE_THAN_CLIENTS_TTL_REFRESH = 691200; // 8 days const LESS_THAN_CLIENTS_TTL_REFRESH = 86400; // 1 day +function test_bad_hmac() { + _("Ensure that Clients engine deletes corrupt records."); + let global = new ServerWBO('global', + {engines: {clients: {version: Clients.version, + syncID: Clients.syncID}}}); + let clientsColl = new ServerCollection({}, true); + let keysWBO = new ServerWBO("keys"); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let collections = collectionsHelper.collections; + + // Watch for deletions in the given collection. + let deleted = false; + function trackDeletedHandler(coll, handler) { + let u = upd(coll, handler); + return function(request, response) { + if (request.method == "DELETE") + deleted = true; + + return u(request, response); + }; + } + + let handlers = { + "/1.0/foo/info/collections": collectionsHelper.handler, + "/1.0/foo/storage/meta/global": upd("meta", global.handler()), + "/1.0/foo/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.0/foo/storage/clients": trackDeletedHandler("crypto", clientsColl.handler()) + }; + + let server = httpd_setup(handlers); + do_test_pending(); + + try { + let passphrase = "abcdeabcdeabcdeabcdeabcdea"; + Service.serverURL = "http://localhost:8080/"; + Service.clusterURL = "http://localhost:8080/"; + Service.login("foo", "ilovejane", passphrase); + + CollectionKeys.generateNewKeys(); + + _("First sync, client record is uploaded"); + do_check_eq(0, clientsColl.count()); + do_check_eq(Clients.lastRecordUpload, 0); + Clients.sync(); + do_check_eq(1, clientsColl.count()); + do_check_true(Clients.lastRecordUpload > 0); + deleted = false; // Initial setup can wipe the server, so clean up. + + _("Records now: " + clientsColl.get({})); + _("Change our keys and our client ID, reupload keys."); + Clients.localID = Utils.makeGUID(); + Clients.resetClient(); + CollectionKeys.generateNewKeys(); + let serverKeys = CollectionKeys.asWBO("crypto", "keys"); + serverKeys.encrypt(Weave.Service.syncKeyBundle); + do_check_true(serverKeys.upload(Weave.Service.cryptoKeysURL).success); + + _("Sync."); + do_check_true(!deleted); + Clients.sync(); + + _("Old record was deleted, new one uploaded."); + do_check_true(deleted); + do_check_eq(1, clientsColl.count()); + _("Records now: " + clientsColl.get({})); + + } finally { + server.stop(do_test_finished); + Svc.Prefs.resetBranch(""); + Records.clearCache(); + } +} + function test_properties() { try { _("Test lastRecordUpload property"); @@ -74,6 +150,9 @@ function test_sync() { function run_test() { + initTestLogging("Trace"); + Log4Moz.repository.getLogger("Engine.Clients").level = Log4Moz.Level.Trace; + test_bad_hmac(); // Needs to run first: doesn't use fake service! test_properties(); test_sync(); } diff --git a/services/sync/tests/unit/test_utils_queryAsync.js b/services/sync/tests/unit/test_utils_queryAsync.js index 45bf7f316a7e..e171bbaa85a2 100644 --- a/services/sync/tests/unit/test_utils_queryAsync.js +++ b/services/sync/tests/unit/test_utils_queryAsync.js @@ -12,18 +12,18 @@ function run_test() { _("Empty out the formhistory table"); let r0 = Utils.queryAsync(c("DELETE FROM moz_formhistory")); - do_check_eq(r0.length, 0); + do_check_eq(r0, null); _("Make sure there's nothing there"); let r1 = Utils.queryAsync(c("SELECT 1 FROM moz_formhistory")); - do_check_eq(r1.length, 0); + do_check_eq(r1, null); _("Insert a row"); let r2 = Utils.queryAsync(c("INSERT INTO moz_formhistory (fieldname, value) VALUES ('foo', 'bar')")); - do_check_eq(r2.length, 0); + do_check_eq(r2, null); _("Request a known value for the one row"); - let r3 = Utils.queryAsync(c("SELECT 42 num FROM moz_formhistory"), "num"); + let r3 = Utils.queryAsync(c("SELECT 42 num FROM moz_formhistory"), ["num"]); do_check_eq(r3.length, 1); do_check_eq(r3[0].num, 42); @@ -41,7 +41,7 @@ function run_test() { _("Add multiple entries (sqlite doesn't support multiple VALUES)"); let r6 = Utils.queryAsync(c("INSERT INTO moz_formhistory (fieldname, value) SELECT 'foo', 'baz' UNION SELECT 'more', 'values'")); - do_check_eq(r6.length, 0); + do_check_eq(r6, null); _("Get multiple rows"); let r7 = Utils.queryAsync(c("SELECT fieldname, value FROM moz_formhistory WHERE fieldname = 'foo'"), ["fieldname", "value"]); @@ -51,7 +51,7 @@ function run_test() { _("Make sure updates work"); let r8 = Utils.queryAsync(c("UPDATE moz_formhistory SET value = 'updated' WHERE fieldname = 'more'")); - do_check_eq(r8.length, 0); + do_check_eq(r8, null); _("Get the updated"); let r9 = Utils.queryAsync(c("SELECT value, fieldname FROM moz_formhistory WHERE fieldname = 'more'"), ["fieldname", "value"]); @@ -60,7 +60,7 @@ function run_test() { do_check_eq(r9[0].value, "updated"); _("Grabbing fewer fields than queried is fine"); - let r10 = Utils.queryAsync(c("SELECT value, fieldname FROM moz_formhistory"), "fieldname"); + let r10 = Utils.queryAsync(c("SELECT value, fieldname FROM moz_formhistory"), ["fieldname"]); do_check_eq(r10.length, 3); _("Generate an execution error"); diff --git a/services/sync/tests/unit/test_utils_sha256HMAC.js b/services/sync/tests/unit/test_utils_sha256HMAC.js index c24623d6faf7..cc001283fc4d 100644 --- a/services/sync/tests/unit/test_utils_sha256HMAC.js +++ b/services/sync/tests/unit/test_utils_sha256HMAC.js @@ -2,8 +2,8 @@ _("Make sure sha256 hmac works with various messages and keys"); Cu.import("resource://services-sync/util.js"); function run_test() { - let key1 = Svc.KeyFactory.keyFromString(Ci.nsIKeyObject.HMAC, "key1"); - let key2 = Svc.KeyFactory.keyFromString(Ci.nsIKeyObject.HMAC, "key2"); + let key1 = Utils.makeHMACKey("key1"); + let key2 = Utils.makeHMACKey("key2"); let mes1 = "message 1"; let mes2 = "message 2";