merge fx-sync to mozilla-central

services/sync/modules/engines.js

@@ -639,6 +639,7 @@ SyncEngine.prototype = {
 
   // Upload outgoing records
   _uploadOutgoing: function SyncEngine__uploadOutgoing() {
+    let failed = {};
     let outnum = [i for (i in this._tracker.changedIDs)].length;
     if (outnum) {
       this._log.trace("Preparing " + outnum + " outgoing records");
@@ -662,6 +663,18 @@ SyncEngine.prototype = {
         if (modified > this.lastSync)
           this.lastSync = modified;
 
+        // Remember changed IDs and timestamp of failed items so we
+        // can mark them changed again.
+        let failed_ids = [];
+        for (let id in resp.obj.failed) {
+          failed[id] = this._tracker.changedIDs[id];
+          failed_ids.push(id);
+        }
+        if (failed_ids.length)
+          this._log.debug("Records that will be uploaded again because "
+                          + "the server couldn't store them: "
+                          + failed_ids.join(", "));
+
         up.clearRecords();
       });
 
@@ -690,6 +703,11 @@ SyncEngine.prototype = {
         doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all");
     }
     this._tracker.clearChangedIDs();
+
+    // Mark failed WBOs as changed again so they are reuploaded next time.
+    for (let id in failed) {
+      this._tracker.addChangedID(id, failed[id]);
+    }
   },
 
   // Any cleanup necessary.

services/sync/tests/unit/head_http_server.js

@@ -153,7 +153,7 @@ ServerCollection.prototype = {
   post: function(input) {
     input = JSON.parse(input);
     let success = [];
-    let failed = [];
+    let failed = {};
 
     // This will count records where we have an existing ServerWBO
     // registered with us as successful and all other records as failed.
@@ -164,10 +164,11 @@ ServerCollection.prototype = {
         wbo.modified = Date.now() / 1000;
         success.push(record.id);
       } else {
-        failed.push(record.id);
+        failed[record.id] = "no wbo configured";
       }
     }
-    return {success: success,
+    return {modified: Date.now() / 1000,
+            success: success,
             failed: failed};
   },
 

services/sync/tests/unit/test_auth_manager.js

@@ -34,6 +34,7 @@ function server_handler(metadata, response) {
 }
 
 function run_test() {
+do_test_pending();
   logger = Log4Moz.repository.getLogger('Test');
   Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
@@ -57,6 +58,6 @@ function run_test() {
     do_check_eq(content, "This path exists and is protected by a UTF8 password");
     do_check_eq(content.status, 200);
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
   }
 }

services/sync/tests/unit/test_records_crypto.js

@@ -37,9 +37,10 @@ function crypto_meta_handler(metadata, response) {
 
 function run_test() {
   let server;
+  do_test_pending();
 
   try {
-    let log = Log4Moz.repository.getLogger();
+    let log = Log4Moz.repository.getLogger("Test");
     Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
     log.info("Setting up server and authenticator");
@@ -119,5 +120,7 @@ function run_test() {
 
     log.info("Done!");
   }
-  finally { server.stop(function() {}); }
+  finally {
+    server.stop(do_test_finished);
+  }
 }

services/sync/tests/unit/test_records_keys.js

@@ -1,12 +1,8 @@
-try {
-  Cu.import("resource://services-sync/base_records/keys.js");
-  Cu.import("resource://services-sync/auth.js");
-  Cu.import("resource://services-sync/log4moz.js");
-  Cu.import("resource://services-sync/identity.js");
-  Cu.import("resource://services-sync/util.js");
-} catch (e) {
-  do_throw(e);
-}
+Cu.import("resource://services-sync/base_records/keys.js");
+Cu.import("resource://services-sync/auth.js");
+Cu.import("resource://services-sync/log4moz.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/util.js");
 
 function pubkey_handler(metadata, response) {
   let obj = {id: "asdf-1234-asdf-1234",
@@ -27,36 +23,27 @@ function privkey_handler(metadata, response) {
 }
 
 function test_get() {
-  let server;
+  let log = Log4Moz.repository.getLogger("Test");
+  Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
-  try {
-    let log = Log4Moz.repository.getLogger();
-    Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
+  log.info("Setting up authenticator");
 
-    log.info("Setting up server and authenticator");
+  let auth = new BasicAuthenticator(new Identity("secret", "guest", "guest"));
+  Auth.defaultAuthenticator = auth;
 
-    server = httpd_setup({"/pubkey": pubkey_handler,
-                          "/privkey": privkey_handler});
+  log.info("Getting a public key");
 
-    let auth = new BasicAuthenticator(new Identity("secret", "guest", "guest"));
-    Auth.defaultAuthenticator = auth;
+  let pubkey = PubKeys.get("http://localhost:8080/pubkey");
+  do_check_eq(pubkey.data.payload.type, "pubkey");
+  do_check_eq(PubKeys.response.status, 200);
 
-    log.info("Getting a public key");
+  log.info("Getting matching private key");
 
-    let pubkey = PubKeys.get("http://localhost:8080/pubkey");
-    do_check_eq(pubkey.data.payload.type, "pubkey");
-    do_check_eq(PubKeys.response.status, 200);
+  let privkey = PrivKeys.get(pubkey.privateKeyUri);
+  do_check_eq(privkey.data.payload.type, "privkey");
+  do_check_eq(PrivKeys.response.status, 200);
 
-    log.info("Getting matching private key");
-
-    let privkey = PrivKeys.get(pubkey.privateKeyUri);
-    do_check_eq(privkey.data.payload.type, "privkey");
-    do_check_eq(PrivKeys.response.status, 200);
-
-    log.info("Done!");
-  }
-  catch (e) { do_throw(e); }
-  finally { server.stop(function() {}); }
+  log.info("Done!");
 }
 
 
@@ -87,6 +74,15 @@ function test_createKeypair() {
 }
 
 function run_test() {
-  test_get();
-  test_createKeypair();
+  do_test_pending();
+  let server;
+  try {
+    server = httpd_setup({"/pubkey": pubkey_handler,
+                          "/privkey": privkey_handler});
+
+    test_get();
+    test_createKeypair();
+  } finally {
+    server.stop(do_test_finished);
+  }
 }

services/sync/tests/unit/test_records_wbo.js

@@ -31,6 +31,7 @@ function coll_handler(metadata, response) {
 
 function run_test() {
   let server;
+  do_test_pending();
 
   try {
     let log = Log4Moz.repository.getLogger('Test');
@@ -74,5 +75,5 @@ function run_test() {
     log.info("Done!");
   }
   catch (e) { do_throw(e); }
-  finally { server.stop(function() {}); }
+  finally { server.stop(do_test_finished); }
 }

services/sync/tests/unit/test_resource.js

@@ -126,6 +126,8 @@ function server_headers(metadata, response) {
 
 
 function run_test() {
+  do_test_pending();
+
   logger = Log4Moz.repository.getLogger('Test');
   Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
@@ -376,5 +378,5 @@ function run_test() {
   do_check_eq(content.status, 401);
   do_check_false(content.success);
 
-  server.stop(function() {});
+  server.stop(do_test_finished);
 }

services/sync/tests/unit/test_service_changePassword.js

@@ -30,6 +30,7 @@ function run_test() {
       "/user/1.0/johndoe/password": send(200, "OK", ""),
       "/user/1.0/janedoe/password": send(401, "Unauthorized", "Forbidden!")
     });
+    do_test_pending();
 
     res = Weave.Service.changePassword("ILoveJane83");
     do_check_true(res);
@@ -59,7 +60,7 @@ function run_test() {
     Weave.Svc.Prefs.resetBranch("");
     Weave.Svc.Login.removeAllLogins();
     if (server) {
-      server.stop(function() {});
+      server.stop(do_test_finished);
     }
   }
 }

services/sync/tests/unit/test_service_checkUsername.js

@@ -8,6 +8,7 @@ function send(statusCode, status, body) {
 }
 
 function run_test() {
+  do_test_pending();
   let server = httpd_setup({
     "/user/1.0/johndoe": send(200, "OK", "1"),
     "/user/1.0/janedoe": send(200, "OK", "0")
@@ -26,6 +27,6 @@ function run_test() {
 
   } finally {
     Weave.Svc.Prefs.resetBranch("");
-    server.stop(function() {});
+    server.stop(do_test_finished);
   }
 }

services/sync/tests/unit/test_service_createAccount.js

@@ -16,6 +16,7 @@ function run_test() {
     };
   }
 
+  do_test_pending();
   let server = httpd_setup({
     "/user/1.0/johndoe": send(200, "OK", "0"),
     "/user/1.0/janedoe": send(400, "Bad Request", "2"),
@@ -60,6 +61,6 @@ function run_test() {
 
   } finally {
     Weave.Svc.Prefs.resetBranch("");
-    server.stop(function() {});
+    server.stop(do_test_finished);
   }
 }

services/sync/tests/unit/test_service_login.js

@@ -24,6 +24,7 @@ function run_test() {
   let logger = Log4Moz.repository.rootLogger;
   Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
+  do_test_pending();
   let server = httpd_setup({
     "/1.0/johndoe/info/collections": login_handler,
     "/1.0/janedoe/info/collections": login_handler
@@ -89,6 +90,6 @@ function run_test() {
 
   } finally {
     Svc.Prefs.resetBranch("");
-    server.stop(function() {});
+    server.stop(do_test_finished);
   }
 }

services/sync/tests/unit/test_service_passphraseUTF8.js

@@ -72,6 +72,7 @@ function run_test() {
     'obj': new ServerWBO('obj', {somedata: "that's going", toget: "wiped"})
   });
 
+  do_test_pending();
   let server = httpd_setup({
     // Need these to make Weave.Service.wipeRemote() etc. happy
     "/1.0/foo/storage/meta/global": new ServerWBO('global', {}).handler(),
@@ -142,7 +143,8 @@ function run_test() {
     do_check_false(Weave.Service._needUpdatedKeys);
 
   } finally {
-    server.stop(function() {});
+    if (server)
+      server.stop(do_test_finished);
     Weave.Svc.Prefs.resetBranch("");
   }
 }

services/sync/tests/unit/test_service_passwordUTF8.js

@@ -52,6 +52,7 @@ function change_password(request, response) {
 }
 
 function run_test() {
+  do_test_pending();
   let server = httpd_setup({
     "/1.0/johndoe/info/collections": info_collections,
     "/user/1.0/johndoe/password": change_password
@@ -79,7 +80,7 @@ function run_test() {
     do_check_eq(server_password, Utils.encodeUTF8(JAPANESE));
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Weave.Svc.Prefs.resetBranch("");
   }
 }

services/sync/tests/unit/test_service_verifyLogin.js

@@ -22,6 +22,7 @@ function run_test() {
   let logger = Log4Moz.repository.rootLogger;
   Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
+  do_test_pending();
   let server = httpd_setup({
     "/1.0/johndoe/info/collections": login_handler
   });
@@ -54,6 +55,6 @@ function run_test() {
 
   } finally {
     Svc.Prefs.resetBranch("");
-    server.stop(function() {});
+    server.stop(do_test_finished);
   }
 }

services/sync/tests/unit/test_syncengine_sync.js

@@ -187,6 +187,7 @@ function test_syncStartup_emptyOrOutdatedGlobalsResetsSync() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
 
   let engine = makeSteamEngine();
@@ -222,7 +223,7 @@ function test_syncStartup_emptyOrOutdatedGlobalsResetsSync() {
     do_check_eq(engine._tracker.changedIDs["rekolok"], 0);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -258,6 +259,7 @@ function test_syncStartup_metaGet404() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
 
   try {
@@ -281,7 +283,7 @@ function test_syncStartup_metaGet404() {
     do_check_eq(key.hmac, "fake-symmetric-key-0                                            ");
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -300,6 +302,7 @@ function test_syncStartup_failedMetaGet() {
       response.bodyOutputStream.write("Fail!", 5);
     }
   });
+  do_test_pending();
 
   let engine = makeSteamEngine();
   try {
@@ -314,7 +317,7 @@ function test_syncStartup_failedMetaGet() {
     do_check_eq(error.failureCode, ENGINE_METARECORD_DOWNLOAD_FAIL);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     syncTesting = new SyncTestingInfrastructure(makeSteamEngine);
@@ -330,6 +333,7 @@ function test_syncStartup_serverHasNewerVersion() {
   let server = httpd_setup({
       "/1.0/foo/storage/meta/global": global.handler()
   });
+  do_test_pending();
 
   let engine = makeSteamEngine();
   try {
@@ -345,7 +349,7 @@ function test_syncStartup_serverHasNewerVersion() {
     do_check_eq(error.failureCode, VERSION_OUT_OF_DATE);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     syncTesting = new SyncTestingInfrastructure(makeSteamEngine);
@@ -362,6 +366,7 @@ function test_syncStartup_syncIDMismatchResetsClient() {
   let server = sync_httpd_setup({
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler()
   });
+  do_test_pending();
 
   // global record with a different syncID than our engine has
   let engine = makeSteamEngine();
@@ -389,7 +394,7 @@ function test_syncStartup_syncIDMismatchResetsClient() {
     do_check_eq(engine.lastSync, 0);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -434,6 +439,7 @@ function test_syncStartup_badKeyWipesServerData() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
 
   try {
@@ -459,7 +465,7 @@ function test_syncStartup_badKeyWipesServerData() {
     do_check_eq(key.hmac, "fake-symmetric-key-1                                            ");
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -480,6 +486,7 @@ function test_processIncoming_emptyServer() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
 
   let engine = makeSteamEngine();
@@ -491,7 +498,7 @@ function test_processIncoming_emptyServer() {
     do_check_eq(engine.toFetch.length, 0);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -522,6 +529,7 @@ function test_processIncoming_createFromServer() {
       "/1.0/foo/storage/steam/flying": collection.wbos.flying.handler(),
       "/1.0/foo/storage/steam/scotsman": collection.wbos.scotsman.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
   createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
 
@@ -545,7 +553,7 @@ function test_processIncoming_createFromServer() {
     do_check_eq(engine._store.items.scotsman, "Flying Scotsman");
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -611,6 +619,7 @@ function test_processIncoming_reconcile() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
   createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
 
@@ -667,7 +676,7 @@ function test_processIncoming_reconcile() {
     do_check_eq(engine._store.items.nukeme, undefined);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -699,6 +708,7 @@ function test_processIncoming_fetchNum() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
   createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
 
@@ -773,7 +783,7 @@ function test_processIncoming_fetchNum() {
     do_check_true('record-no-233' in engine._store.items);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -798,6 +808,7 @@ function test_uploadOutgoing_toEmptyServer() {
       "/1.0/foo/storage/steam/flying": collection.wbos.flying.handler(),
       "/1.0/foo/storage/steam/scotsman": collection.wbos.scotsman.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
   createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
 
@@ -827,6 +838,65 @@ function test_uploadOutgoing_toEmptyServer() {
     // The 'flying' record wasn't marked so it wasn't uploaded
     do_check_eq(collection.wbos.flying.payload, undefined);
 
+  } finally {
+    server.stop(do_test_finished);
+    Svc.Prefs.resetBranch("");
+    Records.clearCache();
+    CryptoMetas.clearCache();
+    syncTesting = new SyncTestingInfrastructure(makeSteamEngine);
+  }
+}
+
+
+function test_uploadOutgoing_failed() {
+  _("SyncEngine._uploadOutgoing doesn't clear the tracker of objects that failed to upload.");
+
+  Svc.Prefs.set("clusterURL", "http://localhost:8080/");
+  Svc.Prefs.set("username", "foo");
+  let crypto_steam = new ServerWBO('steam');
+  let collection = new ServerCollection();
+  // We only define the "flying" WBO on the server, not the "scotsman"
+  // and "peppercorn" ones.
+  collection.wbos.flying = new ServerWBO('flying');
+
+  let server = sync_httpd_setup({
+      "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
+      "/1.0/foo/storage/steam": collection.handler()
+  });
+  createAndUploadKeypair();
+  createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
+
+  let engine = makeSteamEngine();
+  engine._store.items = {flying: "LNER Class A3 4472",
+                         scotsman: "Flying Scotsman",
+                         peppercorn: "Peppercorn Class"};
+  // Mark one of these records as changed 
+  const FLYING_CHANGED = 12345;
+  const SCOTSMAN_CHANGED = 23456;
+  const PEPPERCORN_CHANGED = 34567;
+  engine._tracker.addChangedID('flying', FLYING_CHANGED);
+  engine._tracker.addChangedID('scotsman', SCOTSMAN_CHANGED);
+  engine._tracker.addChangedID('peppercorn', PEPPERCORN_CHANGED);
+
+  try {
+
+    // Confirm initial environment
+    do_check_eq(collection.wbos.flying.payload, undefined);
+    do_check_eq(engine._tracker.changedIDs['flying'], FLYING_CHANGED);
+    do_check_eq(engine._tracker.changedIDs['scotsman'], SCOTSMAN_CHANGED);
+    do_check_eq(engine._tracker.changedIDs['peppercorn'], PEPPERCORN_CHANGED);
+
+    engine._uploadOutgoing();
+
+    // Ensure the 'flying' record has been uploaded and is no longer marked.
+    do_check_true(!!collection.wbos.flying.payload);
+    do_check_eq(engine._tracker.changedIDs['flying'], undefined);
+
+    // The 'scotsman' and 'peppercorn' records couldn't be uploaded so
+    // they weren't cleared from the tracker.
+    do_check_eq(engine._tracker.changedIDs['scotsman'], SCOTSMAN_CHANGED);
+    do_check_eq(engine._tracker.changedIDs['peppercorn'], PEPPERCORN_CHANGED);
+
   } finally {
     server.stop(function() {});
     Svc.Prefs.resetBranch("");
@@ -867,6 +937,7 @@ function test_uploadOutgoing_MAX_UPLOAD_RECORDS() {
       "/1.0/foo/storage/crypto/steam": crypto_steam.handler(),
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
   createAndUploadKeypair();
   createAndUploadSymKey("http://localhost:8080/1.0/foo/storage/crypto/steam");
 
@@ -886,7 +957,7 @@ function test_uploadOutgoing_MAX_UPLOAD_RECORDS() {
     do_check_eq(noOfUploads, Math.ceil(234/MAX_UPLOAD_RECORDS));
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     CryptoMetas.clearCache();
@@ -926,6 +997,7 @@ function test_syncFinish_deleteByIds() {
   let server = httpd_setup({
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
 
   let engine = makeSteamEngine();
   try {
@@ -942,7 +1014,7 @@ function test_syncFinish_deleteByIds() {
     do_check_eq(engine._delete.ids, undefined);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     syncTesting = new SyncTestingInfrastructure(makeSteamEngine);
@@ -979,6 +1051,7 @@ function test_syncFinish_deleteLotsInBatches() {
   let server = httpd_setup({
       "/1.0/foo/storage/steam": collection.handler()
   });
+  do_test_pending();
 
   let engine = makeSteamEngine();
   try {
@@ -1015,7 +1088,7 @@ function test_syncFinish_deleteLotsInBatches() {
     do_check_eq(engine._delete.ids, undefined);
 
   } finally {
-    server.stop(function() {});
+    server.stop(do_test_finished);
     Svc.Prefs.resetBranch("");
     Records.clearCache();
     syncTesting = new SyncTestingInfrastructure(makeSteamEngine);
@@ -1034,6 +1107,7 @@ function run_test() {
   test_processIncoming_reconcile();
   test_processIncoming_fetchNum();
   test_uploadOutgoing_toEmptyServer();
+  test_uploadOutgoing_failed();
   test_uploadOutgoing_MAX_UPLOAD_RECORDS();
   test_syncFinish_noDelete();
   test_syncFinish_deleteByIds();