Bug 684798 - Part 2: Sanitize wipeServer and catch all server maintenance errors. r=rnewman

services/sync/modules/service.js

@@ -1560,22 +1560,34 @@ WeaveSvc.prototype = {
     CollectionKeys.clear();
     this.upgradeSyncKey(this.syncID);
 
+    // Wipe the server.
+    let wipeTimestamp = this.wipeServer();
+
+    // Upload a new meta/global record.
     let meta = new WBORecord("meta", "global");
     meta.payload.syncID = this.syncID;
     meta.payload.storageVersion = STORAGE_VERSION;
     meta.isNew = true;
 
     this._log.debug("New metadata record: " + JSON.stringify(meta.payload));
-    let resp = new Resource(this.metaURL).put(meta);
+    let res = new Resource(this.metaURL);
-    if (!resp.success)
+    // It would be good to set the X-If-Unmodified-Since header to `timestamp`
+    // for this PUT to ensure at least some level of transactionality.
+    // Unfortunately, the servers don't support it after a wipe right now
+    // (bug 693893), so we're going to defer this until bug 692700.
+    let resp = res.put(meta);
+    if (!resp.success) {
+      // If we got into a race condition, we'll abort the sync this way, too.
+      // That's fine. We'll just wait till the next sync. The client that we're
+      // racing is probably busy uploading stuff right now anyway.
       throw resp;
+    }
     Records.set(this.metaURL, meta);
 
     // Wipe everything we know about except meta because we just uploaded it
     let collections = [Clients].concat(Engines.getAll()).map(function(engine) {
       return engine.name;
     });
-    this.wipeServer(collections);
 
     // Generate, upload, and download new keys. Do this last so we don't wipe
     // them...
@@ -1586,36 +1598,51 @@ WeaveSvc.prototype = {
    * Wipe user data from the server.
    *
    * @param collections [optional]
-   *        Array of collections to wipe. If not given, all collections are wiped.
+   *        Array of collections to wipe. If not given, all collections are
+   *        wiped by issuing a DELETE request for `storageURL`.
    *
-   * @param includeKeys [optional]
+   * @return the server's timestamp of the (last) DELETE.
-   *        If true, keys/pubkey and keys/privkey are deleted from the server.
-   *        This is false by default, which will cause the usual upgrade paths
-   *        to leave those keys on the server. This is to solve Bug 614737: old
-   *        clients check for keys *before* checking storage versions.
-   *
-   *        Note that this parameter only has an effect if `collections` is not
-   *        passed. If you explicitly pass a list of collections, they will be
-   *        processed regardless of the value of `includeKeys`.
    */
-  wipeServer: function wipeServer(collections, includeKeyPairs)
+  wipeServer: function wipeServer(collections)
     this._notify("wipe-server", "", function() {
+      let response;
       if (!collections) {
-        collections = [];
+        // Strip the trailing slash.
-        let info = new Resource(this.infoURL).get();
+        let res = new Resource(this.storageURL.slice(0, -1));
-        for (let name in info.obj) {
+        res.setHeader("X-Confirm-Delete", "1");
-          if (includeKeyPairs || (name != "keys"))
+        try {
-            collections.push(name);
+          response = res.delete();
+        } catch (ex) {
+          this._log.debug("Failed to wipe server: " + Utils.exceptionStr(ex));
+          throw ex;
         }
+        if (response.status != 200 && response.status != 404) {
+          this._log.debug("Aborting wipeServer. Server responded with " +
+                          response.status + " response for " + this.storageURL);
+          throw response;
+        }
+        return response.headers["x-weave-timestamp"];
       }
+      let timestamp;
       for each (let name in collections) {
         let url = this.storageURL + name;
-        let response = new Resource(url).delete();
+        try {
+          response = new Resource(url).delete();
+        } catch (ex) {
+          this._log.debug("Failed to wipe '" + name + "' collection: " +
+                          Utils.exceptionStr(ex));
+          throw ex;
+        }
         if (response.status != 200 && response.status != 404) {
-          throw "Aborting wipeServer. Server responded with "
+          this._log.debug("Aborting wipeServer. Server responded with " +
-                + response.status + " response for " + url;
+                          response.status + " response for " + url);
+          throw response;
+        }
+        if ("x-weave-timestamp" in response.headers) {
+          timestamp = response.headers["x-weave-timestamp"];
         }
       }
+      return timestamp;
     })(),
 
   /**

services/sync/tests/unit/head_http_server.js

@@ -10,6 +10,17 @@ function new_timestamp() {
   return Math.round(Date.now() / 10) / 100;
 }
 
+function return_timestamp(request, response, timestamp) {
+  if (!timestamp) {
+    timestamp = new_timestamp();
+  }
+  let body = "" + timestamp;
+  response.setHeader("X-Weave-Timestamp", body);
+  response.setStatusLine(request.httpVersion, 200, "OK");
+  response.bodyOutputStream.write(body, body.length);
+  return timestamp;
+}
+
 function httpd_setup (handlers) {
   let server = new nsHttpServer();
   let port   = 8080;
@@ -608,7 +619,7 @@ SyncServer.prototype = {
    * subject to change: see Bug 650435.
    */
   timestamp: function timestamp() {
-    return Math.round(Date.now() / 10) / 100;
+    return new_timestamp();
   },
 
   /**

services/sync/tests/unit/test_errorhandler.js

@@ -78,23 +78,35 @@ function sync_httpd_setup() {
 
   let handler_401 = httpd_handler(401, "Unauthorized");
   return httpd_setup({
+    // Normal server behaviour.
     "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
     "/1.1/johndoe/info/collections": collectionsHelper.handler,
     "/1.1/johndoe/storage/crypto/keys":
       upd("crypto", (new ServerWBO("keys")).handler()),
     "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
 
+    // Credentials are wrong or node reallocated.
     "/1.1/janedoe/storage/meta/global": handler_401,
     "/1.1/janedoe/info/collections": handler_401,
 
-    "/maintenance/1.1/johnsmith/info/collections": service_unavailable,
+    // Maintenance or overloaded (503 + Retry-After) at info/collections.
+    "/maintenance/1.1/broken.info/info/collections": service_unavailable,
 
-    "/maintenance/1.1/janesmith/storage/meta/global": service_unavailable,
+    // Maintenance or overloaded (503 + Retry-After) at meta/global.
-    "/maintenance/1.1/janesmith/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.meta/storage/meta/global": service_unavailable,
+    "/maintenance/1.1/broken.meta/info/collections": collectionsHelper.handler,
 
-    "/maintenance/1.1/foo/storage/meta/global": upd("meta", global.handler()),
+    // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
-    "/maintenance/1.1/foo/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
-    "/maintenance/1.1/foo/storage/crypto/keys": service_unavailable,
+    "/maintenance/1.1/broken.keys/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.keys/storage/crypto/keys": service_unavailable,
+
+    // Maintenance or overloaded (503 + Retry-After) at wiping collection.
+    "/maintenance/1.1/broken.wipe/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
+    "/maintenance/1.1/broken.wipe/storage/crypto/keys":
+      upd("crypto", (new ServerWBO("keys")).handler()),
+    "/maintenance/1.1/broken.wipe/storage": service_unavailable
   });
 }
 
@@ -732,9 +744,9 @@ add_test(function test_info_collections_login_server_maintenance_error() {
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -771,9 +783,9 @@ add_test(function test_meta_global_login_server_maintenance_error() {
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -810,8 +822,8 @@ add_test(function test_crypto_keys_login_server_maintenance_error() {
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
@@ -877,9 +889,9 @@ add_test(function test_info_collections_login_prolonged_server_maintenance_error
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -909,9 +921,9 @@ add_test(function test_meta_global_login_prolonged_server_maintenance_error(){
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -941,8 +953,8 @@ add_test(function test_download_crypto_keys_login_prolonged_server_maintenance_e
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
@@ -975,7 +987,41 @@ add_test(function test_upload_crypto_keys_login_prolonged_server_maintenance_err
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, SYNC_FAILED);
+    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(PROLONGED_ERROR_DURATION);
+  Service.sync();
+});
+
+add_test(function test_wipeServer_login_prolonged_server_maintenance_error(){
+  // Test crypto/keys prolonged server maintenance errors are reported.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";
@@ -1037,9 +1083,9 @@ add_test(function test_info_collections_login_syncAndReportErrors_server_mainten
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -1070,9 +1116,9 @@ add_test(function test_meta_global_login_syncAndReportErrors_server_maintenance_
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -1103,8 +1149,8 @@ add_test(function test_download_crypto_keys_login_syncAndReportErrors_server_mai
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
@@ -1138,7 +1184,42 @@ add_test(function test_upload_crypto_keys_login_syncAndReportErrors_server_maint
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, LOGIN_FAILED);
+    do_check_eq(Status.login, SERVER_MAINTENANCE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(NON_PROLONGED_ERROR_DURATION);
+  ErrorHandler.syncAndReportErrors();
+});
+
+add_test(function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
+  // Test crypto/keys server maintenance errors are reported
+  // when calling syncAndReportErrors.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";
@@ -1200,9 +1281,9 @@ add_test(function test_info_collections_login_syncAndReportErrors_prolonged_serv
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -1233,9 +1314,9 @@ add_test(function test_meta_global_login_syncAndReportErrors_prolonged_server_ma
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
@@ -1266,8 +1347,8 @@ add_test(function test_download_crypto_keys_login_syncAndReportErrors_prolonged_
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
@@ -1301,7 +1382,42 @@ add_test(function test_upload_crypto_keys_login_syncAndReportErrors_prolonged_se
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, LOGIN_FAILED);
+    do_check_eq(Status.login, SERVER_MAINTENANCE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(PROLONGED_ERROR_DURATION);
+  ErrorHandler.syncAndReportErrors();
+});
+
+add_test(function test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+  // Test crypto/keys server maintenance errors are reported
+  // when calling syncAndReportErrors.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";

services/sync/tests/unit/test_service_sync_remoteSetup.js

@@ -29,7 +29,28 @@ function run_test() {
   let cryptoColl = new ServerCollection({keys: keysWBO});
   let metaColl = new ServerCollection({global: meta_global});
   do_test_pending();
+
+  /**
+   * Handle the bulk DELETE request sent by wipeServer.
+   */
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+
+    _("Wiping out all collections.");
+    cryptoColl.delete({});
+    clients.delete({});
+    metaColl.delete({});
+
+    let ts = new_timestamp();
+    collectionsHelper.update_collection("crypto", ts);
+    collectionsHelper.update_collection("clients", ts);
+    collectionsHelper.update_collection("meta", ts);
+    return_timestamp(request, response, ts);
+  }
+
   let server = httpd_setup({
+    "/1.1/johndoe/storage": storageHandler,
     "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()),
     "/1.1/johndoe/storage/crypto": upd("crypto", cryptoColl.handler()),
     "/1.1/johndoe/storage/clients": upd("clients", clients.handler()),

services/sync/tests/unit/test_service_sync_updateEnabledEngines.js

@@ -66,13 +66,27 @@ function sync_httpd_setup(handlers) {
 function setUp() {
   Service.username = "johndoe";
   Service.password = "ilovejane";
-  Service.passphrase = "sekrit";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/";
+  // So that we can poke at meta/global.
   new FakeCryptoService();
+
+  // Ensure that the server has valid keys so that logging in will work and not
+  // result in a server wipe, rendering many of these tests useless.
+  generateNewKeys();
+  let serverKeys = CollectionKeys.asWBO("crypto", "keys");
+  serverKeys.encrypt(Service.syncKeyBundle);
+  return serverKeys.upload(Service.cryptoKeysURL).success;
 }
 
 const PAYLOAD = 42;
 
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
 add_test(function test_newAccount() {
   _("Test: New account does not disable locally enabled engines.");
   let engine = Engines.get("steam");
@@ -89,7 +103,6 @@ add_test(function test_newAccount() {
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine continues to be enabled.");
@@ -118,7 +131,6 @@ add_test(function test_enabledLocally() {
     engine.enabled = true;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record now contains the new engine.");
@@ -143,6 +155,7 @@ add_test(function test_disabledLocally() {
                       version: engine.version}}
   });
   let steamCollection = new ServerWBO("steam", PAYLOAD);
+
   let server = sync_httpd_setup({
     "/1.1/johndoe/storage/meta/global": metaWBO.handler(),
     "/1.1/johndoe/storage/steam": steamCollection.handler()
@@ -157,7 +170,6 @@ add_test(function test_disabledLocally() {
     engine.enabled = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record no longer contains engine.");
@@ -205,7 +217,6 @@ add_test(function test_enabledRemotely() {
     do_check_false(engine.enabled);
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine is enabled.");
@@ -242,7 +253,6 @@ add_test(function test_disabledRemotelyTwoClients() {
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Disable engine by deleting from meta/global.");
@@ -284,7 +294,6 @@ add_test(function test_disabledRemotely() {
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine is not disabled: only one client.");
@@ -316,7 +325,6 @@ add_test(function test_dependentEnginesEnabledLocally() {
     steamEngine.enabled = true;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record now contains the new engines.");
@@ -348,6 +356,7 @@ add_test(function test_dependentEnginesDisabledLocally() {
 
   let steamCollection = new ServerWBO("steam", PAYLOAD);
   let stirlingCollection = new ServerWBO("stirling", PAYLOAD);
+
   let server = sync_httpd_setup({
     "/1.1/johndoe/storage/meta/global":     metaWBO.handler(),
     "/1.1/johndoe/storage/steam":           steamCollection.handler(),
@@ -365,7 +374,6 @@ add_test(function test_dependentEnginesDisabledLocally() {
     do_check_false(stirlingEngine.enabled);
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record no longer contains engines.");
@@ -384,7 +392,3 @@ add_test(function test_dependentEnginesDisabledLocally() {
     server.stop(run_next_test);
   }
 });
-
-function run_test() {
-  run_next_test();
-}

services/sync/tests/unit/test_service_wipeServer.js

@@ -13,22 +13,19 @@ FakeCollection.prototype = {
     let self = this;
     return function(request, response) {
       let body = "";
+      self.timestamp = new_timestamp();
+      let timestamp = "" + self.timestamp;
       if (request.method == "DELETE") {
-          body = JSON.stringify(Date.now() / 1000);
+          body = timestamp;
           self.deleted = true;
       }
+      response.setHeader("X-Weave-Timestamp", timestamp);
       response.setStatusLine(request.httpVersion, 200, "OK");
       response.bodyOutputStream.write(body, body.length);
     };
   }
 };
 
-function serviceUnavailable(request, response) {
-  let body = "Service Unavailable";
-  response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
-  response.bodyOutputStream.write(body, body.length);
-}
-
 function setUpTestFixtures() {
   let cryptoService = new FakeCryptoService();
 
@@ -37,7 +34,13 @@ function setUpTestFixtures() {
   Service.passphrase = "aabcdeabcdeabcdeabcdeabcde";
 }
 
-function test_withCollectionList_fail() {
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
+add_test(function test_wipeServer_list_success() {
   _("Service.wipeServer() deletes collections given as argument.");
 
   let steam_coll = new FakeCollection();
@@ -45,10 +48,42 @@ function test_withCollectionList_fail() {
 
   let server = httpd_setup({
     "/1.1/johndoe/storage/steam": steam_coll.handler(),
-    "/1.1/johndoe/storage/petrol": serviceUnavailable,
+    "/1.1/johndoe/storage/diesel": diesel_coll.handler(),
+    "/1.1/johndoe/storage/petrol": httpd_handler(404, "Not Found")
+  });
+
+  try {
+    setUpTestFixtures();
+
+    _("Confirm initial environment.");
+    do_check_false(steam_coll.deleted);
+    do_check_false(diesel_coll.deleted);
+
+    _("wipeServer() will happily ignore the non-existent collection and use the timestamp of the last DELETE that was successful.");
+    let timestamp = Service.wipeServer(["steam", "diesel", "petrol"]);
+    do_check_eq(timestamp, diesel_coll.timestamp);
+
+    _("wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted.");
+    do_check_true(steam_coll.deleted);
+    do_check_true(diesel_coll.deleted);
+
+  } finally {
+    server.stop(run_next_test);
+    Svc.Prefs.resetBranch("");
+  }
+});
+
+add_test(function test_wipeServer_list_503() {
+  _("Service.wipeServer() deletes collections given as argument.");
+
+  let steam_coll = new FakeCollection();
+  let diesel_coll = new FakeCollection();
+
+  let server = httpd_setup({
+    "/1.1/johndoe/storage/steam": steam_coll.handler(),
+    "/1.1/johndoe/storage/petrol": httpd_handler(503, "Service Unavailable"),
     "/1.1/johndoe/storage/diesel": diesel_coll.handler()
   });
-  do_test_pending();
 
   try {
     setUpTestFixtures();
@@ -61,84 +96,126 @@ function test_withCollectionList_fail() {
     let error;
     try {
       Service.wipeServer(["non-existent", "steam", "petrol", "diesel"]);
+      do_throw("Should have thrown!");
     } catch(ex) {
       error = ex;
     }
     _("wipeServer() threw this exception: " + error);
-    do_check_true(error != undefined);
+    do_check_eq(error.status, 503);
 
     _("wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted.");
     do_check_true(steam_coll.deleted);
     do_check_false(diesel_coll.deleted);
 
   } finally {
-    server.stop(do_test_finished);
+    server.stop(run_next_test);
     Svc.Prefs.resetBranch("");
   }
-}
+});
 
-function test_wipeServer_leaves_collections() {
+add_test(function test_wipeServer_all_success() {
-  _("Service.wipeServer() deletes everything but keys.");
+  _("Service.wipeServer() deletes all the things.");
   
-  let steam_coll = new FakeCollection();
+  /**
-  let diesel_coll = new FakeCollection();
+   * Handle the bulk DELETE request sent by wipeServer.
-  let keys_coll = new FakeCollection();
+   */
-
+  let deleted = false;
-  function info_collections(request, response) {
+  let serverTimestamp;
-    let collections = {};
+  function storageHandler(request, response) {
-    let timestamp = Date.now() / 1000;
+    do_check_eq("DELETE", request.method);
-    if (!steam_coll.deleted)
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
-      collections.steam = timestamp
+    deleted = true;
-    if (!diesel_coll.deleted)
+    serverTimestamp = return_timestamp(request, response);
-      collections.diesel = timestamp;
-    if (!keys_coll.deleted)
-      collections.keys = timestamp;
-    let body = JSON.stringify(collections);
-    response.setStatusLine(request.httpVersion, 200, "OK");
-    response.bodyOutputStream.write(body, body.length);
   }
 
   let server = httpd_setup({
-    "/1.1/johndoe/storage/steam": steam_coll.handler(),
+    "/1.1/johndoe/storage": storageHandler
-    "/1.1/johndoe/storage/diesel": diesel_coll.handler(),
-    "/1.1/johndoe/storage/keys": keys_coll.handler(),
-    "/1.1/johndoe/info/collections": info_collections
   });
-  do_test_pending();
+  setUpTestFixtures();
 
-  try {
+  _("Try deletion.");
-    setUpTestFixtures();
+  let returnedTimestamp = Service.wipeServer();
-    _("Info URL: " + Service.infoURL);
+  do_check_true(deleted);
+  do_check_eq(returnedTimestamp, serverTimestamp);
 
-    _("Confirm initial environment.");
+  server.stop(run_next_test);
-    do_check_false(steam_coll.deleted);
+  Svc.Prefs.resetBranch("");
-    do_check_false(diesel_coll.deleted);
+});
-    do_check_false(keys_coll.deleted);
 
-    _("Collections: " + new Resource(Service.infoURL).get());
+add_test(function test_wipeServer_all_404() {
-    _("Try deletion.");
+  _("Service.wipeServer() accepts a 404.");
-    Service.wipeServer();
-    _("Collections: " + new Resource(Service.infoURL).get());
   
-    _("Make sure keys is still present.");
+  /**
-    do_check_true(steam_coll.deleted);
+   * Handle the bulk DELETE request sent by wipeServer. Returns a 404.
-    do_check_true(diesel_coll.deleted);
+   */
-    do_check_false(keys_coll.deleted);
+  let deleted = false;
-    
+  let serverTimestamp;
-    _("Delete everything.");
+  function storageHandler(request, response) {
-    Service.wipeServer(null, true);
+    do_check_eq("DELETE", request.method);
-    do_check_true(steam_coll.deleted);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
-    do_check_true(diesel_coll.deleted);
+    deleted = true;
-    do_check_true(keys_coll.deleted);
+    serverTimestamp = new_timestamp();
-    
+    response.setHeader("X-Weave-Timestamp", "" + serverTimestamp);
-  } finally {
+    response.setStatusLine(request.httpVersion, 404, "Not Found");
-    server.stop(do_test_finished);
-    Svc.Prefs.resetBranch("");
   }
-}
 
-function run_test() {
+  let server = httpd_setup({
-  initTestLogging("Trace");
+    "/1.1/johndoe/storage": storageHandler
-  test_withCollectionList_fail();
+  });
-  test_wipeServer_leaves_collections();
+  setUpTestFixtures();
-}
+
+  _("Try deletion.");
+  let returnedTimestamp = Service.wipeServer();
+  do_check_true(deleted);
+  do_check_eq(returnedTimestamp, serverTimestamp);
+
+  server.stop(run_next_test);
+  Svc.Prefs.resetBranch("");
+});
+
+add_test(function test_wipeServer_all_503() {
+  _("Service.wipeServer() throws if it encounters a non-200/404 response.");
+  
+  /**
+   * Handle the bulk DELETE request sent by wipeServer. Returns a 503.
+   */
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+    response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
+  }
+
+  let server = httpd_setup({
+    "/1.1/johndoe/storage": storageHandler
+  });
+  setUpTestFixtures();
+
+  _("Try deletion.");
+  let error;
+  try {
+    Service.wipeServer();
+    do_throw("Should have thrown!");
+  } catch (ex) {
+    error = ex;
+  }
+  do_check_eq(error.status, 503);
+
+  server.stop(run_next_test);
+  Svc.Prefs.resetBranch("");
+});
+
+add_test(function test_wipeServer_all_connectionRefused() {
+  _("Service.wipeServer() throws if it encounters a network problem.");
+  setUpTestFixtures();
+
+  _("Try deletion.");
+  try {
+    Service.wipeServer();
+    do_throw("Should have thrown!");
+  } catch (ex) {
+    do_check_eq(ex.result, Cr.NS_ERROR_CONNECTION_REFUSED);
+  }
+
+  run_next_test();
+  Svc.Prefs.resetBranch("");
+});