Bug 580672 - Implement quota UI (Part 1) [r=mconnor]

Recognize quota warnings from server, implement API calls to retrieve quota information.

services/sync/modules/constants.js

@@ -117,6 +117,9 @@ SETUP_FAILED_NO_PASSPHRASE:            "error.sync.reason.setup_failed_no_passph
 CREDENTIALS_CHANGED:                   "error.sync.reason.credentials_changed",
 ABORT_SYNC_COMMAND:                    "aborting sync, process commands said so",
 NO_SYNC_NODE_FOUND:                    "error.sync.reason.no_node_found",
+OVER_QUOTA:                            "error.sync.reason.over_quota",
+
+RESPONSE_OVER_QUOTA:                   "14",
 
 // engine failure status codes
 ENGINE_UPLOAD_FAIL:                    "error.engine.reason.record_upload_fail",

services/sync/modules/resource.js

@@ -248,9 +248,15 @@ Resource.prototype = {
       if (this._log.level <= Log4Moz.Level.Trace)
         this._log.trace(action + " body: " + this._data);
 
-      // this is a server-side safety valve to allow slowing down clients without hurting performance
+      // This is a server-side safety valve to allow slowing down
+      // clients without hurting performance.
       if (headers["x-weave-backoff"])
-        Observers.notify("weave:service:backoff:interval", parseInt(headers["x-weave-backoff"], 10))
+        Observers.notify("weave:service:backoff:interval",
+                         parseInt(headers["x-weave-backoff"], 10));
+
+      if (success && headers["x-weave-quota-remaining"])
+        Observers.notify("weave:service:quota:remaining",
+                         parseInt(headers["x-weave-quota-remaining"], 10));
     }
     // Got a response but no header; must be cached (use default values)
     catch(ex) {

services/sync/modules/service.js

@@ -207,12 +207,12 @@ WeaveSvc.prototype = {
       return;
 
     let storageAPI = this.clusterURL + Svc.Prefs.get("storageAPI") + "/";
-    let userBase = storageAPI + this.username + "/";
-    this._log.debug("Caching URLs under storage user base: " + userBase);
+    this.userBaseURL = storageAPI + this.username + "/";
+    this._log.debug("Caching URLs under storage user base: " + this.userBaseURL);
 
     // Generate and cache various URLs under the storage API for this user
-    this.infoURL = userBase + "info/collections";
-    this.storageURL = userBase + "storage/";
+    this.infoURL = this.userBaseURL + "info/collections";
+    this.storageURL = this.userBaseURL + "storage/";
     this.metaURL = this.storageURL + "meta/global";
     PubKeys.defaultKeyUri = this.storageURL + "keys/pubkey";
     PrivKeys.defaultKeyUri = this.storageURL + "keys/privkey";
@@ -1379,7 +1379,7 @@ WeaveSvc.prototype = {
 
     // Make sure we have an up-to-date list of clients before sending commands
     this._log.trace("Refreshing client list");
-    Clients.sync();
+    this._syncEngine(Clients);
 
     // Wipe data in the desired direction if necessary
     switch (Svc.Prefs.get("firstSync")) {
@@ -1408,7 +1408,7 @@ WeaveSvc.prototype = {
       }
       finally {
         // Always immediately push back the local client (now without commands)
-        Clients.sync();
+        this._syncEngine(Clients);
       }
     }
 
@@ -1571,8 +1571,11 @@ WeaveSvc.prototype = {
     if (Utils.checkStatus(resp.status, null, [500, [502, 504]])) {
       Status.enforceBackoff = true;
       if (resp.status == 503 && resp.headers["retry-after"])
-        Svc.Obs.notify("weave:service:backoff:interval", parseInt(resp.headers["retry-after"], 10));
+        Svc.Obs.notify("weave:service:backoff:interval",
+                       parseInt(resp.headers["retry-after"], 10));
     }
+    if (resp.status == 400 && resp == RESPONSE_OVER_QUOTA)
+      Status.sync = OVER_QUOTA;
   },
   /**
    * Return a value for a backoff interval.  Maximum is eight hours, unless
@@ -1801,6 +1804,21 @@ WeaveSvc.prototype = {
     this._log.debug("Sending clients: " + [command, args, commandData.desc]);
     Clients.sendCommand(command, args);
   },
+
+  _getInfo: function _getInfo(what)
+    this._catch(this._notify(what, "", function() {
+      let url = this.userBaseURL + "info/" + what;
+      let response = new Resource(url).get();
+      if (response.status != 200)
+        return null;
+      return response.obj;
+    }))(),
+
+  getCollectionUsage: function getCollectionUsage()
+    this._getInfo("collection_usage"),
+
+  getQuota: function getQuota() this._getInfo("quota")
+
 };
 
 // Load Weave on the first time this file is loaded

services/sync/tests/unit/test_resource.js

@@ -99,6 +99,19 @@ function server_backoff(metadata, response) {
   response.bodyOutputStream.write(body, body.length);  
 }
 
+function server_quota_notice(request, response) {
+  let body = "You're approaching quota.";
+  response.setHeader("X-Weave-Quota-Remaining", '1048576', false);
+  response.setStatusLine(request.httpVersion, 200, "OK");
+  response.bodyOutputStream.write(body, body.length);  
+}
+
+function server_quota_error(request, response) {
+  let body = "14";
+  response.setHeader("X-Weave-Quota-Remaining", '-1024', false);
+  response.setStatusLine(request.httpVersion, 400, "OK");
+  response.bodyOutputStream.write(body, body.length);  
+}
 
 function server_headers(metadata, response) {
   let ignore_headers = ["host", "user-agent", "accept", "accept-language",
@@ -131,17 +144,19 @@ function run_test() {
   logger = Log4Moz.repository.getLogger('Test');
   Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
 
-  let server = new nsHttpServer();
-  server.registerPathHandler("/open", server_open);
-  server.registerPathHandler("/protected", server_protected);
-  server.registerPathHandler("/404", server_404);
-  server.registerPathHandler("/upload", server_upload);
-  server.registerPathHandler("/delete", server_delete);
-  server.registerPathHandler("/json", server_json);
-  server.registerPathHandler("/timestamp", server_timestamp);
-  server.registerPathHandler("/headers", server_headers);
-  server.registerPathHandler("/backoff", server_backoff);
-  server.start(8080);
+  let server = httpd_setup({
+    "/open": server_open,
+    "/protected": server_protected,
+    "/404": server_404,
+    "/upload": server_upload,
+    "/delete": server_delete,
+    "/json": server_json,
+    "/timestamp": server_timestamp,
+    "/headers": server_headers,
+    "/backoff": server_backoff,
+    "/quota-notice": server_quota_notice,
+    "/quota-error": server_quota_error
+  });
 
   Utils.prefs.setIntPref("network.numRetries", 1); // speed up test
 
@@ -181,7 +196,7 @@ function run_test() {
   let res2 = new Resource("http://localhost:8080/protected");
   content = res2.get();
   do_check_true(did401);
-  do_check_eq(content, "This path exists and is protected - failed")
+  do_check_eq(content, "This path exists and is protected - failed");
   do_check_eq(content.status, 401);
   do_check_false(content.success);
 
@@ -335,6 +350,25 @@ function run_test() {
   content = res10.get();
   do_check_eq(backoffInterval, 600);
 
+
+  _("X-Weave-Quota-Remaining header notifies observer on successful requests.");
+  let quotaValue;
+  function onQuota(subject, data) {
+    quotaValue = subject;
+  }
+  Observers.add("weave:service:quota:remaining", onQuota);
+
+  res10 = new Resource("http://localhost:8080/quota-error");
+  content = res10.get();
+  do_check_eq(content.status, 400);
+  do_check_eq(quotaValue, undefined); // HTTP 400, so no observer notification.
+
+  res10 = new Resource("http://localhost:8080/quota-notice");
+  content = res10.get();
+  do_check_eq(content.status, 200);
+  do_check_eq(quotaValue, 1048576);
+
+
   _("Error handling in _request() preserves exception information");
   let error;
   let res11 = new Resource("http://localhost:12345/does/not/exist");
@@ -363,7 +397,7 @@ function run_test() {
   do_check_true(content.success);
   do_check_eq(res.data, content);
   do_check_true(did401);
-  do_check_eq(redirRequest.response, "This path exists and is protected - failed")
+  do_check_eq(redirRequest.response, "This path exists and is protected - failed");
   do_check_eq(redirRequest.response.status, 401);
   do_check_false(redirRequest.response.success);
 
@@ -374,7 +408,7 @@ function run_test() {
   let res13 = new Resource("http://localhost:8080/protected");
   content = res13.get();
   do_check_true(did401);
-  do_check_eq(content, "This path exists and is protected - failed")
+  do_check_eq(content, "This path exists and is protected - failed");
   do_check_eq(content.status, 401);
   do_check_false(content.success);
 

services/sync/tests/unit/test_service_attributes.js

@@ -18,6 +18,7 @@ function test_urlsAndIdentities() {
 
     do_check_true(!!Service.serverURL); // actual value may change
     do_check_eq(Service.clusterURL, "");
+    do_check_eq(Service.userBaseURL, undefined);
     do_check_eq(Service.infoURL, undefined);
     do_check_eq(Service.storageURL, undefined);
     do_check_eq(Service.metaURL, undefined);
@@ -31,6 +32,7 @@ function test_urlsAndIdentities() {
 
     // Since we don't have a cluster URL yet, these will still not be defined.
     do_check_eq(Service.infoURL, undefined);
+    do_check_eq(Service.userBaseURL, undefined);
     do_check_eq(Service.storageURL, undefined);
     do_check_eq(Service.metaURL, undefined);
     do_check_eq(PubKeys.defaultKeyUri, undefined);
@@ -49,6 +51,7 @@ function test_urlsAndIdentities() {
     Service.clusterURL = "http://weave.cluster/";
     do_check_eq(Svc.Prefs.get("clusterURL"), "http://weave.cluster/");
 
+    do_check_eq(Service.userBaseURL, "http://weave.cluster/1.0/johndoe/");
     do_check_eq(Service.infoURL,
                 "http://weave.cluster/1.0/johndoe/info/collections");
     do_check_eq(Service.storageURL,

services/sync/tests/unit/test_service_quota.js

@@ -0,0 +1,50 @@
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+
+function send(body) {
+  return function(request, response) {
+    response.setStatusLine(request.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(body, body.length);
+  };
+}
+
+function run_test() {
+  let collection_usage = {steam:  65.11328,
+                          petrol: 82.488281,
+                          diesel: 2.25488281};
+  let quota = [2169.65136, 8192];
+
+  do_test_pending();
+  let server = httpd_setup({
+    "/1.0/johndoe/info/collection_usage": send(JSON.stringify(collection_usage)),
+    "/1.0/johndoe/info/quota":            send(JSON.stringify(quota)),
+    "/1.0/janedoe/info/collection_usage": send("gargabe"),
+    "/1.0/janedoe/info/quota":            send("more garbage")
+  });
+
+  try {
+    Weave.Service.clusterURL = "http://localhost:8080/";
+    Weave.Service.username = "johndoe";
+
+    _("Test getCollectionUsage().");
+    let res = Weave.Service.getCollectionUsage();
+    do_check_true(Utils.deepEquals(res, collection_usage));
+
+    _("Test getQuota().");
+    res = Weave.Service.getQuota();
+    do_check_true(Utils.deepEquals(res, quota));
+
+    _("Both return 'null' for non-200 responses.");
+    Weave.Service.username = "nonexistent";
+    do_check_eq(Weave.Service.getCollectionUsage(), null);
+    do_check_eq(Weave.Service.getQuota(), null);
+
+    _("Both return nothing (undefined) if the return value can't be parsed.");
+    Weave.Service.username = "janedoe";
+    do_check_eq(Weave.Service.getCollectionUsage(), undefined);
+    do_check_eq(Weave.Service.getQuota(), undefined);
+
+  } finally {
+    server.stop(do_test_finished);
+  }
+}

services/sync/tests/unit/test_service_sync_checkServerError.js

@@ -0,0 +1,136 @@
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/status.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/util.js");
+
+initTestLogging();
+
+function CatapultEngine() {
+  SyncEngine.call(this, "Catapult");
+}
+CatapultEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  exception: null, // tests fill this in
+  sync: function sync() {
+    throw this.exception;
+  }
+};
+
+function sync_httpd_setup() {
+  let handlers = {};
+  handlers["/1.0/johndoe/info/collections"]
+      = (new ServerWBO("collections", {})).handler(),
+  handlers["/1.0/johndoe/storage/keys/pubkey"]
+      = (new ServerWBO("pubkey")).handler();
+  handlers["/1.0/johndoe/storage/keys/privkey"]
+      = (new ServerWBO("privkey")).handler();
+  handlers["/1.0/johndoe/storage/clients"]
+      = (new ServerCollection()).handler();
+  handlers["/1.0/johndoe/storage/crypto"]
+      = (new ServerCollection()).handler();
+  handlers["/1.0/johndoe/storage/crypto/clients"]
+      = (new ServerWBO("clients", {})).handler();
+  handlers["/1.0/johndoe/storage/meta/global"]
+      = (new ServerWBO("global", {})).handler();
+  return httpd_setup(handlers);
+}
+
+function setUp() {
+  Service.username = "johndoe";
+  Service.password = "ilovejane";
+  Service.passphrase = "sekrit";
+  Service.clusterURL = "http://localhost:8080/";
+  new FakeCryptoService();
+}
+
+function test_backoff500() {
+  _("Test: HTTP 500 sets backoff status.");
+  let server = sync_httpd_setup();
+  do_test_pending();
+  setUp();
+
+  Engines.register(CatapultEngine);
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.exception = {status: 500};
+
+  try {
+    do_check_false(Status.enforceBackoff);
+    Service.login();
+    Service.sync();
+    do_check_true(Status.enforceBackoff);
+  } finally {
+    server.stop(do_test_finished);
+    Engines.unregister("catapult");
+    Status.resetBackoff();
+    Service.startOver();
+  }
+}
+
+function test_backoff503() {
+  _("Test: HTTP 503 with Retry-After header leads to backoff notification and sets backoff status.");
+  let server = sync_httpd_setup();
+  do_test_pending();
+  setUp();
+
+  const BACKOFF = 42;
+  Engines.register(CatapultEngine);
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.exception = {status: 503,
+                      headers: {"retry-after": BACKOFF}};
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function (subject) {
+    backoffInterval = subject;
+  });
+
+  try {
+    do_check_false(Status.enforceBackoff);
+
+    Service.login();
+    Service.sync();
+
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, BACKOFF);
+  } finally {
+    server.stop(do_test_finished);
+    Engines.unregister("catapult");
+    Status.resetBackoff();
+    Service.startOver();
+  }
+}
+
+function test_overQuota() {
+  _("Test: HTTP 400 with body error code 14 means over quota.");
+  let server = sync_httpd_setup();
+  do_test_pending();
+  setUp();
+
+  Engines.register(CatapultEngine);
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.exception = {status: 400,
+                      toString: function() "14"};
+
+  try {
+    do_check_eq(Status.sync, SYNC_SUCCEEDED);
+
+    Service.login();
+    Service.sync();
+
+    do_check_eq(Status.sync, OVER_QUOTA);
+  } finally {
+    server.stop(do_test_finished);
+    Engines.unregister("catapult");
+    Status.resetSync();
+    Service.startOver();
+  }
+}
+
+function run_test() {
+  test_backoff500();
+  test_backoff503();
+  test_overQuota();
+}