зеркало из https://github.com/mozilla/gecko-dev.git
bug 511549 - make detailedStatus much smarter about errors and backoff, r=edilee
--HG-- extra : rebase_source : 42e3eb9c89d2567d18c02e7583202e284dd69b0e
This commit is contained in:
Mike Connor
Родитель
2aea4ab80a
Коммит
cc109400c6
|
|
@ -20,6 +20,6 @@ error.login.description = Weave encountered an error while signing you in: %1$S.
|
|||
error.logout.title = Error While Signing Out
|
||||
error.logout.description = Weave encountered an error while signing you out. It's probably ok, and you don't have to do anything about it.
|
||||
error.sync.title = Error While Syncing
|
||||
error.sync.description = Weave encountered an error while syncing. You might want to try syncing manually to make sure you are up-to-date.
|
||||
error.sync.description = Weave encountered an error while syncing: %1$S. Weave will automatically retry this action.
|
||||
error.sync.tryAgainButton.label = Sync Now
|
||||
error.sync.tryAgainButton.accesskey = S
|
||||
|
|
|
|||
|
|
@ -40,20 +40,23 @@ const EXPORTED_SYMBOLS = ["WEAVE_VERSION", "COMPATIBLE_VERSION",
|
|||
'MODE_CREATE', 'MODE_APPEND', 'MODE_TRUNCATE',
|
||||
'PERMS_FILE', 'PERMS_PASSFILE', 'PERMS_DIRECTORY',
|
||||
'ONE_BYTE', 'ONE_KILOBYTE', 'ONE_MEGABYTE',
|
||||
'CONNECTION_TIMEOUT', 'MAX_UPLOAD_RECORDS',
|
||||
'WEAVE_STATUS_OK', 'WEAVE_STATUS_FAILED',
|
||||
'WEAVE_STATUS_PARTIAL', 'SERVER_LOW_QUOTA',
|
||||
'SERVER_DOWNTIME', 'SERVER_UNREACHABLE',
|
||||
'LOGIN_FAILED_NO_USERNAME', 'LOGIN_FAILED_NO_PASSWORD',
|
||||
'LOGIN_FAILED_NETWORK_ERROR','LOGIN_FAILED_INVALID_PASSPHRASE',
|
||||
'LOGIN_FAILED_LOGIN_REJECTED', 'METARECORD_DOWNLOAD_FAIL',
|
||||
'VERSION_OUT_OF_DATE', 'DESKTOP_VERSION_OUT_OF_DATE',
|
||||
'KEYS_DOWNLOAD_FAIL', 'NO_KEYS_NO_KEYGEN', 'KEYS_UPLOAD_FAIL',
|
||||
'SETUP_FAILED_NO_PASSPHRASE', 'ABORT_SYNC_COMMAND',
|
||||
'kSyncWeaveDisabled', 'kSyncNotLoggedIn',
|
||||
'kSyncNetworkOffline', 'kSyncInPrivateBrowsing',
|
||||
'kSyncNotScheduled',
|
||||
'FIREFOX_ID', 'THUNDERBIRD_ID', 'FENNEC_ID', 'SEAMONKEY_ID'];
|
||||
'CONNECTION_TIMEOUT', 'MAX_UPLOAD_RECORDS',
|
||||
'SYNC_SUCCEEDED', 'LOGIN_SUCCEEDED', 'ENGINE_SUCCEEDED',
|
||||
'STATUS_OK', 'LOGIN_FAILED', 'SYNC_FAILED',
|
||||
'SYNC_FAILED_PARTIAL', 'STATUS_DISABLED', 'SERVER_LOW_QUOTA',
|
||||
'SERVER_DOWNTIME', 'SERVER_UNREACHABLE',
|
||||
'LOGIN_FAILED_NO_USERNAME', 'LOGIN_FAILED_NO_PASSWORD',
|
||||
'LOGIN_FAILED_NETWORK_ERROR','LOGIN_FAILED_INVALID_PASSPHRASE',
|
||||
'LOGIN_FAILED_LOGIN_REJECTED', 'METARECORD_DOWNLOAD_FAIL',
|
||||
'VERSION_OUT_OF_DATE', 'DESKTOP_VERSION_OUT_OF_DATE',
|
||||
'KEYS_DOWNLOAD_FAIL', 'NO_KEYS_NO_KEYGEN', 'KEYS_UPLOAD_FAIL',
|
||||
'ENGINE_UPLOAD_FAIL', 'ENGINE_DOWNLOAD_FAIL', 'ENGINE_UNKNOWN_FAIL',
|
||||
'ENGINE_METARECORD_UPLOAD_FAIL',
|
||||
'SETUP_FAILED_NO_PASSPHRASE', 'ABORT_SYNC_COMMAND',
|
||||
'kSyncWeaveDisabled', 'kSyncNotLoggedIn',
|
||||
'kSyncNetworkOffline', 'kSyncInPrivateBrowsing',
|
||||
'kSyncNotScheduled',
|
||||
'FIREFOX_ID', 'THUNDERBIRD_ID', 'FENNEC_ID', 'SEAMONKEY_ID'];
|
||||
|
||||
const WEAVE_VERSION = "@weave_version@";
|
||||
|
||||
|
|
@ -89,36 +92,52 @@ const CONNECTION_TIMEOUT = 30000;
|
|||
const MAX_UPLOAD_RECORDS = 100;
|
||||
|
||||
// Top-level statuses:
|
||||
const WEAVE_STATUS_OK = "Sync succeeded.";
|
||||
const WEAVE_STATUS_FAILED = "Sync failed.";
|
||||
const WEAVE_STATUS_PARTIAL = "Sync partially succeeded, some data failed to sync.";
|
||||
const STATUS_OK = "success.status_ok";
|
||||
const SYNC_FAILED = "error.sync.failed";
|
||||
const LOGIN_FAILED = "error.login.failed";
|
||||
const SYNC_FAILED_PARTIAL = "error.sync.failed_partial";
|
||||
const STATUS_DISABLED = "service.disabled";
|
||||
|
||||
// success states
|
||||
const LOGIN_SUCCEEDED = "success.login";
|
||||
const SYNC_SUCCEEDED = "success.sync";
|
||||
const ENGINE_SUCCEEDED = "success.engine";
|
||||
|
||||
// login failure status codes:
|
||||
const LOGIN_FAILED_NO_USERNAME = "error.login.reason.no_username";
|
||||
const LOGIN_FAILED_NO_PASSWORD = "error.login.reason.no_password";
|
||||
const LOGIN_FAILED_NETWORK_ERROR = "error.login.reason.network";
|
||||
const LOGIN_FAILED_INVALID_PASSPHRASE = "error.login.reason.passphrase.";
|
||||
const LOGIN_FAILED_LOGIN_REJECTED = "error.login.reason.password";
|
||||
|
||||
// sync failure status codes
|
||||
const METARECORD_DOWNLOAD_FAIL = "error.sync.reason.metarecord_download_fail";
|
||||
const VERSION_OUT_OF_DATE = "error.sync.reason.version_out_of_date";
|
||||
const DESKTOP_VERSION_OUT_OF_DATE = "error.sync.reason.desktop_version_out_of_date";
|
||||
const KEYS_DOWNLOAD_FAIL = "error.sync.reason.keys_download_fail";
|
||||
const NO_KEYS_NO_KEYGEN = "error.sync.reason.no_keys_no_keygen";
|
||||
const KEYS_UPLOAD_FAIL = "error.sync.reason.keys_upload_fail";
|
||||
const SETUP_FAILED_NO_PASSPHRASE = "error.sync.reason.setup_failed_no_passphrase";
|
||||
|
||||
// engine failure status codes
|
||||
const ENGINE_UPLOAD_FAIL = "error.engine.reason.record_upload_fail";
|
||||
const ENGINE_DOWNLOAD_FAIL = "error.engine.reason.record_download_fail";
|
||||
const ENGINE_UNKNOWN_FAIL = "error.engine.reason.unknown_fail";
|
||||
const ENGINE_METARECORD_UPLOAD_FAIL = "error.engine.reason.metarecord_upload_fail";
|
||||
|
||||
// Server statuses (Not mutually exclusive):
|
||||
const SERVER_LOW_QUOTA = "Getting close to your Weave server storage quota.";
|
||||
const SERVER_DOWNTIME = "Weave server is overloaded, try agian in 30 sec.";
|
||||
const SERVER_UNREACHABLE = "Weave server is unreachable.";
|
||||
|
||||
// Ways that a sync can fail during setup or login:
|
||||
const LOGIN_FAILED_NO_USERNAME = "No username set, login failed.";
|
||||
const LOGIN_FAILED_NO_PASSWORD = "No password set, login failed.";
|
||||
const LOGIN_FAILED_NETWORK_ERROR = "Weave failed to connect to the server.";
|
||||
const LOGIN_FAILED_INVALID_PASSPHRASE = "Incorrect passphrase given.";
|
||||
const LOGIN_FAILED_LOGIN_REJECTED = "Incorrect username or password.";
|
||||
const METARECORD_DOWNLOAD_FAIL = "Can't download metadata record, HTTP error.";
|
||||
const VERSION_OUT_OF_DATE = "This copy of Weave needs to be updated.";
|
||||
const DESKTOP_VERSION_OUT_OF_DATE = "Weave needs updating on your desktop browser.";
|
||||
const KEYS_DOWNLOAD_FAIL = "Can't download keys from server, HTTP error.";
|
||||
const NO_KEYS_NO_KEYGEN = "Key generation disabled. Sync from the desktop first.";
|
||||
const KEYS_UPLOAD_FAIL = "Could not upload keys.";
|
||||
const SETUP_FAILED_NO_PASSPHRASE = "Could not get encryption passphrase.";
|
||||
|
||||
// Ways that a sync can be disabled
|
||||
const kSyncWeaveDisabled = "Weave is disabled";
|
||||
const kSyncNotLoggedIn = "User is not logged in";
|
||||
const kSyncNetworkOffline = "Network is offline";
|
||||
const kSyncInPrivateBrowsing = "Private browsing is enabled";
|
||||
const kSyncNotScheduled = "Not scheduled to do sync";
|
||||
// If one of these happens, leave the top-level status the same!
|
||||
const kSyncBackoffNotMet = "Trying to sync before the server said it's okay";
|
||||
|
||||
|
||||
// Ways that a sync can be aborted:
|
||||
const ABORT_SYNC_COMMAND = "aborting sync, process commands said so";
|
||||
|
|
|
|||
|
|
@ -361,9 +361,12 @@ SyncEngine.prototype = {
|
|||
meta.generateIV();
|
||||
meta.addUnwrappedKey(pubkey, symkey);
|
||||
let res = new Resource(meta.uri);
|
||||
let resp = res.put(meta);
|
||||
if (!resp.success)
|
||||
let resp = res.put(meta.serialize());
|
||||
if (!resp.success) {
|
||||
this._log.debug("Metarecord upload fail:" + resp);
|
||||
resp.failureCode = ENGINE_METARECORD_UPLOAD_FAIL;
|
||||
throw resp;
|
||||
}
|
||||
|
||||
// Cache the cryto meta that we just put on the server
|
||||
CryptoMetas.set(meta.uri, meta);
|
||||
|
|
@ -430,8 +433,10 @@ SyncEngine.prototype = {
|
|||
// Only bother getting data from the server if there's new things
|
||||
if (this.lastModified > this.lastSync) {
|
||||
let resp = newitems.get();
|
||||
if (!resp.success)
|
||||
if (!resp.success) {
|
||||
resp.failureCode = ENGINE_DOWNLOAD_FAIL;
|
||||
throw resp;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we got the maximum that we requested; get the rest if so
|
||||
|
|
@ -464,8 +469,11 @@ SyncEngine.prototype = {
|
|||
|
||||
// Reuse the existing record handler set earlier
|
||||
let resp = newitems.get();
|
||||
if (!resp.success)
|
||||
if (!resp.success) {
|
||||
resp.failureCode = ENGINE_DOWNLOAD_FAIL;
|
||||
throw resp;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (this.lastSync < this.lastModified)
|
||||
|
|
@ -591,8 +599,11 @@ SyncEngine.prototype = {
|
|||
let doUpload = Utils.bind2(this, function(desc) {
|
||||
this._log.info("Uploading " + desc + " of " + outnum + " records");
|
||||
let resp = up.post();
|
||||
if (!resp.success)
|
||||
if (!resp.success) {
|
||||
this._log.debug("Uploading records failed: " + resp);
|
||||
resp.failureCode = ENGINE_UPLOAD_FAIL;
|
||||
throw resp;
|
||||
}
|
||||
|
||||
// Record the modified time of the upload
|
||||
let modified = resp.headers["X-Weave-Timestamp"];
|
||||
|
|
|
|||
|
|
@ -252,6 +252,10 @@ Resource.prototype = {
|
|||
|
||||
this._log[log](mesg);
|
||||
}
|
||||
|
||||
// this is a server-side safety valve to allow slowing down clients without hurting performance
|
||||
if (headers["X-Weave-Backoff"])
|
||||
Observers.notify("weave:service:backoff:interval", parseInt(headers["X-Weave-Backoff"], 10))
|
||||
}
|
||||
// Got a response but no header; must be cached (use default values)
|
||||
catch(ex) {
|
||||
|
|
|
|||
|
|
@ -122,21 +122,44 @@ function StatusRecord() {
|
|||
}
|
||||
StatusRecord.prototype = {
|
||||
_init: function() {
|
||||
this.server = [];
|
||||
this.service = null;
|
||||
this.sync = null;
|
||||
this.login = null;
|
||||
this.engines = {};
|
||||
},
|
||||
|
||||
addServerStatus: function(statusCode) {
|
||||
this.server.push(statusCode);
|
||||
this._resetBackoff();
|
||||
},
|
||||
|
||||
setSyncStatus: function(statusCode) {
|
||||
if (statusCode == SYNC_SUCCEEDED) {
|
||||
this.service == STATUS_OK;
|
||||
this._resetBackoff();
|
||||
}
|
||||
else
|
||||
this.service = SYNC_FAILED;
|
||||
|
||||
this.sync = statusCode;
|
||||
},
|
||||
|
||||
setLoginStatus: function(statusCode) {
|
||||
this.service = statusCode == LOGIN_SUCCEEDED ? STATUS_OK : LOGIN_FAILED;
|
||||
this.login = statusCode;
|
||||
},
|
||||
|
||||
setEngineStatus: function(engineName, statusCode) {
|
||||
if (statusCode != ENGINE_SUCCEEDED)
|
||||
this.service = this.sync = SYNC_FAILED_PARTIAL;
|
||||
|
||||
this.engines[engineName] = statusCode;
|
||||
},
|
||||
|
||||
resetEngineStatus: function() {
|
||||
this.engines = {};
|
||||
},
|
||||
|
||||
_resetBackoff: function () {
|
||||
this.enforceBackoff = false;
|
||||
this.backoffInterval = 0;
|
||||
this.minimumNextSync = 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -159,10 +182,8 @@ WeaveSvc.prototype = {
|
|||
_syncInProgress: false,
|
||||
_keyGenEnabled: true,
|
||||
|
||||
// WEAVE_STATUS_OK, WEAVE_STATUS_FAILED, or WEAVE_STATUS_PARTIAL
|
||||
_weaveStatusCode: null,
|
||||
// More detailed status info:
|
||||
_detailedStatus: null,
|
||||
// the status object
|
||||
_status: null,
|
||||
|
||||
// object for caching public and private keys
|
||||
_keyPair: {},
|
||||
|
|
@ -231,8 +252,7 @@ WeaveSvc.prototype = {
|
|||
get enabled() { return Svc.Prefs.get("enabled"); },
|
||||
set enabled(value) { Svc.Prefs.set("enabled", value); },
|
||||
|
||||
get statusCode() { return this._weaveStatusCode; },
|
||||
get detailedStatus() { return this._detailedStatus; },
|
||||
get status() { return this._status; },
|
||||
|
||||
get locked() { return this._locked; },
|
||||
lock: function Svc_lock() {
|
||||
|
|
@ -245,11 +265,6 @@ WeaveSvc.prototype = {
|
|||
this._locked = false;
|
||||
},
|
||||
|
||||
_setSyncFailure: function WeavSvc__setSyncFailure(code) {
|
||||
this._weaveStatusCode = WEAVE_STATUS_FAILED;
|
||||
this._detailedStatus.setSyncStatus(code);
|
||||
},
|
||||
|
||||
_genKeyURLs: function WeaveSvc__genKeyURLs() {
|
||||
let url = this.userURL;
|
||||
PubKeys.defaultKeyUri = url + "/storage/keys/pubkey";
|
||||
|
|
@ -281,7 +296,7 @@ WeaveSvc.prototype = {
|
|||
this._initLogs();
|
||||
this._log.info("Weave " + WEAVE_VERSION + " initializing");
|
||||
this._registerEngines();
|
||||
this._detailedStatus = new StatusRecord();
|
||||
this._status = new StatusRecord();
|
||||
|
||||
// Reset our sync id if we're upgrading, so sync knows to reset local data
|
||||
if (WEAVE_VERSION != Svc.Prefs.get("lastversion")) {
|
||||
|
|
@ -305,6 +320,7 @@ WeaveSvc.prototype = {
|
|||
Svc.Observer.addObserver(this, "quit-application", true);
|
||||
Svc.Observer.addObserver(this, "weave:service:sync:finish", true);
|
||||
Svc.Observer.addObserver(this, "weave:service:sync:error", true);
|
||||
Svc.Observer.addObserver(this, "weave:service:backoff:interval", true);
|
||||
|
||||
if (!this.enabled)
|
||||
this._log.info("Weave Sync disabled");
|
||||
|
|
@ -417,7 +433,12 @@ WeaveSvc.prototype = {
|
|||
break;
|
||||
case "weave:service:sync:finish":
|
||||
this._scheduleNextSync();
|
||||
this._serverErrors = 0;
|
||||
this._syncErrors = 0;
|
||||
break;
|
||||
case "weave:service:backoff:interval":
|
||||
let interval = data + Math.random() * data * 0.25; // required backoff + up to 25%
|
||||
this.status.backoffInterval = interval;
|
||||
this.status.minimumNextSync = Date.now() + data;
|
||||
break;
|
||||
case "idle":
|
||||
this._log.trace("Idle time hit, trying to sync");
|
||||
|
|
@ -452,7 +473,7 @@ WeaveSvc.prototype = {
|
|||
}
|
||||
} catch (e) {
|
||||
this._log.debug("Network error on findCluster");
|
||||
this._setSyncFailure(LOGIN_FAILED_NETWORK_ERROR);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_NETWORK_ERROR);
|
||||
throw e;
|
||||
}
|
||||
},
|
||||
|
|
@ -495,24 +516,26 @@ WeaveSvc.prototype = {
|
|||
switch (test.status) {
|
||||
case 200:
|
||||
if (!this._verifyPassphrase()) {
|
||||
this._setSyncFailure(LOGIN_FAILED_INVALID_PASSPHRASE);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_INVALID_PASSPHRASE);
|
||||
return false;
|
||||
}
|
||||
this.status.setLoginStatus(LOGIN_SUCCEEDED);
|
||||
return true;
|
||||
case 401:
|
||||
if (this._updateCluster())
|
||||
return this._verifyLogin();
|
||||
|
||||
this._setSyncFailure(LOGIN_FAILED_LOGIN_REJECTED);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_LOGIN_REJECTED);
|
||||
this._log.debug("verifyLogin failed: login failed")
|
||||
return false;
|
||||
default:
|
||||
this._checkServerError(test.status);
|
||||
throw "unexpected HTTP response: " + test.status;
|
||||
}
|
||||
} catch (e) {
|
||||
// if we get here, we have either a busted channel or a network error
|
||||
this._log.debug("verifyLogin failed: " + e)
|
||||
this._setSyncFailure(LOGIN_FAILED_NETWORK_ERROR);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_NETWORK_ERROR);
|
||||
throw e;
|
||||
}
|
||||
}))(),
|
||||
|
|
@ -606,7 +629,7 @@ WeaveSvc.prototype = {
|
|||
return;
|
||||
}
|
||||
|
||||
failureReason = this.detailedStatus.sync;
|
||||
failureReason = this.status.sync;
|
||||
}
|
||||
catch (ex) {
|
||||
failureReason = ex;
|
||||
|
|
@ -621,10 +644,9 @@ WeaveSvc.prototype = {
|
|||
}));
|
||||
this._autoConnectTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
|
||||
|
||||
// back off slowly, with some random fuzz to avoid repeated collisions
|
||||
let interval = Math.floor(Math.random() * SCHEDULED_SYNC_INTERVAL +
|
||||
SCHEDULED_SYNC_INTERVAL * this._autoConnectAttempts);
|
||||
this._autoConnectAttempts++;
|
||||
let interval = this._calculateBackoff(this._autoConnectAttempts,
|
||||
SCHEDULED_SYNC_INTERVAL);
|
||||
this._autoConnectTimer.initWithCallback(listener, interval,
|
||||
Ci.nsITimer.TYPE_ONE_SHOT);
|
||||
this._log.debug("Scheduling next autoconnect attempt in " +
|
||||
|
|
@ -643,7 +665,6 @@ WeaveSvc.prototype = {
|
|||
login: function WeaveSvc_login(username, password, passphrase)
|
||||
this._catch(this._lock(this._notify("login", "", function() {
|
||||
this._loggedIn = false;
|
||||
this._detailedStatus = new StatusRecord();
|
||||
if (Svc.IO.offline)
|
||||
throw "Application is offline, login should not be called";
|
||||
|
||||
|
|
@ -655,22 +676,22 @@ WeaveSvc.prototype = {
|
|||
this.passphrase = passphrase;
|
||||
|
||||
if (!this.username) {
|
||||
this._setSyncFailure(LOGIN_FAILED_NO_USERNAME);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_NO_USERNAME);
|
||||
throw "No username set, login failed";
|
||||
}
|
||||
if (!this.password) {
|
||||
this._setSyncFailure(LOGIN_FAILED_NO_PASSWORD);
|
||||
this.status.setLoginStatus(LOGIN_FAILED_NO_PASSWORD);
|
||||
throw "No password given or found in password manager";
|
||||
}
|
||||
this._log.info("Logging in user " + this.username);
|
||||
|
||||
if (!this._verifyLogin()) {
|
||||
// verifyLogin sets the failure states here
|
||||
throw "Login failed: " + this.detailedStatus.sync;
|
||||
throw "Login failed: " + this.status.login;
|
||||
}
|
||||
|
||||
// Try starting the sync timer now that we're logged in
|
||||
this._loggedIn = true;
|
||||
// Try starting the sync timer now that we're logged in
|
||||
this._checkSyncStatus();
|
||||
if (this._autoConnectTimer) {
|
||||
this._autoConnectTimer.cancel();
|
||||
|
|
@ -785,7 +806,8 @@ WeaveSvc.prototype = {
|
|||
// abort the server wipe if the GET status was anything other than 404 or 200
|
||||
let status = Records.response.status;
|
||||
if (status != 200 && status != 404) {
|
||||
this._setSyncFailure(METARECORD_DOWNLOAD_FAIL);
|
||||
this._checkServerError(Records.response);
|
||||
this.status.setSyncStatus(METARECORD_DOWNLOAD_FAIL);
|
||||
this._log.warn("Unknown error while downloading metadata record. " +
|
||||
"Aborting sync.");
|
||||
return false;
|
||||
|
|
@ -801,7 +823,7 @@ WeaveSvc.prototype = {
|
|||
if (!this._keyGenEnabled) {
|
||||
this._log.info("...and key generation is disabled. Not wiping. " +
|
||||
"Aborting sync.");
|
||||
this._setSyncFailure(DESKTOP_VERSION_OUT_OF_DATE);
|
||||
this.status.setSyncStatus(DESKTOP_VERSION_OUT_OF_DATE);
|
||||
return false;
|
||||
}
|
||||
reset = true;
|
||||
|
|
@ -816,7 +838,7 @@ WeaveSvc.prototype = {
|
|||
"upgrade (" + remoteVersion + " -> " + WEAVE_VERSION + ")");
|
||||
|
||||
} else if (Svc.Version.compare(remoteVersion, WEAVE_VERSION) > 0) {
|
||||
this._setSyncFailure(VERSION_OUT_OF_DATE);
|
||||
this.status.setSyncStatus(VERSION_OUT_OF_DATE);
|
||||
this._log.warn("Server data is of a newer Weave version, this client " +
|
||||
"needs to be upgraded. Aborting sync.");
|
||||
return false;
|
||||
|
|
@ -856,14 +878,16 @@ WeaveSvc.prototype = {
|
|||
this._log.warn("Couldn't download keys from server, aborting sync");
|
||||
this._log.debug("PubKey HTTP status: " + PubKeys.response.status);
|
||||
this._log.debug("PrivKey HTTP status: " + PrivKeys.response.status);
|
||||
this._setSyncFailure(KEYS_DOWNLOAD_FAIL);
|
||||
this._checkServerError(PubKeys.response);
|
||||
this._checkServerError(PrivKeys.response);
|
||||
this.status.setSyncStatus(KEYS_DOWNLOAD_FAIL);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!this._keyGenEnabled) {
|
||||
this._log.warn("Couldn't download keys from server, and key generation" +
|
||||
"is disabled. Aborting sync");
|
||||
this._setSyncFailure(NO_KEYS_NO_KEYGEN);
|
||||
this.status.setSyncStatus(NO_KEYS_NO_KEYGEN);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -884,11 +908,11 @@ WeaveSvc.prototype = {
|
|||
PrivKeys.set(keys.privkey.uri, keys.privkey);
|
||||
return true;
|
||||
} catch (e) {
|
||||
this._setSyncFailure(KEYS_UPLOAD_FAIL);
|
||||
this.status.setSyncStatus(KEYS_UPLOAD_FAIL);
|
||||
this._log.error("Could not upload keys: " + Utils.exceptionStr(e));
|
||||
}
|
||||
} else {
|
||||
this._setSyncFailure(SETUP_FAILED_NO_PASSPHRASE);
|
||||
this.status.setSyncStatus(SETUP_FAILED_NO_PASSPHRASE);
|
||||
this._log.warn("Could not get encryption passphrase");
|
||||
}
|
||||
}
|
||||
|
|
@ -921,6 +945,8 @@ WeaveSvc.prototype = {
|
|||
reason = kSyncInPrivateBrowsing;
|
||||
else if (Svc.Prefs.get("schedule", 0) != 1)
|
||||
reason = kSyncNotScheduled;
|
||||
else if (this.status.minimumNextSync > Date.now())
|
||||
reason = kSyncBackoffNotMet;
|
||||
|
||||
return reason;
|
||||
},
|
||||
|
|
@ -930,7 +956,9 @@ WeaveSvc.prototype = {
|
|||
*/
|
||||
_checkSyncStatus: function WeaveSvc__checkSyncStatus() {
|
||||
// Should we be syncing now, if not, cancel any sync timers and return
|
||||
if (this._checkSync()) {
|
||||
// if we're in backoff, we'll schedule the next sync
|
||||
let reason = this._checkSync();
|
||||
if (reason && reason != kSyncBackoffNotMet) {
|
||||
if (this._syncTimer) {
|
||||
this._syncTimer.cancel();
|
||||
this._syncTimer = null;
|
||||
|
|
@ -940,6 +968,7 @@ WeaveSvc.prototype = {
|
|||
Svc.Idle.removeIdleObserver(this, IDLE_TIME);
|
||||
} catch(e) {} // this throws if there isn't an observer, but that's fine
|
||||
|
||||
this.status.service = STATUS_DISABLED;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -962,7 +991,7 @@ WeaveSvc.prototype = {
|
|||
*/
|
||||
_scheduleNextSync: function WeaveSvc__scheduleNextSync(interval) {
|
||||
if (!interval)
|
||||
interval = SCHEDULED_SYNC_INTERVAL;
|
||||
interval = this.status.backoffInterval || SCHEDULED_SYNC_INTERVAL;
|
||||
|
||||
// if there's an existing timer, cancel it and restart
|
||||
if (this._syncTimer)
|
||||
|
|
@ -980,61 +1009,28 @@ WeaveSvc.prototype = {
|
|||
this._log.debug("Next sync call in: " + this._syncTimer.delay / 1000 + " seconds.")
|
||||
},
|
||||
|
||||
_serverErrors: 0,
|
||||
_syncErrors: 0,
|
||||
/**
|
||||
* Deal with sync errors appropriately
|
||||
*/
|
||||
_handleSyncError: function WeaveSvc__handleSyncError() {
|
||||
let shouldBackoff = false;
|
||||
this._syncErrors++;
|
||||
|
||||
let err = Weave.Service.detailedStatus.sync;
|
||||
// we'll assume the server is just borked a little for these
|
||||
switch (err) {
|
||||
case METARECORD_DOWNLOAD_FAIL:
|
||||
case KEYS_DOWNLOAD_FAIL:
|
||||
case KEYS_UPLOAD_FAIL:
|
||||
shouldBackoff = true;
|
||||
}
|
||||
|
||||
// specifcally handle 500, 502, 503, 504 errors
|
||||
// xxxmpc: what else should be in this list?
|
||||
// this is sort of pseudocode, need a way to get at the
|
||||
if (!shouldBackoff) {
|
||||
try {
|
||||
shouldBackoff = Utils.checkStatus(Records.response.status, null,
|
||||
[500, [502, 504]]);
|
||||
}
|
||||
catch (e) {
|
||||
// if responseStatus throws, we have a network issue in play
|
||||
shouldBackoff = true;
|
||||
// do nothing on the first couple of failures, if we're not in backoff due to 5xx errors
|
||||
if (!this.status.enforceBackoff) {
|
||||
if (this._syncErrors < 3) {
|
||||
this._scheduleNextSync();
|
||||
return;
|
||||
}
|
||||
this.status.enforceBackoff = true;
|
||||
}
|
||||
|
||||
// if this is a client error, do the next sync as normal and return
|
||||
if (!shouldBackoff) {
|
||||
this._scheduleNextSync();
|
||||
return;
|
||||
}
|
||||
const MINIMUM_BACKOFF_INTERVAL = 15 * 60 * 1000; // 15 minutes
|
||||
let interval = this._calculateBackoff(this._syncErrors, MINIMUM_BACKOFF_INTERVAL);
|
||||
|
||||
// ok, something failed connecting to the server, rev the counter
|
||||
this._serverErrors++;
|
||||
this._scheduleNextSync(interval);
|
||||
|
||||
// do nothing on the first failure, if we fail again we'll back off
|
||||
if (this._serverErrors < 2) {
|
||||
this._scheduleNextSync();
|
||||
return;
|
||||
}
|
||||
|
||||
// 30-60 minute backoff interval, increasing each time
|
||||
const MINIMUM_BACKOFF_INTERVAL = 15 * 60 * 1000; // 15 minutes * >= 2
|
||||
const MAXIMUM_BACKOFF_INTERVAL = 8 * 60 * 60 * 1000; // 8 hours
|
||||
let backoffInterval = this._serverErrors *
|
||||
(Math.floor(Math.random() * MINIMUM_BACKOFF_INTERVAL) +
|
||||
MINIMUM_BACKOFF_INTERVAL);
|
||||
backoffInterval = Math.min(backoffInterval, MAXIMUM_BACKOFF_INTERVAL);
|
||||
this._scheduleNextSync(backoffInterval);
|
||||
|
||||
let d = new Date(Date.now() + backoffInterval);
|
||||
let d = new Date(Date.now() + interval);
|
||||
this._log.config("Starting backoff, next sync at:" + d.toString());
|
||||
},
|
||||
|
||||
|
|
@ -1046,7 +1042,7 @@ WeaveSvc.prototype = {
|
|||
*/
|
||||
sync: function WeaveSvc_sync(fullSync)
|
||||
this._catch(this._lock(this._notify("sync", "", function() {
|
||||
|
||||
this.status.resetEngineStatus();
|
||||
fullSync = true; // not doing thresholds yet
|
||||
|
||||
// Use thresholds to determine what to sync only if it's not a full sync
|
||||
|
|
@ -1058,12 +1054,16 @@ WeaveSvc.prototype = {
|
|||
let reason = this._checkSync();
|
||||
if (reason && (useThresh || reason != kSyncNotScheduled)) {
|
||||
// this is a purposeful abort rather than a failure, so don't set
|
||||
// WEAVE_STATUS_FAILED; instead, leave it as it was.
|
||||
this._detailedStatus.setSyncStatus(reason);
|
||||
// any status bits
|
||||
reason = "Can't sync: " + reason;
|
||||
throw reason;
|
||||
}
|
||||
|
||||
if (this._autoConnectTimer) {
|
||||
this._autoConnectTimer.cancel();
|
||||
this._autoConnectTimer = null;
|
||||
}
|
||||
|
||||
if (!(this._remoteSetup()))
|
||||
throw "aborting sync, remote setup failed";
|
||||
|
||||
|
|
@ -1083,7 +1083,7 @@ WeaveSvc.prototype = {
|
|||
if (Clients.getClients()[Clients.clientID].commands) {
|
||||
try {
|
||||
if (!(this.processCommands())) {
|
||||
this._detailedStatus.setSyncStatus(ABORT_SYNC_COMMAND);
|
||||
this.status.setSyncStatus(ABORT_SYNC_COMMAND);
|
||||
throw "aborting sync, process commands said so";
|
||||
}
|
||||
|
||||
|
|
@ -1132,7 +1132,7 @@ WeaveSvc.prototype = {
|
|||
}
|
||||
|
||||
// If there's any problems with syncing the engine, report the failure
|
||||
if (!(this._syncEngine(engine))) {
|
||||
if (!(this._syncEngine(engine)) || this.status.enforceBackoff) {
|
||||
this._log.info("Aborting sync");
|
||||
break;
|
||||
}
|
||||
|
|
@ -1147,7 +1147,7 @@ WeaveSvc.prototype = {
|
|||
this._log.warn("Some engines did not sync correctly");
|
||||
else {
|
||||
Svc.Prefs.set("lastSync", new Date().toString());
|
||||
this._weaveStatusCode = WEAVE_STATUS_OK;
|
||||
this.status.setSyncStatus(SYNC_SUCCEEDED);
|
||||
this._log.info("Sync completed successfully");
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -1167,9 +1167,10 @@ WeaveSvc.prototype = {
|
|||
if (e.status == 401 && this._updateCluster())
|
||||
return this._syncEngine(engine);
|
||||
|
||||
this._checkServerError(e);
|
||||
|
||||
this.status.setEngineStatus(engine.name, e.failureCode || ENGINE_UNKNOWN_FAIL);
|
||||
this._syncError = true;
|
||||
this._weaveStatusCode = WEAVE_STATUS_PARTIAL;
|
||||
this._detailedStatus.setEngineStatus(engine.name, e);
|
||||
this._log.debug(Utils.exceptionStr(e));
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1201,6 +1202,30 @@ WeaveSvc.prototype = {
|
|||
if (!resp.success)
|
||||
throw resp;
|
||||
},
|
||||
|
||||
|
||||
/**
|
||||
* Check to see if this is a failure
|
||||
*
|
||||
*/
|
||||
_checkServerError: function WeaveSvc__checkServerError(resp) {
|
||||
if (Utils.checkStatus(resp.status, null, [500, [502, 504]])) {
|
||||
this.status.enforceBackoff = true;
|
||||
if (resp.status == 503 && resp.headers["Retry-After"])
|
||||
Observers.notify("weave:service:backoff:interval", parseInt(resp.headers["Retry-After"], 10));
|
||||
}
|
||||
},
|
||||
/**
|
||||
* Return a value for a backoff interval. Maximum is eight hours, unless this.status.backoffInterval is higher.
|
||||
*
|
||||
*/
|
||||
_calculateBackoff: function WeaveSvc__calculateBackoff(attempts, base_interval) {
|
||||
const MAXIMUM_BACKOFF_INTERVAL = 8 * 60 * 60 * 1000; // 8 hours
|
||||
let backoffInterval = attempts *
|
||||
(Math.floor(Math.random() * base_interval) +
|
||||
base_interval);
|
||||
return Math.max(Math.min(backoffInterval, MAXIMUM_BACKOFF_INTERVAL), this.status.backoffInterval);
|
||||
},
|
||||
|
||||
/**
|
||||
* Wipe all user data from the server.
|
||||
|
|
|
|||
|
|
@ -699,21 +699,12 @@ let Utils = {
|
|||
},
|
||||
|
||||
getErrorString: function Utils_getErrorString(error, args) {
|
||||
switch (error) {
|
||||
case Weave.LOGIN_FAILED_NETWORK_ERROR:
|
||||
errorString = "error.login.reason.network";
|
||||
break;
|
||||
case Weave.LOGIN_FAILED_INVALID_PASSPHRASE:
|
||||
errorString = "error.login.reason.passphrase";
|
||||
break;
|
||||
case Weave.LOGIN_FAILED_LOGIN_REJECTED:
|
||||
errorString = "error.login.reason.password";
|
||||
break;
|
||||
default:
|
||||
errorString = "error.login.reason.unknown";
|
||||
break;
|
||||
}
|
||||
return this._errorBundle.get(errorString, args || null);
|
||||
try {
|
||||
return this._errorBundle.get(error, args || null);
|
||||
} catch (e) {}
|
||||
|
||||
// basically returns "Unknown Error"
|
||||
return this._errorBundle.get("error.reason.unknown");
|
||||
},
|
||||
|
||||
// assumes an nsIConverterInputStream
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче