From e34471c4c9d46d55f3e559c403d13c73374a54d9 Mon Sep 17 00:00:00 2001 From: Irving Reid Date: Thu, 8 Aug 2013 15:56:26 -0400 Subject: [PATCH] Bug 853388: Upgrade existing SQLITE databases to JSON; r=unfocused --- toolkit/mozapps/extensions/XPIProvider.jsm | 30 +- .../mozapps/extensions/XPIProviderUtils.js | 609 +++++++----------- .../extensions/test/xpcshell/head_addons.js | 10 +- .../extensions/test/xpcshell/test_bad_json.js | 54 ++ .../test/xpcshell/test_bootstrap.js | 10 +- .../test/xpcshell/test_bug559800.js | 10 +- .../extensions/test/xpcshell/test_corrupt.js | 6 +- .../xpcshell/test_corrupt_strictcompat.js | 6 +- .../extensions/test/xpcshell/test_locked.js | 13 +- .../extensions/test/xpcshell/test_locked2.js | 12 +- .../test/xpcshell/test_locked_strictcompat.js | 12 +- .../extensions/test/xpcshell/xpcshell.ini | 27 +- 12 files changed, 320 insertions(+), 479 deletions(-) create mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_bad_json.js diff --git a/toolkit/mozapps/extensions/XPIProvider.jsm b/toolkit/mozapps/extensions/XPIProvider.jsm index ca763ec78741..2794fd309749 100644 --- a/toolkit/mozapps/extensions/XPIProvider.jsm +++ b/toolkit/mozapps/extensions/XPIProvider.jsm @@ -595,14 +595,8 @@ function isAddonDisabled(aAddon) { return aAddon.appDisabled || aAddon.softDisabled || aAddon.userDisabled; } -Object.defineProperty(this, "gRDF", { - get: function gRDFGetter() { - delete this.gRDF; - return this.gRDF = Cc["@mozilla.org/rdf/rdf-service;1"]. - getService(Ci.nsIRDFService); - }, - configurable: true -}); +XPCOMUtils.defineLazyServiceGetter(this, "gRDF", "@mozilla.org/rdf/rdf-service;1", + Ci.nsIRDFService); function EM_R(aProperty) { return gRDF.GetResource(PREFIX_NS_EM + aProperty); @@ -1765,7 +1759,7 @@ var XPIProvider = { null); this.minCompatiblePlatformVersion = Prefs.getCharPref(PREF_EM_MIN_COMPAT_PLATFORM_VERSION, null); - this.enabledAddons = []; + this.enabledAddons = ""; Services.prefs.addObserver(PREF_EM_MIN_COMPAT_APP_VERSION, this, false); Services.prefs.addObserver(PREF_EM_MIN_COMPAT_PLATFORM_VERSION, this, false); @@ -2904,6 +2898,7 @@ var XPIProvider = { let newDBAddon = null; try { // Update the database. + // XXX I don't think this can throw any more newDBAddon = XPIDatabase.addAddonMetadata(newAddon, aAddonState.descriptor); } catch (e) { @@ -3005,9 +3000,8 @@ var XPIProvider = { let addonStates = aSt.addons; // Check if the database knows about any add-ons in this install location. - let pos = knownLocations.indexOf(installLocation.name); - if (pos >= 0) { - knownLocations.splice(pos, 1); + if (knownLocations.has(installLocation.name)) { + knownLocations.delete(installLocation.name); let addons = XPIDatabase.getAddonsInLocation(installLocation.name); // Iterate through the add-ons installed the last time the application // ran @@ -3084,12 +3078,12 @@ var XPIProvider = { // have any add-ons installed in them, or the locations no longer exist. // The metadata for the add-ons that were in them must be removed from the // database. - knownLocations.forEach(function(aLocation) { - let addons = XPIDatabase.getAddonsInLocation(aLocation); + for (let location of knownLocations) { + let addons = XPIDatabase.getAddonsInLocation(location); addons.forEach(function(aOldAddon) { changed = removeMetadata(aOldAddon) || changed; }, this); - }, this); + } // Tell Telemetry what we found AddonManagerPrivate.recordSimpleMeasure("modifiedUnpacked", modifiedUnpacked); @@ -5379,6 +5373,8 @@ AddonInstall.prototype = { reason, extraParams); } else { + // XXX this makes it dangerous to do many things in onInstallEnded + // listeners because important cleanup hasn't been done yet XPIProvider.unloadBootstrapScope(this.addon.id); } } @@ -5756,8 +5752,8 @@ UpdateChecker.prototype = { /** * The AddonInternal is an internal only representation of add-ons. It may - * have come from the database (see DBAddonInternal below) or an install - * manifest. + * have come from the database (see DBAddonInternal in XPIProviderUtils.jsm) + * or an install manifest. */ function AddonInternal() { } diff --git a/toolkit/mozapps/extensions/XPIProviderUtils.js b/toolkit/mozapps/extensions/XPIProviderUtils.js index 465b663058c5..87c0d910f014 100644 --- a/toolkit/mozapps/extensions/XPIProviderUtils.js +++ b/toolkit/mozapps/extensions/XPIProviderUtils.js @@ -16,7 +16,6 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository", XPCOMUtils.defineLazyModuleGetter(this, "FileUtils", "resource://gre/modules/FileUtils.jsm"); - ["LOG", "WARN", "ERROR"].forEach(function(aName) { Object.defineProperty(this, aName, { get: function logFuncGetter () { @@ -93,14 +92,8 @@ const PREFIX_ITEM_URI = "urn:mozilla:item:"; const RDFURI_ITEM_ROOT = "urn:mozilla:item:root" const PREFIX_NS_EM = "http://www.mozilla.org/2004/em-rdf#"; -Object.defineProperty(this, "gRDF", { - get: function gRDFGetter() { - delete this.gRDF; - return this.gRDF = Cc["@mozilla.org/rdf/rdf-service;1"]. - getService(Ci.nsIRDFService); - }, - configurable: true -}); +XPCOMUtils.defineLazyServiceGetter(this, "gRDF", "@mozilla.org/rdf/rdf-service;1", + Ci.nsIRDFService); function EM_R(aProperty) { return gRDF.GetResource(PREFIX_NS_EM + aProperty); @@ -138,60 +131,6 @@ function getRDFProperty(aDs, aResource, aProperty) { return getRDFValue(aDs.GetTarget(aResource, EM_R(aProperty), true)); } - -/** - * A mozIStorageStatementCallback that will asynchronously build DBAddonInternal - * instances from the results it receives. Once the statement has completed - * executing and all of the metadata for all of the add-ons has been retrieved - * they will be passed as an array to aCallback. - * - * @param aCallback - * A callback function to pass the array of DBAddonInternals to - */ -function AsyncAddonListCallback(aCallback) { - this.callback = aCallback; - this.addons = []; -} - -AsyncAddonListCallback.prototype = { - callback: null, - complete: false, - count: 0, - addons: null, - - handleResult: function AsyncAddonListCallback_handleResult(aResults) { - let row = null; - while ((row = aResults.getNextRow())) { - this.count++; - let self = this; - XPIDatabase.makeAddonFromRowAsync(row, function handleResult_makeAddonFromRowAsync(aAddon) { - function completeAddon(aRepositoryAddon) { - aAddon._repositoryAddon = aRepositoryAddon; - aAddon.compatibilityOverrides = aRepositoryAddon ? - aRepositoryAddon.compatibilityOverrides : - null; - self.addons.push(aAddon); - if (self.complete && self.addons.length == self.count) - self.callback(self.addons); - } - - if ("getCachedAddonByID" in AddonRepository) - AddonRepository.getCachedAddonByID(aAddon.id, completeAddon); - else - completeAddon(null); - }); - } - }, - - handleError: asyncErrorLogger, - - handleCompletion: function AsyncAddonListCallback_handleCompletion(aReason) { - this.complete = true; - if (this.addons.length == this.count) - this.callback(this.addons); - } -}; - /** * Asynchronously fill in the _repositoryAddon field for one addon */ @@ -293,24 +232,6 @@ function asyncErrorLogger(aError) { logSQLError(aError.result, aError.message); } -/** - * A helper function to execute a statement synchronously and log any error - * that occurs. - * - * @param aStatement - * A mozIStorageStatement to execute - */ -function executeStatement(aStatement) { - try { - aStatement.execute(); - } - catch (e) { - logSQLError(XPIDatabase.connection.lastError, - XPIDatabase.connection.lastErrorString); - throw e; - } -} - /** * A helper function to step a statement synchronously and log any error that * occurs. @@ -372,12 +293,6 @@ function copyRowProperties(aRow, aProperties, aTarget) { return aTarget; } -/** - * Create a DBAddonInternal from the fields saved in the JSON database - * or loaded into an AddonInternal from an XPI manifest. - * @return a DBAddonInternal populated with the loaded data - */ - /** * The DBAddonInternal is a special AddonInternal that has been retrieved from * the database. The constructor will initialize the DBAddonInternal with a set @@ -389,6 +304,7 @@ function copyRowProperties(aRow, aProperties, aTarget) { */ function DBAddonInternal(aLoaded) { copyProperties(aLoaded, PROP_JSON_FIELDS, this); + if (aLoaded._installLocation) { this._installLocation = aLoaded._installLocation; this.location = aLoaded._installLocation._name; @@ -396,7 +312,9 @@ function DBAddonInternal(aLoaded) { else if (aLoaded.location) { this._installLocation = XPIProvider.installLocationsByName[this.location]; } + this._key = this.location + ":" + this.id; + try { this._sourceBundle = this._installLocation.getLocationForID(this.id); } @@ -406,20 +324,20 @@ function DBAddonInternal(aLoaded) { // this change is being detected. } - Object.defineProperty(this, "pendingUpgrade", { - get: function DBA_pendingUpgradeGetter() { - delete this.pendingUpgrade; + // XXX Can we redesign pendingUpgrade? + XPCOMUtils.defineLazyGetter(this, "pendingUpgrade", + function DBA_pendingUpgradeGetter() { for (let install of XPIProvider.installs) { if (install.state == AddonManager.STATE_INSTALLED && !(install.addon.inDatabase) && install.addon.id == this.id && install.installLocation == this._installLocation) { + delete this.pendingUpgrade; return this.pendingUpgrade = install.addon; } }; - }, - configurable: true - }); + return null; + }); } DBAddonInternal.prototype = { @@ -437,8 +355,13 @@ DBAddonInternal.prototype = { XPIProvider.updateAddonDisabledState(this); XPIDatabase.commitTransaction(); }, + get inDatabase() { return true; + }, + + toJSON: function() { + return copyProperties(this, PROP_JSON_FIELDS); } } @@ -447,94 +370,21 @@ DBAddonInternal.prototype.__proto__ = AddonInternal.prototype; this.XPIDatabase = { // true if the database connection has been opened initialized: false, - // A cache of statements that are used and need to be finalized on shutdown - statementCache: {}, - // A cache of weak referenced DBAddonInternals so we can reuse objects where - // possible - addonCache: [], // The nested transaction count transactionCount: 0, // The database file - dbfile: FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true), jsonFile: FileUtils.getFile(KEY_PROFILEDIR, [FILE_JSON_DB], true), // Migration data loaded from an old version of the database. migrateData: null, // Active add-on directories loaded from extensions.ini and prefs at startup. activeBundles: null, + // Special handling for when the database is locked at first load + lockedDatabase: false, - // The statements used by the database - statements: { - _getDefaultLocale: "SELECT id, name, description, creator, homepageURL " + - "FROM locale WHERE id=:id", - _getLocales: "SELECT addon_locale.locale, locale.id, locale.name, " + - "locale.description, locale.creator, locale.homepageURL " + - "FROM addon_locale JOIN locale ON " + - "addon_locale.locale_id=locale.id WHERE " + - "addon_internal_id=:internal_id", - _getTargetApplications: "SELECT addon_internal_id, id, minVersion, " + - "maxVersion FROM targetApplication WHERE " + - "addon_internal_id=:internal_id", - _getTargetPlatforms: "SELECT os, abi FROM targetPlatform WHERE " + - "addon_internal_id=:internal_id", - _readLocaleStrings: "SELECT locale_id, type, value FROM locale_strings " + - "WHERE locale_id=:id", - - clearVisibleAddons: "UPDATE addon SET visible=0 WHERE id=:id", - updateAddonActive: "UPDATE addon SET active=:active WHERE " + - "internal_id=:internal_id", - - getActiveAddons: "SELECT " + FIELDS_ADDON + " FROM addon WHERE active=1 AND " + - "type<>'theme' AND bootstrap=0", - getActiveTheme: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " + - "internalName=:internalName AND type='theme'", - getThemes: "SELECT " + FIELDS_ADDON + " FROM addon WHERE type='theme'", - - getAddonInLocation: "SELECT " + FIELDS_ADDON + " FROM addon WHERE id=:id " + - "AND location=:location", - getAddons: "SELECT " + FIELDS_ADDON + " FROM addon", - getAddonsByType: "SELECT " + FIELDS_ADDON + " FROM addon WHERE type=:type", - getAddonsInLocation: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " + - "location=:location", - getInstallLocations: "SELECT DISTINCT location FROM addon", - getVisibleAddonForID: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " + - "visible=1 AND id=:id", - getVisibleAddonForInternalName: "SELECT " + FIELDS_ADDON + " FROM addon " + - "WHERE visible=1 AND internalName=:internalName", - getVisibleAddons: "SELECT " + FIELDS_ADDON + " FROM addon WHERE visible=1", - getVisibleAddonsWithPendingOperations: "SELECT " + FIELDS_ADDON + " FROM " + - "addon WHERE visible=1 " + - "AND (pendingUninstall=1 OR " + - "MAX(userDisabled,appDisabled)=active)", - getAddonBySyncGUID: "SELECT " + FIELDS_ADDON + " FROM addon " + - "WHERE syncGUID=:syncGUID", - makeAddonVisible: "UPDATE addon SET visible=1 WHERE internal_id=:internal_id", - removeAddonMetadata: "DELETE FROM addon WHERE internal_id=:internal_id", - // Equates to active = visible && !userDisabled && !softDisabled && - // !appDisabled && !pendingUninstall - setActiveAddons: "UPDATE addon SET active=MIN(visible, 1 - userDisabled, " + - "1 - softDisabled, 1 - appDisabled, 1 - pendingUninstall)", - setAddonProperties: "UPDATE addon SET userDisabled=:userDisabled, " + - "appDisabled=:appDisabled, " + - "softDisabled=:softDisabled, " + - "pendingUninstall=:pendingUninstall, " + - "applyBackgroundUpdates=:applyBackgroundUpdates WHERE " + - "internal_id=:internal_id", - setAddonDescriptor: "UPDATE addon SET descriptor=:descriptor WHERE " + - "internal_id=:internal_id", - setAddonSyncGUID: "UPDATE addon SET syncGUID=:syncGUID WHERE " + - "internal_id=:internal_id", - updateTargetApplications: "UPDATE targetApplication SET " + - "minVersion=:minVersion, maxVersion=:maxVersion " + - "WHERE addon_internal_id=:internal_id AND id=:id", - - createSavepoint: "SAVEPOINT 'default'", - releaseSavepoint: "RELEASE SAVEPOINT 'default'", - rollbackSavepoint: "ROLLBACK TO SAVEPOINT 'default'" - }, - + // XXX may be able to refactor this away get dbfileExists() { delete this.dbfileExists; - return this.dbfileExists = this.dbfile.exists(); + return this.dbfileExists = this.jsonFile.exists(); }, set dbfileExists(aValue) { delete this.dbfileExists; @@ -545,12 +395,19 @@ this.XPIDatabase = { * Converts the current internal state of the XPI addon database to JSON * and writes it to the user's profile. Synchronous for now, eventually must * be async, reliable, etc. + * XXX should we remove the JSON file if it would be empty? Not sure if that + * would ever happen, given the default theme */ writeJSON: function XPIDB_writeJSON() { // XXX should have a guard here for if the addonDB hasn't been auto-loaded yet + + // Don't mess with an existing database on disk, if it was locked at start up + if (this.lockedDatabase) + return; + let addons = []; - for (let aKey in this.addonDB) { - addons.push(copyProperties(this.addonDB[aKey], PROP_JSON_FIELDS)); + for (let [key, addon] of this.addonDB) { + addons.push(addon); } let toSave = { schemaVersion: DB_SCHEMA, @@ -563,11 +420,17 @@ this.XPIDatabase = { try { converter.init(stream, "UTF-8", 0, 0x0000); // XXX pretty print the JSON while debugging - converter.writeString(JSON.stringify(toSave, null, 2)); + let out = JSON.stringify(toSave, null, 2); + // dump("Writing JSON:\n" + out + "\n"); + converter.writeString(out); converter.flush(); // nsConverterOutputStream doesn't finish() safe output streams on close() FileUtils.closeSafeFileOutputStream(stream); converter.close(); + this.dbfileExists = true; + // XXX probably only want to do this if the version is different + Services.prefs.setIntPref(PREF_DB_SCHEMA, DB_SCHEMA); + Services.prefs.savePrefFile(null); // XXX is this bad sync I/O? } catch(e) { ERROR("Failed to save database to JSON", e); @@ -575,66 +438,6 @@ this.XPIDatabase = { } }, - /** - * Open and parse the JSON XPI extensions database. - * @return true: the DB was successfully loaded - * false: The DB either needs upgrade or did not exist at all. - * XXX upgrade and errors handled in a following patch - */ - openJSONDatabase: function XPIDB_openJSONDatabase() { - dump("XPIDB_openJSONDatabase\n"); - try { - let data = ""; - let fstream = Components.classes["@mozilla.org/network/file-input-stream;1"]. - createInstance(Components.interfaces.nsIFileInputStream); - let cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"]. - createInstance(Components.interfaces.nsIConverterInputStream); - fstream.init(this.jsonFile, -1, 0, 0); - cstream.init(fstream, "UTF-8", 0, 0); - let (str = {}) { - let read = 0; - do { - read = cstream.readString(0xffffffff, str); // read as much as we can and put it in str.value - data += str.value; - } while (read != 0); - } - cstream.close(); - let inputAddons = JSON.parse(data); - // Now do some sanity checks on our JSON db - if (!("schemaVersion" in inputAddons) || !("addons" in inputAddons)) { - // XXX Content of JSON file is bad, need to rebuild from scratch - ERROR("bad JSON file contents"); - delete this.addonDB; - this.addonDB = {}; - return false; - } - if (inputAddons.schemaVersion != DB_SCHEMA) { - // XXX UPGRADE FROM PREVIOUS VERSION OF JSON DB - ERROR("JSON schema upgrade needed"); - return false; - } - // If we got here, we probably have good data - // Make AddonInternal instances from the loaded data and save them - delete this.addonDB; - let addonDB = {} - inputAddons.addons.forEach(function(loadedAddon) { - let newAddon = new DBAddonInternal(loadedAddon); - addonDB[newAddon._key] = newAddon; - }); - this.addonDB = addonDB; - // dump("Finished reading DB: " + this.addonDB.toSource() + "\n"); - return true; - } - catch(e) { - // XXX handle missing JSON database - ERROR("Failed to load XPI JSON data from profile", e); - // XXX for now, start from scratch - delete this.addonDB; - this.addonDB = {}; - return false; - } - }, - /** * Begins a new transaction in the database. Transactions may be nested. Data * written by an inner transaction may be rolled back on its own. Rolling back @@ -681,170 +484,197 @@ this.XPIDatabase = { }, /** - * Attempts to open the database file. If it fails it will try to delete the - * existing file and create an empty database. If that fails then it will - * open an in-memory database that can be used during this session. + * Pull upgrade information from an existing SQLITE database * - * @param aDBFile - * The nsIFile to open - * @return the mozIStorageConnection for the database + * @return false if there is no SQLITE database + * true and sets this.migrateData to null if the SQLITE DB exists + * but does not contain useful information + * true and sets this.migrateData to + * {location: {id1:{addon1}, id2:{addon2}}, location2:{...}, ...} + * if there is useful information */ - openDatabaseFile: function XPIDB_openDatabaseFile(aDBFile) { - LOG("Opening database"); + loadSqliteData: function XPIDB_loadSqliteData() { let connection = null; - + let dbfile = FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true); + if (!dbfile.exists()) { + return false; + } // Attempt to open the database try { - connection = Services.storage.openUnsharedDatabase(aDBFile); - this.dbfileExists = true; + connection = Services.storage.openUnsharedDatabase(dbfile); } catch (e) { - ERROR("Failed to open database (1st attempt)", e); - // If the database was locked for some reason then assume it still - // has some good data and we should try to load it the next time around. - if (e.result != Cr.NS_ERROR_STORAGE_BUSY) { - try { - aDBFile.remove(true); - } - catch (e) { - ERROR("Failed to remove database that could not be opened", e); - } - try { - connection = Services.storage.openUnsharedDatabase(aDBFile); - } - catch (e) { - ERROR("Failed to open database (2nd attempt)", e); - - // If we have got here there seems to be no way to open the real - // database, instead open a temporary memory database so things will - // work for this session. - return Services.storage.openSpecialDatabase("memory"); - } - } - else { - return Services.storage.openSpecialDatabase("memory"); - } + // exists but SQLITE can't open it + WARN("Failed to open sqlite database " + dbfile.path + " for upgrade", e); + this.migrateData = null; + return true; } - - connection.executeSimpleSQL("PRAGMA synchronous = FULL"); - connection.executeSimpleSQL("PRAGMA locking_mode = EXCLUSIVE"); - - return connection; + LOG("Migrating data from sqlite"); + this.migrateData = this.getMigrateDataFromDatabase(connection); + connection.close(); + return true; }, /** - * Opens a new connection to the database file. + * Opens and reads the database file, upgrading from old + * databases or making a new DB if needed. * + * The possibilities, in order of priority, are: + * 1) Perfectly good, up to date database + * 2) Out of date JSON database needs to be upgraded => upgrade + * 3) JSON database exists but is mangled somehow => build new JSON + * 4) no JSON DB, but a useable SQLITE db we can upgrade from => upgrade + * 5) useless SQLITE DB => build new JSON + * 6) useable RDF DB => upgrade + * 7) useless RDF DB => build new JSON + * 8) Nothing at all => build new JSON * @param aRebuildOnError * A boolean indicating whether add-on information should be loaded * from the install locations if the database needs to be rebuilt. + * (if false, caller is XPIProvider.checkForChanges() which will rebuild) */ openConnection: function XPIDB_openConnection(aRebuildOnError, aForceOpen) { - this.openJSONDatabase(); + // XXX TELEMETRY report opens with aRebuildOnError true (which implies delayed open) + // vs. aRebuildOnError false (DB loaded during startup) + delete this.addonDB; + this.migrateData = null; + let fstream = null; + let data = ""; + try { + LOG("Opening XPI database " + this.jsonFile.path); + fstream = Components.classes["@mozilla.org/network/file-input-stream;1"]. + createInstance(Components.interfaces.nsIFileInputStream); + fstream.init(this.jsonFile, -1, 0, 0); + let cstream = null; + try { + cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"]. + createInstance(Components.interfaces.nsIConverterInputStream); + cstream.init(fstream, "UTF-8", 0, 0); + let (str = {}) { + let read = 0; + do { + read = cstream.readString(0xffffffff, str); // read as much as we can and put it in str.value + data += str.value; + } while (read != 0); + } + // dump("Loaded JSON:\n" + data + "\n"); + let inputAddons = JSON.parse(data); + // Now do some sanity checks on our JSON db + if (!("schemaVersion" in inputAddons) || !("addons" in inputAddons)) { + // Content of JSON file is bad, need to rebuild from scratch + ERROR("bad JSON file contents"); + this.rebuildDatabase(aRebuildOnError); + } + if (inputAddons.schemaVersion != DB_SCHEMA) { + // Handle mismatched JSON schema version. For now, we assume backward/forward + // compatibility as long as we preserve unknown fields during save & restore + // XXX preserve schema version and unknown fields during save/restore + LOG("JSON schema mismatch: expected " + DB_SCHEMA + + ", actual " + inputAddons.schemaVersion); + } + // If we got here, we probably have good data + // Make AddonInternal instances from the loaded data and save them + let addonDB = new Map(); + inputAddons.addons.forEach(function(loadedAddon) { + let newAddon = new DBAddonInternal(loadedAddon); + addonDB.set(newAddon._key, newAddon); + }); + this.addonDB = addonDB; + LOG("Successfully read XPI database"); + } + catch(e) { + // If we catch and log a SyntaxError from the JSON + // parser, the xpcshell test harness fails the test for us: bug 870828 + if (e.name == "SyntaxError") { + ERROR("Syntax error parsing saved XPI JSON data"); + } + else { + ERROR("Failed to load XPI JSON data from profile", e); + } + this.rebuildDatabase(aRebuildOnError); + } + finally { + if (cstream) + cstream.close(); + } + } + catch (e) { + if (e.result == Cr.NS_ERROR_FILE_NOT_FOUND) { + // XXX re-implement logic to decide whether to upgrade database + // by checking the DB_SCHEMA_VERSION preference. + // Fall back to attempting database upgrades + WARN("Extensions database not found; attempting to upgrade"); + // See if there is SQLITE to migrate from + if (!this.loadSqliteData()) { + // Nope, try RDF + this.migrateData = this.getMigrateDataFromRDF(); + } + + this.rebuildDatabase(aRebuildOnError); + } + else { + WARN("Extensions database " + this.jsonFile.path + + " exists but is not readable; rebuilding in memory", e); + // XXX open question - if we can overwrite at save time, should we, or should we + // leave the locked database in case we can recover from it next time we start up? + this.lockedDatabase = true; + // XXX TELEMETRY report when this happens? + this.rebuildDatabase(aRebuildOnError); + } + } + finally { + if (fstream) + fstream.close(); + } + this.initialized = true; return; - // XXX IRVING deal with the migration logic below and in openDatabaseFile... - - delete this.connection; + // XXX what about aForceOpen? Appears to handle the case of "don't open DB file if there aren't any extensions"? if (!aForceOpen && !this.dbfileExists) { this.connection = null; return; } + }, - this.migrateData = null; + /** + * Rebuild the database from addon install directories. If this.migrateData + * is available, uses migrated information for settings on the addons found + * during rebuild + * @param aRebuildOnError + * A boolean indicating whether add-on information should be loaded + * from the install locations if the database needs to be rebuilt. + * (if false, caller is XPIProvider.checkForChanges() which will rebuild) + */ + rebuildDatabase: function XIPDB_rebuildDatabase(aRebuildOnError) { + // If there is no migration data then load the list of add-on directories + // that were active during the last run + this.addonDB = new Map(); + if (!this.migrateData) + this.activeBundles = this.getActiveBundles(); - this.connection = this.openDatabaseFile(this.dbfile); - - // If the database was corrupt or missing then the new blank database will - // have a schema version of 0. - let schemaVersion = this.connection.schemaVersion; - if (schemaVersion != DB_SCHEMA) { - // A non-zero schema version means that a schema has been successfully - // created in the database in the past so we might be able to get useful - // information from it - if (schemaVersion != 0) { - LOG("Migrating data from schema " + schemaVersion); - this.migrateData = this.getMigrateDataFromDatabase(); - - // Delete the existing database - this.connection.close(); - try { - if (this.dbfileExists) - this.dbfile.remove(true); - - // Reopen an empty database - this.connection = this.openDatabaseFile(this.dbfile); - } - catch (e) { - ERROR("Failed to remove old database", e); - // If the file couldn't be deleted then fall back to an in-memory - // database - this.connection = Services.storage.openSpecialDatabase("memory"); - } - } - else { - let dbSchema = 0; - try { - dbSchema = Services.prefs.getIntPref(PREF_DB_SCHEMA); - } catch (e) {} - - if (dbSchema == 0) { - // Only migrate data from the RDF if we haven't done it before - this.migrateData = this.getMigrateDataFromRDF(); - } - } - - // At this point the database should be completely empty + if (aRebuildOnError) { + WARN("Rebuilding add-ons database from installed extensions."); + this.beginTransaction(); try { - this.createSchema(); + let state = XPIProvider.getInstallLocationStates(); + XPIProvider.processFileChanges(state, {}, false); + // Make sure to update the active add-ons and add-ons list on shutdown + Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true); + this.commitTransaction(); } catch (e) { - // If creating the schema fails, then the database is unusable, - // fall back to an in-memory database. - this.connection = Services.storage.openSpecialDatabase("memory"); - } - - // If there is no migration data then load the list of add-on directories - // that were active during the last run - if (!this.migrateData) - this.activeBundles = this.getActiveBundles(); - - if (aRebuildOnError) { - WARN("Rebuilding add-ons database from installed extensions."); - this.beginTransaction(); - try { - let state = XPIProvider.getInstallLocationStates(); - XPIProvider.processFileChanges(state, {}, false); - // Make sure to update the active add-ons and add-ons list on shutdown - Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true); - this.commitTransaction(); - } - catch (e) { - ERROR("Error processing file changes", e); - this.rollbackTransaction(); - } + ERROR("Error processing file changes", e); + this.rollbackTransaction(); } } - - // If the database connection has a file open then it has the right schema - // by now so make sure the preferences reflect that. - if (this.connection.databaseFile) { - Services.prefs.setIntPref(PREF_DB_SCHEMA, DB_SCHEMA); - Services.prefs.savePrefFile(null); - } - - // Begin any pending transactions - for (let i = 0; i < this.transactionCount; i++) - this.connection.executeSimpleSQL("SAVEPOINT 'default'"); }, /** * Lazy getter for the addons database */ get addonDB() { - delete this.addonDB; - this.openJSONDatabase(); + this.openConnection(true); return this.addonDB; }, @@ -964,13 +794,13 @@ this.XPIDatabase = { * @return an object holding information about what add-ons were previously * userDisabled and any updated compatibility information */ - getMigrateDataFromDatabase: function XPIDB_getMigrateDataFromDatabase() { + getMigrateDataFromDatabase: function XPIDB_getMigrateDataFromDatabase(aConnection) { let migrateData = {}; // Attempt to migrate data from a different (even future!) version of the // database try { - var stmt = this.connection.createStatement("PRAGMA table_info(addon)"); + var stmt = aConnection.createStatement("PRAGMA table_info(addon)"); const REQUIRED = ["internal_id", "id", "location", "userDisabled", "installDate", "version"]; @@ -996,7 +826,7 @@ this.XPIDatabase = { } stmt.finalize(); - stmt = this.connection.createStatement("SELECT " + props.join(",") + " FROM addon"); + stmt = aConnection.createStatement("SELECT " + props.join(",") + " FROM addon"); for (let row in resultRows(stmt)) { if (!(row.location in migrateData)) migrateData[row.location] = {}; @@ -1015,7 +845,7 @@ this.XPIDatabase = { }) } - var taStmt = this.connection.createStatement("SELECT id, minVersion, " + + var taStmt = aConnection.createStatement("SELECT id, minVersion, " + "maxVersion FROM " + "targetApplication WHERE " + "addon_internal_id=:internal_id"); @@ -1063,10 +893,8 @@ this.XPIDatabase = { // If we are running with an in-memory database then force a new // extensions.ini to be written to disk on the next startup - // XXX IRVING special case for if we fail to save extensions.json? - // XXX maybe doesn't need to be at shutdown? - // if (!this.connection.databaseFile) - // Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true); + if (this.lockedDatabase) + Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true); this.initialized = false; @@ -1075,7 +903,7 @@ this.XPIDatabase = { delete this.addonDB; Object.defineProperty(this, "addonDB", { get: function addonsGetter() { - this.openJSONDatabase(); + this.openConnection(true); return this.addonDB; }, configurable: true @@ -1098,17 +926,17 @@ this.XPIDatabase = { * installed add-ons, occasionally a superset when an install location no * longer exists. * - * @return an array of names of install locations + * @return a Set of names of install locations */ getInstallLocations: function XPIDB_getInstallLocations() { + let locations = new Set(); if (!this.addonDB) - return []; + return locations; - let locations = {}; - for each (let addon in this.addonDB) { - locations[addon.location] = 1; + for (let [, addon] of this.addonDB) { + locations.add(addon.location); } - return Object.keys(locations); + return locations; }, /** @@ -1123,8 +951,7 @@ this.XPIDatabase = { return []; let addonList = []; - for (let key in this.addonDB) { - let addon = this.addonDB[key]; + for (let [key, addon] of this.addonDB) { if (aFilter(addon)) { addonList.push(addon); } @@ -1144,8 +971,7 @@ this.XPIDatabase = { if (!this.addonDB) return null; - for (let key in this.addonDB) { - let addon = this.addonDB[key]; + for (let [key, addon] of this.addonDB) { if (aFilter(addon)) { return addon; } @@ -1178,7 +1004,7 @@ this.XPIDatabase = { * A callback to pass the DBAddonInternal to */ getAddonInLocation: function XPIDB_getAddonInLocation(aId, aLocation, aCallback) { - getRepositoryAddon(this.addonDB[aLocation + ":" + aId], aCallback); + getRepositoryAddon(this.addonDB.get(aLocation + ":" + aId), aCallback); }, /** @@ -1305,7 +1131,7 @@ this.XPIDatabase = { let newAddon = new DBAddonInternal(aAddon); newAddon.descriptor = aDescriptor; - this.addonDB[newAddon._key] = newAddon; + this.addonDB.set(newAddon._key, newAddon); if (newAddon.visible) { this.makeAddonVisible(newAddon); } @@ -1358,7 +1184,7 @@ this.XPIDatabase = { */ removeAddonMetadata: function XPIDB_removeAddonMetadata(aAddon) { this.beginTransaction(); - delete this.addonDB[aAddon._key]; + this.addonDB.delete(aAddon._key); this.commitTransaction(); }, @@ -1374,8 +1200,7 @@ this.XPIDatabase = { makeAddonVisible: function XPIDB_makeAddonVisible(aAddon) { this.beginTransaction(); LOG("Make addon " + aAddon._key + " visible"); - for (let key in this.addonDB) { - let otherAddon = this.addonDB[key]; + for (let [key, otherAddon] of this.addonDB) { if ((otherAddon.id == aAddon.id) && (otherAddon._key != aAddon._key)) { LOG("Hide addon " + otherAddon._key); otherAddon.visible = false; @@ -1461,14 +1286,20 @@ this.XPIDatabase = { // XXX IRVING this may get called during XPI-utils shutdown // XXX need to make sure PREF_PENDING_OPERATIONS handling is clean LOG("Updating add-on states"); - this.beginTransaction(); - for (let key in this.addonDB) { - let addon = this.addonDB[key]; - addon.active = (addon.visible && !addon.userDisabled && + let changed = false; + for (let [key, addon] of this.addonDB) { + let newActive = (addon.visible && !addon.userDisabled && !addon.softDisabled && !addon.appDisabled && !addon.pendingUninstall); + if (newActive != addon.active) { + addon.active = newActive; + changed = true; + } + } + if (changed) { + this.beginTransaction(); + this.commitTransaction(); } - this.commitTransaction(); }, /** diff --git a/toolkit/mozapps/extensions/test/xpcshell/head_addons.js b/toolkit/mozapps/extensions/test/xpcshell/head_addons.js index e0c34ee1aa35..781f38e21c30 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/head_addons.js +++ b/toolkit/mozapps/extensions/test/xpcshell/head_addons.js @@ -1395,16 +1395,16 @@ function do_exception_wrap(func) { } const EXTENSIONS_DB = "extensions.json"; +let gExtensionsJSON = gProfD.clone(); +gExtensionsJSON.append(EXTENSIONS_DB); /** * Change the schema version of the JSON extensions database */ function changeXPIDBVersion(aNewVersion) { - let dbfile = gProfD.clone(); - dbfile.append(EXTENSIONS_DB); - let jData = loadJSON(dbfile); + let jData = loadJSON(gExtensionsJSON); jData.schemaVersion = aNewVersion; - saveJSON(jData, dbfile); + saveJSON(jData, gExtensionsJSON); } /** @@ -1426,7 +1426,7 @@ function loadJSON(aFile) { } while (read != 0); } cstream.close(); - do_print("Loaded JSON file " + aFile.spec); + do_print("Loaded JSON file " + aFile.path); return(JSON.parse(data)); } diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bad_json.js b/toolkit/mozapps/extensions/test/xpcshell/test_bad_json.js new file mode 100644 index 000000000000..d3ccf68f3a0d --- /dev/null +++ b/toolkit/mozapps/extensions/test/xpcshell/test_bad_json.js @@ -0,0 +1,54 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +// Tests that we rebuild the database correctly if it contains +// JSON data that parses correctly but doesn't contain required fields + +var addon1 = { + id: "addon1@tests.mozilla.org", + version: "2.0", + name: "Test 1", + targetApplications: [{ + id: "xpcshell@tests.mozilla.org", + minVersion: "1", + maxVersion: "1" + }] +}; + +const profileDir = gProfD.clone(); +profileDir.append("extensions"); + +function run_test() { + do_test_pending("Bad JSON"); + + createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2"); + + // This addon will be auto-installed at startup + writeInstallRDFForExtension(addon1, profileDir); + + startupManager(); + + shutdownManager(); + + // First startup/shutdown finished + // Replace the JSON store with something bogus + saveJSON({not: "what we expect to find"}, gExtensionsJSON); + + startupManager(false); + // Retrieve an addon to force the database to rebuild + AddonManager.getAddonsByIDs([addon1.id], callback_soon(after_db_rebuild)); +} + +function after_db_rebuild([a1]) { + do_check_eq(a1.id, addon1.id); + + shutdownManager(); + + // Make sure our JSON database has schemaVersion and our installed extension + let data = loadJSON(gExtensionsJSON); + do_check_true("schemaVersion" in data); + do_check_eq(data.addons[0].id, addon1.id); + + do_test_finished("Bad JSON"); +} diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js index 6d201640f989..9a72b50c6733 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js @@ -146,10 +146,9 @@ function run_test() { startupManager(); - let file = gProfD.clone(); - file.append(EXTENSIONS_DB); - do_check_false(file.exists()); + do_check_false(gExtensionsJSON.exists()); + let file = gProfD.clone(); file.leafName = "extensions.ini"; do_check_false(file.exists()); @@ -205,10 +204,9 @@ function run_test_1() { } function check_test_1(installSyncGUID) { - let file = gProfD.clone(); - file.append(EXTENSIONS_DB); - do_check_true(file.exists()); + do_check_true(gExtensionsJSON.exists()); + let file = gProfD.clone(); file.leafName = "extensions.ini"; do_check_false(file.exists()); diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bug559800.js b/toolkit/mozapps/extensions/test/xpcshell/test_bug559800.js index 789819cc6b87..af1c845f19ff 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_bug559800.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_bug559800.js @@ -47,9 +47,7 @@ function run_test_1() { shutdownManager(); - let db = gProfD.clone(); - db.append(EXTENSIONS_DB); - db.remove(true); + gExtensionsJSON.remove(true); do_execute_soon(check_test_1); }); @@ -62,10 +60,8 @@ function check_test_1() { do_check_neq(a1, null); do_check_eq(a1.version, "1.0"); - let db = gProfD.clone(); - db.append(EXTENSIONS_DB); - do_check_true(db.exists()); - do_check_true(db.fileSize > 0); + do_check_true(gExtensionsJSON.exists()); + do_check_true(gExtensionsJSON.fileSize > 0); end_test(); }); diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_corrupt.js b/toolkit/mozapps/extensions/test/xpcshell/test_corrupt.js index 50dd782da360..4c8b3750d695 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_corrupt.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_corrupt.js @@ -251,10 +251,8 @@ function run_test_1() { // serves this purpose). On startup the add-ons manager won't rebuild // because there is a file there still. shutdownManager(); - var dbfile = gProfD.clone(); - dbfile.append("extensions.json"); - dbfile.remove(true); - dbfile.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755); + gExtensionsJSON.remove(true); + gExtensionsJSON.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); startupManager(false); // Accessing the add-ons should open and recover the database diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_corrupt_strictcompat.js b/toolkit/mozapps/extensions/test/xpcshell/test_corrupt_strictcompat.js index e465b47bbacb..3ba6d213beb0 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_corrupt_strictcompat.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_corrupt_strictcompat.js @@ -252,10 +252,8 @@ function run_test_1() { // serves this purpose). On startup the add-ons manager won't rebuild // because there is a file there still. shutdownManager(); - var dbfile = gProfD.clone(); - dbfile.append("extensions.json"); - dbfile.remove(true); - dbfile.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755); + gExtensionsJSON.remove(true); + gExtensionsJSON.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); startupManager(false); // Accessing the add-ons should open and recover the database diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_locked.js b/toolkit/mozapps/extensions/test/xpcshell/test_locked.js index 883ea08c6d1f..91d09c07c7e2 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_locked.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_locked.js @@ -257,10 +257,8 @@ function run_test_1() { // After shutting down the database won't be open so we can // mess with permissions shutdownManager(); - var dbfile = gProfD.clone(); - dbfile.append(EXTENSIONS_DB); - var savedPermissions = dbfile.permissions; - dbfile.permissions = 0; + var savedPermissions = gExtensionsJSON.permissions; + gExtensionsJSON.permissions = 0; startupManager(false); @@ -428,11 +426,12 @@ function run_test_1() { do_check_eq(t2.pendingOperations, AddonManager.PENDING_NONE); do_check_true(isThemeInAddonsList(profileDir, t2.id)); - dbfile.permissions = savedPermissions; - // After allowing access to the original DB things should go back to as // they were previously - restartManager(); + shutdownManager(); + gExtensionsJSON.permissions = savedPermissions; + startupManager(); + // Shouldn't have seen any startup changes check_startup_changes(AddonManager.STARTUP_CHANGE_INSTALLED, []); diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_locked2.js b/toolkit/mozapps/extensions/test/xpcshell/test_locked2.js index 6e21df540d1f..79c3439fd4e8 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_locked2.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_locked2.js @@ -144,10 +144,8 @@ function run_test() { // After shutting down the database won't be open so we can lock it shutdownManager(); - var dbfile = gProfD.clone(); - dbfile.append(EXTENSIONS_DB); - var savedPermissions = dbfile.permissions; - dbfile.permissions = 0; + var savedPermissions = gExtensionsJSON.permissions; + gExtensionsJSON.permissions = 0; startupManager(false); @@ -199,11 +197,11 @@ function run_test() { do_check_eq(a6.pendingOperations, AddonManager.PENDING_NONE); do_check_true(isExtensionInAddonsList(profileDir, a6.id)); - dbfile.permissions = savedPermissions; - // After allowing access to the original DB things should still be // applied correctly - restartManager(); + shutdownManager(); + gExtensionsJSON.permissions = savedPermissions; + startupManager(); // These things happened when we had no access to the database so // they are seen as external changes when we get the database back :( diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_locked_strictcompat.js b/toolkit/mozapps/extensions/test/xpcshell/test_locked_strictcompat.js index 2c31171318be..f38549cdaae2 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/test_locked_strictcompat.js +++ b/toolkit/mozapps/extensions/test/xpcshell/test_locked_strictcompat.js @@ -256,10 +256,8 @@ function run_test_1() { // After shutting down the database won't be open so we can lock it shutdownManager(); - var dbfile = gProfD.clone(); - dbfile.append(EXTENSIONS_DB); - var savedPermissions = dbfile.permissions; - dbfile.permissions = 0; + var savedPermissions = gExtensionsJSON.permissions; + gExtensionsJSON.permissions = 0; startupManager(false); @@ -425,11 +423,11 @@ function run_test_1() { do_check_eq(t2.pendingOperations, AddonManager.PENDING_NONE); do_check_true(isThemeInAddonsList(profileDir, t2.id)); - dbfile.permissions = savedPermissions; - // After allowing access to the original DB things should go back to as // they were previously - restartManager(); + shutdownManager(); + gExtensionsJSON.permissions = savedPermissions; + startupManager(false); // Shouldn't have seen any startup changes check_startup_changes(AddonManager.STARTUP_CHANGE_INSTALLED, []); diff --git a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini index 49fef590e1c0..6113454b47a2 100644 --- a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini +++ b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini @@ -16,12 +16,9 @@ skip-if = os == "android" [test_DeferredSave.js] [test_LightweightThemeManager.js] [test_backgroundupdate.js] +[test_bad_json.js] [test_badschema.js] -# Needs rewrite for JSON XPIDB -fail-if = true [test_blocklistchange.js] -# Needs rewrite for JSON XPIDB -fail-if = true # Bug 676992: test consistently hangs on Android skip-if = os == "android" [test_blocklist_regexp.js] @@ -142,8 +139,6 @@ fail-if = os == "android" [test_bug620837.js] [test_bug655254.js] [test_bug659772.js] -# needs to be converted from sqlite to JSON -fail-if = true [test_bug675371.js] [test_bug740612.js] [test_bug753900.js] @@ -153,11 +148,7 @@ fail-if = true [test_ChromeManifestParser.js] [test_compatoverrides.js] [test_corrupt.js] -# needs to be converted from sqlite to JSON -fail-if = true [test_corrupt_strictcompat.js] -# needs to be converted from sqlite to JSON -fail-if = true [test_dictionary.js] [test_langpack.js] [test_disable.js] @@ -202,33 +193,17 @@ skip-if = os == "android" run-sequentially = Uses hardcoded ports in xpi files. [test_locale.js] [test_locked.js] -# Needs sqlite->JSON conversion -fail-if = true [test_locked2.js] -# Needs sqlite->JSON conversion -fail-if = true [test_locked_strictcompat.js] -# Needs sqlite->JSON conversion -fail-if = true [test_manifest.js] [test_mapURIToAddonID.js] # Same as test_bootstrap.js skip-if = os == "android" [test_migrate1.js] -# Needs sqlite->JSON conversion -fail-if = true [test_migrate2.js] -# Needs sqlite->JSON conversion -fail-if = true [test_migrate3.js] -# Needs sqlite->JSON conversion -fail-if = true [test_migrate4.js] -# Needs sqlite->JSON conversion -fail-if = true [test_migrate5.js] -# Needs sqlite->JSON conversion -fail-if = true [test_migrateAddonRepository.js] [test_onPropertyChanged_appDisabled.js] [test_permissions.js]