diff --git a/toolkit/mozapps/extensions/XPIProvider.jsm b/toolkit/mozapps/extensions/XPIProvider.jsm
index 8d0330c4efff..eea9c70a0eac 100644
--- a/toolkit/mozapps/extensions/XPIProvider.jsm
+++ b/toolkit/mozapps/extensions/XPIProvider.jsm
@@ -4288,8 +4288,14 @@ AddonInstall.prototype = {
       let stagedJSON = stagedAddon.clone();
       stagedAddon.append(this.addon.id);
       stagedJSON.append(this.addon.id + ".json");
-      if (stagedAddon.exists())
+      if (stagedAddon.exists()) {
         stagedAddon.remove(true);
+      }
+      else {
+        stagedAddon.leafName += ".xpi";
+        if (stagedAddon.exists())
+          stagedAddon.remove(false);
+      }
       if (stagedJSON.exists())
         stagedJSON.remove(true);
       this.state = AddonManager.STATE_CANCELLED;
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_install.js b/toolkit/mozapps/extensions/test/xpcshell/test_install.js
index 048d7d041955..9b0a450b22cc 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_install.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_install.js
@@ -591,7 +591,14 @@ function check_test_10(install) {
   AddonManager.getAllInstalls(function(activeInstalls) {
     do_check_eq(activeInstalls.length, 0);
 
-    run_test_11();
+    restartManager();
+
+    // Check that the install did not complete
+    AddonManager.getAddonByID("addon3@tests.mozilla.org", function(a3) {
+      do_check_eq(a3, null);
+
+      run_test_11();
+    });
   });
 }
 
@@ -934,6 +941,13 @@ function check_test_13(install) {
     do_check_false(hasFlag(olda2.pendingOperations, AddonManager.PENDING_UPGRADE));
     do_check_eq(olda2.pendingUpgrade, null);
 
-    end_test();
+    restartManager();
+
+    // Check that the upgrade did not complete
+    AddonManager.getAddonByID("addon2@tests.mozilla.org", function(a2) {
+      do_check_eq(a2.version, "2.0");
+
+      end_test();
+    });
   });
 }