Bug 394548 - Store download progress in database when changing states. r=sdwilsh, a=mconnor

toolkit/components/downloads/public/nsIDownload.idl

@@ -54,7 +54,7 @@ interface nsIMIMEInfo;
  *       nsIDownloadManager::DOWNLOAD_FAILED
  *       nsIDownloadManager::DOWNLOAD_CANCELED
  */
-[scriptable, uuid(e5900f21-7a8a-4f9f-ac67-5379a65c18b6)]
+[scriptable, uuid(4ee8befe-b05a-4ca2-9b30-6c47a9c4a622)]
 interface nsIDownload : nsITransfer {
     
     /**
@@ -66,18 +66,18 @@ interface nsIDownload : nsITransfer {
      * The percentage of transfer completed.
      * If the file size is unknown it'll be -1 here.
      */
-    readonly attribute PRInt32 percentComplete;
+    readonly attribute long percentComplete;
 
     /**
      * The amount of bytes downloaded so far.
      */
-    readonly attribute PRUint64 amountTransferred;
+    readonly attribute long long amountTransferred;
 
     /**
      * The size of file in bytes.
-     * Unknown size is represented by LL_MAXUINT.
+     * Unknown size is represented by -1.
      */
-    readonly attribute PRUint64 size;
+    readonly attribute long long size;
     
     /**
      * The source of the transfer.

toolkit/components/downloads/src/nsDownloadManager.cpp

@@ -92,7 +92,7 @@ static PRBool gStoppingDownloads = PR_FALSE;
 
 static const PRInt64 gUpdateInterval = 400 * PR_USEC_PER_MSEC;
 
-#define DM_SCHEMA_VERSION      5
+#define DM_SCHEMA_VERSION      6
 #define DM_DB_NAME             NS_LITERAL_STRING("downloads.sqlite")
 #define DM_DB_CORRUPT_FILENAME NS_LITERAL_STRING("downloads.sqlite.corrupt")
 
@@ -376,6 +376,25 @@ nsDownloadManager::InitDB(PRBool *aDoImport)
     }
     // Fallthrough to the next upgrade
 
+  case 5:
+    {
+      rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "ALTER TABLE moz_downloads "
+        "ADD COLUMN currBytes INTEGER NOT NULL DEFAULT 0"));
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "ALTER TABLE moz_downloads "
+        "ADD COLUMN maxBytes INTEGER NOT NULL DEFAULT -1"));
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      // Finally, update the schemaVersion variable and the database schema
+      schemaVersion = 6;
+      rv = mDBConn->SetSchemaVersion(schemaVersion);
+      NS_ENSURE_SUCCESS(rv, rv);
+    }
+    // Fallthrough to the next upgrade
+
   // Extra sanity checking for developers
 #ifndef DEBUG
   case DM_SCHEMA_VERSION:
@@ -406,7 +425,7 @@ nsDownloadManager::InitDB(PRBool *aDoImport)
       nsCOMPtr<mozIStorageStatement> stmt;
       rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
         "SELECT id, name, source, target, tempPath, startTime, endTime, state, "
-               "referrer, entityID "
+               "referrer, entityID, currBytes, maxBytes "
         "FROM moz_downloads"), getter_AddRefs(stmt));
       if (NS_SUCCEEDED(rv))
         break;
@@ -449,7 +468,9 @@ nsDownloadManager::CreateTable()
       "endTime INTEGER, "
       "state INTEGER, "
       "referrer TEXT, "
-      "entityID TEXT"
+      "entityID TEXT, "
+      "currBytes INTEGER NOT NULL DEFAULT 0, "
+      "maxBytes INTEGER NOT NULL DEFAULT -1"
     ")"));
 }
 
@@ -775,8 +796,9 @@ nsDownloadManager::Init()
 
   rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
     "UPDATE moz_downloads "
-    "SET startTime = ?1, endTime = ?2, state = ?3, referrer = ?4, entityID = ?5 "
-    "WHERE id = ?6"), getter_AddRefs(mUpdateDownloadStatement));
+    "SET startTime = ?1, endTime = ?2, state = ?3, referrer = ?4, "
+        "entityID = ?5, currBytes = ?6, maxBytes = ?7 "
+    "WHERE id = ?8"), getter_AddRefs(mUpdateDownloadStatement));
   NS_ENSURE_SUCCESS(rv, rv);
 
   // The following three AddObserver calls must be the last lines in this function,
@@ -821,7 +843,7 @@ nsDownloadManager::GetDownloadFromDB(PRUint32 aID, nsDownload **retVal)
   nsCOMPtr<mozIStorageStatement> stmt;
   nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
     "SELECT id, state, startTime, source, target, tempPath, name, referrer, "
-           "entityID "
+           "entityID, currBytes, maxBytes "
     "FROM moz_downloads "
     "WHERE id = ?1"), getter_AddRefs(stmt));
   NS_ENSURE_SUCCESS(rv, rv);
@@ -872,32 +894,11 @@ nsDownloadManager::GetDownloadFromDB(PRUint32 aID, nsDownload **retVal)
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
-  nsCOMPtr<nsILocalFile> file;
-  rv = dl->GetTargetFile(getter_AddRefs(file));
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  PRBool fileExists;
-  if (NS_SUCCEEDED(file->Exists(&fileExists)) && fileExists) {
-    if (dl->mDownloadState == nsIDownloadManager::DOWNLOAD_FINISHED) {
-      dl->mPercentComplete = 100;
-
-      PRInt64 size;
-      rv = file->GetFileSize(&size);
-      NS_ENSURE_SUCCESS(rv, rv);
-      dl->mMaxBytes = dl->mCurrBytes = size;
-    } else {
-      dl->mPercentComplete = -1;
-      dl->mMaxBytes = LL_MAXUINT;
-    }
-  } else {
-    dl->mPercentComplete = 0;
-    dl->mMaxBytes = LL_MAXUINT;
-    dl->mCurrBytes = 0;
-  }
-
   rv = stmt->GetUTF8String(i++, dl->mEntityID);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  dl->SetProgressBytes(stmt->AsInt64(i++), stmt->AsInt64(i++));
+
   // Addrefing and returning
   NS_ADDREF(*retVal = dl);
   return NS_OK;
@@ -1252,7 +1253,9 @@ nsDownloadManager::RetryDownload(PRUint32 aID)
       dl->mDownloadState != nsIDownloadManager::DOWNLOAD_CANCELED)
     return NS_ERROR_FAILURE;
 
+  // reset time and download progress
   dl->SetStartTime(PR_Now());
+  dl->SetProgressBytes(0, -1);
 
   nsCOMPtr<nsIWebBrowserPersist> wbp =
     do_CreateInstance("@mozilla.org/embedding/browser/nsWebBrowserPersist;1", &rv);
@@ -1575,7 +1578,7 @@ nsDownload::nsDownload() : mDownloadState(nsIDownloadManager::DOWNLOAD_NOTSTARTE
                            mID(0),
                            mPercentComplete(0),
                            mCurrBytes(0),
-                           mMaxBytes(LL_MAXUINT),
+                           mMaxBytes(-1),
                            mStartTime(0),
                            mLastUpdate(PR_Now() - (PRUint32)gUpdateInterval),
                            mResumedAt(-1),
@@ -1825,17 +1828,12 @@ nsDownload::OnProgressChange64(nsIWebProgress *aWebProgress,
     }
   }
 
-  mCurrBytes = aCurTotalProgress;
-  mMaxBytes = aMaxTotalProgress;
+  SetProgressBytes(aCurTotalProgress, aMaxTotalProgress);
 
-  PRUint64 currBytes, maxBytes;
+  // Report to the listener our real sizes
+  PRInt64 currBytes, maxBytes;
   (void)GetAmountTransferred(&currBytes);
   (void)GetSize(&maxBytes);
-  if (aMaxTotalProgress > 0)
-    mPercentComplete = (PRInt32)((PRFloat64)currBytes * 100 / maxBytes + .5);
-  else
-    mPercentComplete = -1;
-
   mDownloadManager->NotifyListenersOnProgressChange(
     aWebProgress, aRequest, currBytes, maxBytes, currBytes, maxBytes, this);
 
@@ -1941,11 +1939,30 @@ nsDownload::OnStateChange(nsIWebProgress *aWebProgress,
     }
   } else if (aStateFlags & STATE_STOP) {
     if (IsFinishable()) {
-      // Set file size at the end of a transfer (for unknown transfer amounts)
-      if (mMaxBytes == LL_MAXUINT)
+      // We can't completely trust the bytes we've added up because we might be
+      // missing on some/all of the progress updates (especially from cache).
+      // Our best bet is the file itself, but if for some reason it's gone, the
+      // next best is what we've calculated.
+      PRInt64 fileSize;
+      nsCOMPtr<nsILocalFile> file;
+      //  We need a nsIFile clone to deal with file size caching issues. :(
+      nsCOMPtr<nsIFile> clone;
+      if (NS_SUCCEEDED(GetTargetFile(getter_AddRefs(file))) &&
+          NS_SUCCEEDED(file->Clone(getter_AddRefs(clone))) &&
+          NS_SUCCEEDED(clone->GetFileSize(&fileSize)) && fileSize > 0) {
+        mCurrBytes = mMaxBytes = fileSize;
+
+        // If we resumed, keep the fact that we did and fix size calculations
+        if (WasResumed())
+          mResumedAt = 0;
+      } else if (mMaxBytes == -1) {
         mMaxBytes = mCurrBytes;
+      } else {
+        mCurrBytes = mMaxBytes;
+      }
 
       mPercentComplete = 100;
+      mLastUpdate = PR_Now();
 
 #ifdef XP_WIN
       (void)SetState(nsIDownloadManager::DOWNLOAD_SCANNING);
@@ -2036,16 +2053,16 @@ nsDownload::GetPercentComplete(PRInt32 *aPercentComplete)
 }
 
 NS_IMETHODIMP
-nsDownload::GetAmountTransferred(PRUint64 *aAmountTransferred)
+nsDownload::GetAmountTransferred(PRInt64 *aAmountTransferred)
 {
   *aAmountTransferred = mCurrBytes + (WasResumed() ? mResumedAt : 0);
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsDownload::GetSize(PRUint64 *aSize)
+nsDownload::GetSize(PRInt64 *aSize)
 {
-  *aSize = mMaxBytes + (WasResumed() && mMaxBytes != LL_MAXUINT ? mResumedAt : 0);
+  *aSize = mMaxBytes + (WasResumed() && mMaxBytes != -1 ? mResumedAt : 0);
   return NS_OK;
 }
 
@@ -2093,6 +2110,25 @@ nsDownload::GetReferrer(nsIURI **referrer)
   return NS_OK;
 }
 
+void
+nsDownload::SetProgressBytes(PRInt64 aCurrBytes, PRInt64 aMaxBytes)
+{
+  mCurrBytes = aCurrBytes;
+  mMaxBytes = aMaxBytes;
+
+  // Get the real bytes that include resume position
+  PRInt64 currBytes, maxBytes;
+  (void)GetAmountTransferred(&currBytes);
+  (void)GetSize(&maxBytes);
+
+  if (currBytes == maxBytes)
+    mPercentComplete = 100;
+  else if (maxBytes <= 0)
+    mPercentComplete = -1;
+  else
+    mPercentComplete = (PRInt32)((PRFloat64)currBytes / maxBytes * 100 + .5);
+}
+
 nsresult
 nsDownload::Pause()
 {
@@ -2181,7 +2217,7 @@ nsDownload::RealResume()
   // Track where we resumed because progress notifications restart at 0
   mResumedAt = fileSize;
   mCurrBytes = 0;
-  mMaxBytes = LL_MAXUINT;
+  mMaxBytes = -1;
 
   // Set the referrer
   if (mReferrer) {
@@ -2286,6 +2322,18 @@ nsDownload::UpdateDB()
   rv = stmt->BindUTF8StringParameter(i++, mEntityID);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // currBytes
+  PRInt64 currBytes;
+  (void)GetAmountTransferred(&currBytes);
+  rv = stmt->BindInt64Parameter(i++, currBytes);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // maxBytes
+  PRInt64 maxBytes;
+  (void)GetSize(&maxBytes);
+  rv = stmt->BindInt64Parameter(i++, maxBytes);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   // id
   rv = stmt->BindInt64Parameter(i++, mID);
   NS_ENSURE_SUCCESS(rv, rv);

toolkit/components/downloads/src/nsDownloadManager.h

@@ -214,6 +214,12 @@ public:
 protected:
   void SetStartTime(PRInt64 aStartTime);
 
+  /**
+   * Update the amount of bytes transferred and max bytes; and recalculate the
+   * download percent.
+   */
+  void SetProgressBytes(PRInt64 aCurrBytes, PRInt64 aMaxBytes);
+
   /**
    * Pause the download, but in certain cases it might get fake-paused instead
    * of real-paused.
@@ -291,8 +297,8 @@ private:
    * doesn't necessarily mean we have nothing. Use GetAmountTransferred and
    * GetSize for the real transferred amount and size.
    */
-  PRUint64 mCurrBytes;
-  PRUint64 mMaxBytes;
+  PRInt64 mCurrBytes;
+  PRInt64 mMaxBytes;
 
   PRTime mStartTime;
   PRTime mLastUpdate;

toolkit/components/downloads/test/schema_migration/test_migration_to_6.js

@@ -0,0 +1,79 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Download Manager Test Code.
+ *
+ * The Initial Developer of the Original Code is
+ * Mozilla Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2007
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Shawn Wilsher <me@shawnwilsher.com> (Original Author)
+ *   Srirang G Doddihal <brahmana@doddihal.com>
+ *   Edward Lee <edward.lee@engineering.uiuc.edu>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by declaring the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not declare
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+// This file tests migration from v5 to v6
+
+function run_test()
+{
+  // First import the downloads.sqlite file
+  importDatabaseFile("v5.sqlite");
+
+  // ok, now it is OK to init the download manager - this will perform the
+  // migration!
+  var dm = Cc["@mozilla.org/download-manager;1"].
+           getService(Ci.nsIDownloadManager);
+  var dbConn = dm.DBConnection;
+
+  // check schema version
+  do_check_true(dbConn.schemaVersion >= 6);
+
+  // Check that the columns exist (no throw) and entries are correct
+  var stmt = dbConn.createStatement(
+    "SELECT name, source, target, startTime, endTime, state, referrer, " +
+           "entityID, tempPath, currBytes, maxBytes " +
+    "FROM moz_downloads " +
+    "WHERE id = 27");
+  stmt.executeStep();
+  do_check_eq("Firefox 2.0.0.6.dmg", stmt.getString(0));
+  do_check_eq("http://ftp-mozilla.netscape.com/pub/mozilla.org/firefox/releases/2.0.0.6/mac/en-US/Firefox%202.0.0.6.dmg",
+              stmt.getUTF8String(1));
+  do_check_eq("file:///Users/sdwilsh/Desktop/Firefox%202.0.0.6.dmg",
+              stmt.getUTF8String(2));
+  do_check_eq(1187390974170783, stmt.getInt64(3));
+  do_check_eq(1187391001257446, stmt.getInt64(4));
+  do_check_eq(1, stmt.getInt32(5));
+  do_check_eq("http://www.mozilla.com/en-US/products/download.html?product=firefox-2.0.0.6&os=osx&lang=en-US",stmt.getUTF8String(6));
+  do_check_true(stmt.getIsNull(7));
+  do_check_true(stmt.getIsNull(8));
+  do_check_eq(0, stmt.getInt64(9));
+  do_check_eq(-1, stmt.getInt64(10));
+  stmt.reset();
+
+  cleanup();
+}