diff --git a/browser/components/downloads/content/downloads.js b/browser/components/downloads/content/downloads.js
index 695ae02faebf..f81b43c8c5fc 100644
--- a/browser/components/downloads/content/downloads.js
+++ b/browser/components/downloads/content/downloads.js
@@ -322,9 +322,9 @@ const DownloadsPanel = {
     // Since at most one popup is open at any given time, we can set globally.
     DownloadsCommon.getIndicatorData(window).attentionSuppressed = true;
 
-    // Ensure that an item is selected when the panel is focused.
+    // Ensure that the first item is selected when the panel is focused.
     if (DownloadsView.richListBox.itemCount > 0 &&
-        !DownloadsView.richListBox.selectedItem) {
+        DownloadsView.richListBox.selectedIndex == -1) {
       DownloadsView.richListBox.selectedIndex = 0;
     }
 
@@ -417,6 +417,10 @@ const DownloadsPanel = {
         aEvent.keyCode == Ci.nsIDOMKeyEvent.DOM_VK_DOWN) &&
         !this.keyFocusing) {
       this.keyFocusing = true;
+      // Ensure there's a selection, we will show the focus ring around it and
+      // prevent the richlistbox from changing the selection.
+      if (DownloadsView.richListBox.selectedIndex == -1)
+        DownloadsView.richListBox.selectedIndex = 0;
       aEvent.preventDefault();
       return;
     }
@@ -904,8 +908,10 @@ const DownloadsView = {
     let element = this.getViewItem(aDataItem)._element;
     let previousSelectedIndex = this.richListBox.selectedIndex;
     this.richListBox.removeChild(element);
-    this.richListBox.selectedIndex = Math.min(previousSelectedIndex,
-                                              this.richListBox.itemCount - 1);
+    if (previousSelectedIndex != -1) {
+      this.richListBox.selectedIndex = Math.min(previousSelectedIndex,
+                                                this.richListBox.itemCount - 1);
+    }
     delete this._viewItems[aDataItem.downloadGuid];
   },
 
@@ -963,6 +969,29 @@ const DownloadsView = {
     }
   },
 
+
+  /**
+   * Mouse listeners to handle selection on hover.
+   */
+  onDownloadMouseOver: function DV_onDownloadMouseOver(aEvent)
+  {
+    if (aEvent.originalTarget.parentNode == this.richListBox)
+      this.richListBox.selectedItem = aEvent.originalTarget;
+  },
+  onDownloadMouseOut: function DV_onDownloadMouseOut(aEvent)
+  {
+    if (aEvent.originalTarget.parentNode == this.richListBox) {
+      // If the destination element is outside of the richlistitem, clear the
+      // selection.
+      let element = aEvent.relatedTarget;
+      while (element && element != aEvent.originalTarget) {
+        element = element.parentNode;
+      }
+      if (!element)
+        this.richListBox.selectedIndex = -1;
+    }
+  },
+
   onDownloadContextMenu: function DV_onDownloadContextMenu(aEvent)
   {
     let element = this.richListBox.selectedItem;
diff --git a/browser/components/downloads/content/downloadsOverlay.xul b/browser/components/downloads/content/downloadsOverlay.xul
index 569cadc4899d..971124d3b3d1 100644
--- a/browser/components/downloads/content/downloadsOverlay.xul
+++ b/browser/components/downloads/content/downloadsOverlay.xul
@@ -101,6 +101,8 @@
                    class="plain"
                    flex="1"
                    context="downloadsContextMenu"
+                   onmouseover="DownloadsView.onDownloadMouseOver(event);"
+                   onmouseout="DownloadsView.onDownloadMouseOut(event);"
                    oncontextmenu="DownloadsView.onDownloadContextMenu(event);"
                    ondragstart="DownloadsView.onDownloadDragStart(event);"/>