From 987f6f9922d368f3e3e6e0310bd0cb96ff3093c9 Mon Sep 17 00:00:00 2001 From: Francois Marier Date: Tue, 24 Mar 2015 20:47:00 -0400 Subject: [PATCH] Bug 1111741 - Enable SafeBrowsing remote lookups for mac and linux. r=mmc --HG-- rename : toolkit/components/downloads/test/unit/test_app_rep_windows.js => toolkit/components/downloads/test/unit/test_app_rep_maclinux.js --- browser/app/profile/firefox.js | 7 +- .../downloads/ApplicationReputation.cpp | 17 +- .../test/unit/test_app_rep_maclinux.js | 303 ++++++++++++++++++ .../test/unit/test_app_rep_windows.js | 2 +- .../downloads/test/unit/xpcshell.ini | 2 + 5 files changed, 317 insertions(+), 14 deletions(-) create mode 100644 toolkit/components/downloads/test/unit/test_app_rep_maclinux.js diff --git a/browser/app/profile/firefox.js b/browser/app/profile/firefox.js index 4b7127772bcd..afba2e4743a9 100644 --- a/browser/app/profile/firefox.js +++ b/browser/app/profile/firefox.js @@ -969,12 +969,7 @@ pref("gecko.handlerService.allowRegisterFromDifferentHost", false); pref("browser.safebrowsing.enabled", true); pref("browser.safebrowsing.malware.enabled", true); pref("browser.safebrowsing.downloads.enabled", true); -// Remote lookups are only enabled for Windows in Nightly and Aurora -#if defined(XP_WIN) pref("browser.safebrowsing.downloads.remote.enabled", true); -#else -pref("browser.safebrowsing.downloads.remote.enabled", false); -#endif pref("browser.safebrowsing.debug", false); pref("browser.safebrowsing.updateURL", "https://safebrowsing.google.com/safebrowsing/downloads?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2&key=%GOOGLE_API_KEY%"); @@ -1015,7 +1010,7 @@ pref("urlclassifier.downloadBlockTable", "goog-badbinurl-shavar"); #ifdef XP_WIN // Only download the whitelist on Windows, since the whitelist is // only useful for suppressing remote lookups for signed binaries which we can -// only verify on Windows (Bug 974579). +// only verify on Windows (Bug 974579). Other platforms always do remote lookups. pref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256"); #endif #endif diff --git a/toolkit/components/downloads/ApplicationReputation.cpp b/toolkit/components/downloads/ApplicationReputation.cpp index 2f7576bb9ca2..8ae98772281c 100644 --- a/toolkit/components/downloads/ApplicationReputation.cpp +++ b/toolkit/components/downloads/ApplicationReputation.cpp @@ -429,23 +429,17 @@ PendingLookup::LookupNext() nsRefPtr lookup(new PendingDBLookup(this)); return lookup->LookupSpec(spec, true); } -#ifdef XP_WIN // There are no more URIs to check against local list. If the file is // not eligible for remote lookup, bail. if (!IsBinaryFile()) { LOG(("Not eligible for remote lookups [this=%x]", this)); return OnComplete(false, NS_OK); } - // Send the remote query if we are on Windows. nsresult rv = SendRemoteQuery(); if (NS_FAILED(rv)) { return OnComplete(false, rv); } return NS_OK; -#else - LOG(("PendingLookup: Nothing left to check [this=%p]", this)); - return OnComplete(false, NS_OK); -#endif } nsCString @@ -818,6 +812,7 @@ PendingLookup::SendRemoteQueryInternal() { // If we aren't supposed to do remote lookups, bail. if (!Preferences::GetBool(PREF_SB_DOWNLOADS_REMOTE_ENABLED, false)) { + LOG(("Remote lookups are disabled [this = %p]", this)); return NS_ERROR_NOT_AVAILABLE; } // If the remote lookup URL is empty or absent, bail. @@ -825,6 +820,7 @@ PendingLookup::SendRemoteQueryInternal() NS_ENSURE_SUCCESS(Preferences::GetCString(PREF_SB_APP_REP_URL, &serviceUrl), NS_ERROR_NOT_AVAILABLE); if (serviceUrl.EqualsLiteral("")) { + LOG(("Remote lookup URL is empty [this = %p]", this)); return NS_ERROR_NOT_AVAILABLE; } @@ -834,13 +830,18 @@ PendingLookup::SendRemoteQueryInternal() NS_ENSURE_SUCCESS(Preferences::GetCString(PREF_DOWNLOAD_BLOCK_TABLE, &table), NS_ERROR_NOT_AVAILABLE); if (table.EqualsLiteral("")) { + LOG(("Blocklist is empty [this = %p]", this)); return NS_ERROR_NOT_AVAILABLE; } +#ifdef XP_WIN + // The allowlist is only needed to do signature verification on Windows NS_ENSURE_SUCCESS(Preferences::GetCString(PREF_DOWNLOAD_ALLOW_TABLE, &table), NS_ERROR_NOT_AVAILABLE); if (table.EqualsLiteral("")) { + LOG(("Allowlist is empty [this = %p]", this)); return NS_ERROR_NOT_AVAILABLE; } +#endif LOG(("Sending remote query for application reputation [this = %p]", this)); @@ -891,6 +892,8 @@ PendingLookup::SendRemoteQueryInternal() if (!mRequest.SerializeToString(&serialized)) { return NS_ERROR_UNEXPECTED; } + LOG(("Serialized protocol buffer [this = %p]: %s", this, + serialized.c_str())); // Set the input stream to the serialized protocol buffer nsCOMPtr sstream = @@ -1020,7 +1023,7 @@ PendingLookup::OnStopRequestInternal(nsIRequest *aRequest, std::string buf(mResponse.Data(), mResponse.Length()); safe_browsing::ClientDownloadResponse response; if (!response.ParseFromString(buf)) { - NS_WARNING("Could not parse protocol buffer"); + LOG(("Invalid protocol buffer response [this = %p]: %s", this, buf.c_str())); Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_SERVER, SERVER_RESPONSE_INVALID); return NS_ERROR_CANNOT_CONVERT_DATA; diff --git a/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js b/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js new file mode 100644 index 000000000000..1df17b2d7d01 --- /dev/null +++ b/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js @@ -0,0 +1,303 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This file tests signature extraction using Windows Authenticode APIs of + * downloaded files. + */ + +//////////////////////////////////////////////////////////////////////////////// +//// Globals + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", + "resource://gre/modules/NetUtil.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Promise", + "resource://gre/modules/Promise.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Task", + "resource://gre/modules/Task.jsm"); + +const gAppRep = Cc["@mozilla.org/downloads/application-reputation-service;1"]. + getService(Ci.nsIApplicationReputationService); +let gStillRunning = true; +let gTables = {}; +let gHttpServer = null; + +function readFileToString(aFilename) { + let f = do_get_file(aFilename); + let stream = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + stream.init(f, -1, 0, 0); + let buf = NetUtil.readInputStreamToString(stream, stream.available()); + return buf; +} + +function registerTableUpdate(aTable, aFilename) { + // If we haven't been given an update for this table yet, add it to the map + if (!(aTable in gTables)) { + gTables[aTable] = []; + } + + // The number of chunks associated with this table. + let numChunks = gTables[aTable].length + 1; + let redirectPath = "/" + aTable + "-" + numChunks; + let redirectUrl = "localhost:4444" + redirectPath; + + // Store redirect url for that table so we can return it later when we + // process an update request. + gTables[aTable].push(redirectUrl); + + gHttpServer.registerPathHandler(redirectPath, function(request, response) { + do_print("Mock safebrowsing server handling request for " + redirectPath); + let contents = readFileToString(aFilename); + do_print("Length of " + aFilename + ": " + contents.length); + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(contents, contents.length); + }); +} + +//////////////////////////////////////////////////////////////////////////////// +//// Tests + +function run_test() +{ + run_next_test(); +} + +add_task(function test_setup() +{ + // Wait 10 minutes, that is half of the external xpcshell timeout. + do_timeout(10 * 60 * 1000, function() { + if (gStillRunning) { + do_throw("Test timed out."); + } + }); + // Set up a local HTTP server to return bad verdicts. + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/download"); + // Ensure safebrowsing is enabled for this test, even if the app + // doesn't have it enabled. + Services.prefs.setBoolPref("browser.safebrowsing.malware.enabled", true); + Services.prefs.setBoolPref("browser.safebrowsing.downloads.enabled", true); + // Set block table explicitly, no need for the allow table though + Services.prefs.setCharPref("urlclassifier.downloadBlockTable", + "goog-badbinurl-shavar"); + // SendRemoteQueryInternal needs locale preference. + let locale = Services.prefs.getCharPref("general.useragent.locale"); + Services.prefs.setCharPref("general.useragent.locale", "en-US"); + + do_register_cleanup(function() { + Services.prefs.clearUserPref("browser.safebrowsing.malware.enabled"); + Services.prefs.clearUserPref("browser.safebrowsing.downloads.enabled"); + Services.prefs.clearUserPref("urlclassifier.downloadBlockTable"); + Services.prefs.setCharPref("general.useragent.locale", locale); + }); + + gHttpServer = new HttpServer(); + gHttpServer.registerDirectory("/", do_get_cwd()); + + function createVerdict(aShouldBlock) { + // We can't programmatically create a protocol buffer here, so just + // hardcode some already serialized ones. + let blob = String.fromCharCode(parseInt(0x08, 16)); + if (aShouldBlock) { + // A safe_browsing::ClientDownloadRequest with a DANGEROUS verdict + blob += String.fromCharCode(parseInt(0x01, 16)); + } else { + // A safe_browsing::ClientDownloadRequest with a SAFE verdict + blob += String.fromCharCode(parseInt(0x00, 16)); + } + return blob; + } + + gHttpServer.registerPathHandler("/throw", function(request, response) { + do_throw("We shouldn't be getting here"); + }); + + gHttpServer.registerPathHandler("/download", function(request, response) { + do_print("Querying remote server for verdict"); + response.setHeader("Content-Type", "application/octet-stream", false); + let buf = NetUtil.readInputStreamToString( + request.bodyInputStream, + request.bodyInputStream.available()); + do_print("Request length: " + buf.length); + // A garbage response. By default this produces NS_CANNOT_CONVERT_DATA as + // the callback status. + let blob = "this is not a serialized protocol buffer"; + // We can't actually parse the protocol buffer here, so just switch on the + // length instead of inspecting the contents. + if (buf.length == 65) { + // evil.com + blob = createVerdict(true); + } else if (buf.length == 71) { + // mozilla.com + blob = createVerdict(false); + } + response.bodyOutputStream.write(blob, blob.length); + }); + + gHttpServer.start(4444); +}); + +// Construct a response with redirect urls. +function processUpdateRequest() { + let response = "n:1000\n"; + for (let table in gTables) { + response += "i:" + table + "\n"; + for (let i = 0; i < gTables[table].length; ++i) { + response += "u:" + gTables[table][i] + "\n"; + } + } + do_print("Returning update response: " + response); + return response; +} + +// Set up the local whitelist. +function waitForUpdates() { + let deferred = Promise.defer(); + gHttpServer.registerPathHandler("/downloads", function(request, response) { + let buf = NetUtil.readInputStreamToString(request.bodyInputStream, + request.bodyInputStream.available()); + let blob = processUpdateRequest(); + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(blob, blob.length); + }); + + let streamUpdater = Cc["@mozilla.org/url-classifier/streamupdater;1"] + .getService(Ci.nsIUrlClassifierStreamUpdater); + + // Load up some update chunks for the safebrowsing server to serve. This + // particular chunk contains the hash of whitelisted.com/ and + // sb-ssl.google.com/safebrowsing/csd/certificate/. + registerTableUpdate("goog-downloadwhite-digest256", "data/digest.chunk"); + + // Resolve the promise once processing the updates is complete. + function updateSuccess(aEvent) { + // Timeout of n:1000 is constructed in processUpdateRequest above and + // passed back in the callback in nsIUrlClassifierStreamUpdater on success. + do_check_eq("1000", aEvent); + do_print("All data processed"); + deferred.resolve(true); + } + // Just throw if we ever get an update or download error. + function handleError(aEvent) { + do_throw("We didn't download or update correctly: " + aEvent); + deferred.reject(); + } + streamUpdater.downloadUpdates( + "goog-downloadwhite-digest256", + "goog-downloadwhite-digest256;\n", + "http://localhost:4444/downloads", + updateSuccess, handleError, handleError); + return deferred.promise; +} + +function promiseQueryReputation(query, expectedShouldBlock) { + let deferred = Promise.defer(); + function onComplete(aShouldBlock, aStatus) { + do_check_eq(Cr.NS_OK, aStatus); + do_check_eq(aShouldBlock, expectedShouldBlock); + deferred.resolve(true); + } + gAppRep.queryReputation(query, onComplete); + return deferred.promise; +} + +add_task(function() +{ + // Wait for Safebrowsing local list updates to complete. + yield waitForUpdates(); +}); + +add_task(function test_blocked_binary() +{ + // We should reach the remote server for a verdict. + Services.prefs.setBoolPref("browser.safebrowsing.downloads.remote.enabled", + true); + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/download"); + // evil.com should return a malware verdict from the remote server. + yield promiseQueryReputation({sourceURI: createURI("http://evil.com"), + suggestedFileName: "noop.bat", + fileSize: 12}, true); +}); + +add_task(function test_non_binary() +{ + // We should not reach the remote server for a verdict for non-binary files. + Services.prefs.setBoolPref("browser.safebrowsing.downloads.remote.enabled", + true); + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/throw"); + yield promiseQueryReputation({sourceURI: createURI("http://evil.com"), + suggestedFileName: "noop.txt", + fileSize: 12}, false); +}); + +add_task(function test_good_binary() +{ + // We should reach the remote server for a verdict. + Services.prefs.setBoolPref("browser.safebrowsing.downloads.remote.enabled", + true); + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/download"); + // mozilla.com should return a not-guilty verdict from the remote server. + yield promiseQueryReputation({sourceURI: createURI("http://mozilla.com"), + suggestedFileName: "noop.bat", + fileSize: 12}, false); +}); + +add_task(function test_disabled() +{ + // Explicitly disable remote checks + Services.prefs.setBoolPref("browser.safebrowsing.downloads.remote.enabled", + false); + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/throw"); + let query = {sourceURI: createURI("http://example.com"), + suggestedFileName: "noop.bat", + fileSize: 12}; + let deferred = Promise.defer(); + gAppRep.queryReputation(query, + function onComplete(aShouldBlock, aStatus) { + // We should be getting NS_ERROR_NOT_AVAILABLE if the service is disabled + do_check_eq(Cr.NS_ERROR_NOT_AVAILABLE, aStatus); + do_check_false(aShouldBlock); + deferred.resolve(true); + } + ); + yield deferred.promise; +}); + +add_task(function test_disabled_through_lists() +{ + Services.prefs.setBoolPref("browser.safebrowsing.downloads.remote.enabled", + false); + Services.prefs.setCharPref("browser.safebrowsing.appRepURL", + "http://localhost:4444/download"); + Services.prefs.setCharPref("urlclassifier.downloadBlockTable", ""); + let query = {sourceURI: createURI("http://example.com"), + suggestedFileName: "noop.bat", + fileSize: 12}; + let deferred = Promise.defer(); + gAppRep.queryReputation(query, + function onComplete(aShouldBlock, aStatus) { + // We should be getting NS_ERROR_NOT_AVAILABLE if the service is disabled + do_check_eq(Cr.NS_ERROR_NOT_AVAILABLE, aStatus); + do_check_false(aShouldBlock); + deferred.resolve(true); + } + ); + yield deferred.promise; +}); +add_task(function test_teardown() +{ + gStillRunning = false; +}); diff --git a/toolkit/components/downloads/test/unit/test_app_rep_windows.js b/toolkit/components/downloads/test/unit/test_app_rep_windows.js index 515fe73bc54f..444aa311ad50 100644 --- a/toolkit/components/downloads/test/unit/test_app_rep_windows.js +++ b/toolkit/components/downloads/test/unit/test_app_rep_windows.js @@ -186,7 +186,7 @@ add_task(function test_setup() "goog-badbinurl-shavar"); Services.prefs.setCharPref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256"); - // On Windows SendRemoteQueryInternal needs locale preference. + // SendRemoteQueryInternal needs locale preference. let locale = Services.prefs.getCharPref("general.useragent.locale"); Services.prefs.setCharPref("general.useragent.locale", "en-US"); diff --git a/toolkit/components/downloads/test/unit/xpcshell.ini b/toolkit/components/downloads/test/unit/xpcshell.ini index e40e6254f70a..a6844e4171a4 100644 --- a/toolkit/components/downloads/test/unit/xpcshell.ini +++ b/toolkit/components/downloads/test/unit/xpcshell.ini @@ -12,6 +12,8 @@ support-files = [test_app_rep.js] [test_app_rep_windows.js] skip-if = os != "win" +[test_app_rep_maclinux.js] +skip-if = os == "win" [test_bug_382825.js] [test_bug_384744.js] [test_bug_395092.js]