Bug 1559132 - Add ability to download Remote Settings attachments as bytes r=glasserc

Differential Revision: https://phabricator.services.mozilla.com/D53812

--HG--
extra : moz-landing-system : lando

services/common/docs/RemoteSettings.rst

@@ -133,6 +133,7 @@ Remote files are not downloaded automatically. In order to keep attachments in s
 
 The provided helper will:
 - fetch the remote binary content
+- write the file in the profile folder
 - check the file size
 - check the content SHA256 hash
 - do nothing if the file is already present and sound locally.
@@ -141,6 +142,7 @@ The provided helper will:
 
     The following aspects are not taken care of (yet! help welcome):
 
+    - report Telemetry when download fails
     - check available disk space
     - preserve bandwidth
     - resume downloads of large files
@@ -151,6 +153,10 @@ The provided helper will:
     This will allow us to package attachments as part of a Firefox release (see `Bug 1542177 <https://bugzilla.mozilla.org/show_bug.cgi?id=1542177>`_)
     and return them to calling code as ``resource://`` from within a package archive.
 
+.. notes::
+
+    A ``downloadAsBytes()`` method returning an ``ArrayBuffer`` is also available, if writing the attachment into the user profile is not necessary.
+
 
 .. _services/initial-data:
 

services/settings/Attachments.jsm

@@ -60,7 +60,7 @@ class Downloader {
   async download(record, options = {}) {
     const { retries = 3 } = options;
     const {
-      attachment: { location, filename, hash, size },
+      attachment: { filename, size, hash },
     } = record;
     const localFilePath = OS.Path.join(
       OS.Constants.Path.localProfileDir,
@@ -72,7 +72,6 @@ class Downloader {
       ...this.folders,
       filename,
     ].join("/")}`;
-    const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
 
     await this._makeDirs();
 
@@ -86,7 +85,54 @@ class Downloader {
         throw new Downloader.BadContentError(localFilePath);
       }
       try {
-        await this._fetchAttachment(remoteFileUrl, localFilePath);
+        // Download and write on disk.
+        const buffer = await this.downloadAsBytes(record, {
+          checkHash: false, // Hash will be checked on file.
+          retries: 0, // Already in a retry loop.
+        });
+        await OS.File.writeAtomic(localFilePath, buffer, {
+          tmpPath: `${localFilePath}.tmp`,
+        });
+      } catch (e) {
+        if (retried >= retries) {
+          throw e;
+        }
+      }
+      retried++;
+    }
+  }
+
+  /**
+   * Download the record attachment and return its content as bytes.
+   *
+   * @param {Object} record A Remote Settings entry with attachment.
+   * @param {Object} options Some download options.
+   * @param {Number} options.retries Number of times download should be retried (default: `3`)
+   * @param {Number} options.checkHash Check content integrity (default: `true`)
+   * @throws {Downloader.DownloadError} if the file could not be fetched.
+   * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
+   * @returns {ArrayBuffer} the file content.
+   */
+  async downloadAsBytes(record, options = {}) {
+    const {
+      attachment: { location, hash, size },
+    } = record;
+
+    const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
+
+    const { retries = 3, checkHash = true } = options;
+    let retried = 0;
+    while (true) {
+      try {
+        const buffer = await this._fetchAttachment(remoteFileUrl);
+        if (!checkHash) {
+          return buffer;
+        }
+        if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
+          return buffer;
+        }
+        // Content is corrupted.
+        throw new Downloader.BadContentError(location);
       } catch (e) {
         if (retried >= retries) {
           throw e;
@@ -131,17 +177,14 @@ class Downloader {
     return this._cdnURL;
   }
 
-  async _fetchAttachment(url, destination) {
+  async _fetchAttachment(url) {
     const headers = new Headers();
     headers.set("Accept-Encoding", "gzip");
     const resp = await fetch(url, { headers });
     if (!resp.ok) {
       throw new Downloader.DownloadError(url, resp);
     }
-    const buffer = await resp.arrayBuffer();
+    return resp.arrayBuffer();
-    await OS.File.writeAtomic(destination, buffer, {
-      tmpPath: `${destination}.tmp`,
-    });
   }
 
   async _makeDirs() {

services/settings/RemoteSettingsWorker.js

@@ -86,14 +86,25 @@ const Agent = {
       // File does not exist.
       return false;
     }
+    const buffer = await resp.arrayBuffer();
+    const bytes = new Uint8Array(buffer);
+    return this.checkContentHash(bytes, size, hash);
+  },
+
+  /**
+   * Check that the specified content matches the expected size and SHA-256 hash.
+   * @param {ArrayBuffer} buffer binary content
+   * @param {Number} size expected file size
+   * @param {String} size expected file SHA-256 as hex string
+   * @returns {boolean}
+   */
+  async checkContentHash(buffer, size, hash) {
+    const bytes = new Uint8Array(buffer);
     // Has expected size? (saves computing hash)
-    const fileSize = parseInt(resp.headers.get("Content-Length"), 10);
+    if (bytes.length !== size) {
-    if (fileSize !== size) {
       return false;
     }
     // Has expected content?
-    const buffer = await resp.arrayBuffer();
-    const bytes = new Uint8Array(buffer);
     const hashBuffer = await crypto.subtle.digest("SHA-256", bytes);
     const hashBytes = new Uint8Array(hashBuffer);
     const toHex = b => b.toString(16).padStart(2, "0");

services/settings/RemoteSettingsWorker.jsm

@@ -115,6 +115,10 @@ class Worker {
   async checkFileHash(filepath, size, hash) {
     return this._execute("checkFileHash", [filepath, size, hash]);
   }
+
+  async checkContentHash(buffer, size, hash) {
+    return this._execute("checkContentHash", [buffer, size, hash]);
+  }
 }
 
 // Now, first add a shutdown blocker. If that fails, we must have

services/settings/test/unit/test_attachments_downloader.js

@@ -92,6 +92,14 @@ add_task(async function test_download_writes_file_in_profile() {
 });
 add_task(clear_state);
 
+add_task(async function test_download_as_bytes() {
+  const bytes = await downloader.downloadAsBytes(RECORD);
+
+  // See *.pem file in tests data.
+  Assert.ok(bytes.byteLength > 1500, `Wrong bytes size: ${bytes.byteLength}`);
+});
+add_task(clear_state);
+
 add_task(async function test_file_is_redownloaded_if_size_does_not_match() {
   const fileURL = await downloader.download(RECORD);
   const localFilePath = pathFromURL(fileURL);
@@ -134,9 +142,9 @@ add_task(async function test_download_is_retried_3_times_if_download_fails() {
 
   let called = 0;
   const _fetchAttachment = downloader._fetchAttachment;
-  downloader._fetchAttachment = (url, destination) => {
+  downloader._fetchAttachment = async url => {
     called++;
-    return _fetchAttachment(url, destination);
+    return _fetchAttachment(url);
   };
 
   let error;
@@ -159,7 +167,10 @@ add_task(async function test_download_is_retried_3_times_if_content_fails() {
     },
   };
   let called = 0;
-  downloader._fetchAttachment = () => called++;
+  downloader._fetchAttachment = async () => {
+    called++;
+    return new ArrayBuffer();
+  };
 
   let error;
   try {