Bug 1620621 - Add support for fallback to dumps for attachments r=Gijs,leplatrem

With this piece, it is now possible for RemoteSettings clients to always
have a valid attachment.

This adds the client APIs that could support bug 1542177.

Differential Revision: https://phabricator.services.mozilla.com/D72417

services/common/docs/RemoteSettings.rst

@@ -162,6 +162,9 @@ The provided helper will:
     This enables callers to always have a valid pair of attachment and record,
     provided that the attachment has been retrieved at least once.
 
+    The ``fallbackToDump`` option activates a fallback to a dump that has been
+    packaged with the client, when other ways to load the attachment have failed.
+
 .. note::
 
     A ``downloadAsBytes()`` method returning an ``ArrayBuffer`` is also available, if writing the attachment into the user profile is not necessary.

services/settings/Attachments.jsm

@@ -71,6 +71,9 @@ class Downloader {
    * @param {Boolean} options.fallbackToCache Return the cached attachment when the
    *                                          input record cannot be fetched.
    *                                          (default: false)
+   * @param {Boolean} options.fallbackToDump Use the remote settings dump as a
+   *                                         potential source of the attachment.
+   *                                         (default: false)
    * @throws {Downloader.DownloadError} if the file could not be fetched.
    * @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
    * @returns {Object} An object with two properties:
@@ -85,6 +88,7 @@ class Downloader {
       attachmentId = record?.id,
       useCache = false,
       fallbackToCache = false,
+      fallbackToDump = false,
     } = options || {};
 
     if (!useCache) {
@@ -131,6 +135,28 @@ class Downloader {
 
     let errorIfAllFails;
 
+    // No cached attachment available. Check if an attachment is available in
+    // the dump that is packaged with the client.
+    let dumpInfo;
+    if (fallbackToDump && record) {
+      try {
+        dumpInfo = await this._readAttachmentDump(attachmentId);
+        // Check if there is a match. If there is no match, we will fall through
+        // to the next case (downloading from the network). We may still use the
+        // dump at the end of the function as the ultimate fallback.
+        if (record.attachment.hash === dumpInfo.record.attachment.hash) {
+          return {
+            buffer: await dumpInfo.readBuffer(),
+            record: dumpInfo.record,
+            _source: "dump_match",
+          };
+        }
+      } catch (e) {
+        fallbackToDump = false;
+        errorIfAllFails = e;
+      }
+    }
+
     // There is no local version that matches the requested record.
     // Try to download the attachment specified in record.
     if (record && record.attachment) {
@@ -164,6 +190,20 @@ class Downloader {
       }
     }
 
+    // Unable to find a valid attachment, fall back to the packaged dump.
+    if (fallbackToDump) {
+      try {
+        dumpInfo = dumpInfo || (await this._readAttachmentDump(attachmentId));
+        return {
+          buffer: await dumpInfo.readBuffer(),
+          record: dumpInfo.record,
+          _source: "dump_fallback",
+        };
+      } catch (e) {
+        errorIfAllFails = e;
+      }
+    }
+
     if (errorIfAllFails) {
       throw errorIfAllFails;
     }
@@ -318,6 +358,30 @@ class Downloader {
     return resp.arrayBuffer();
   }
 
+  async _readAttachmentDump(attachmentId) {
+    async function fetchResource(resourceUrl) {
+      try {
+        return await fetch(resourceUrl);
+      } catch (e) {
+        throw new Downloader.DownloadError(resourceUrl);
+      }
+    }
+    const resourceUrlPrefix =
+      Downloader._RESOURCE_BASE_URL + "/" + this.folders.join("/") + "/";
+    const recordUrl = `${resourceUrlPrefix}${attachmentId}.meta.json`;
+    const attachmentUrl = `${resourceUrlPrefix}${attachmentId}`;
+    const record = await (await fetchResource(recordUrl)).json();
+    return {
+      record,
+      async readBuffer() {
+        return (await fetchResource(attachmentUrl)).arrayBuffer();
+      },
+    };
+  }
+
+  // Separate variable to allow tests to override this.
+  static _RESOURCE_BASE_URL = "resource://app/defaults";
+
   async _makeDirs() {
     const dirPath = OS.Path.join(
       OS.Constants.Path.localProfileDir,

services/settings/test/unit/test_attachments_downloader.js

@@ -27,6 +27,16 @@ const RECORD = {
   },
 };
 
+const RECORD_OF_DUMP = {
+  id: "filename-of-dump.txt",
+  attachment: {
+    filename: "filename-of-dump.txt",
+    hash: "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
+    size: 25,
+  },
+  some_key: "some metadata",
+};
+
 let downloader;
 let server;
 
@@ -388,3 +398,73 @@ add_task(async function test_download_cached() {
   );
 });
 add_task(clear_state);
+
+add_task(async function test_download_from_dump() {
+  const bucketNamePref = "services.testing.custom-bucket-name-in-this-test";
+  Services.prefs.setCharPref(bucketNamePref, "dump-bucket");
+  const client = RemoteSettings("dump-collection", { bucketNamePref });
+
+  // Temporarily replace the resource:-URL with another resource:-URL.
+  const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL;
+  Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test";
+  const resProto = Services.io
+    .getProtocolHandler("resource")
+    .QueryInterface(Ci.nsIResProtocolHandler);
+  resProto.setSubstitution(
+    "rs-downloader-test",
+    Services.io.newFileURI(do_get_file("test_attachments_downloader"))
+  );
+
+  function checkInfo(result, expectedSource) {
+    Assert.equal(
+      new TextDecoder().decode(new Uint8Array(result.buffer)),
+      "This would be a RS dump.\n",
+      "expected content from dump"
+    );
+    Assert.deepEqual(result.record, RECORD_OF_DUMP, "expected record for dump");
+    Assert.equal(result._source, expectedSource, "expected source of dump");
+  }
+
+  // If record matches, should happen before network request.
+  const dump1 = await client.attachments.download(RECORD_OF_DUMP, {
+    // Note: attachmentId not set, so should fall back to record.id.
+    useCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump1, "dump_match");
+
+  // If no record given, should try network first, but then fall back to dump.
+  const dump2 = await client.attachments.download(null, {
+    attachmentId: RECORD_OF_DUMP.id,
+    useCache: true,
+    fallbackToDump: true,
+  });
+  checkInfo(dump2, "dump_fallback");
+
+  await Assert.rejects(
+    client.attachments.download(null, {
+      attachmentId: "filename-without-meta.txt",
+      useCache: true,
+      fallbackToDump: true,
+    }),
+    /Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-meta\.txt/,
+    "Cannot download dump that lacks a .meta.json file"
+  );
+
+  await Assert.rejects(
+    client.attachments.download(null, {
+      attachmentId: "filename-without-content.txt",
+      useCache: true,
+      fallbackToDump: true,
+    }),
+    /Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-content\.txt/,
+    "Cannot download dump that is missing, despite the existing .meta.json"
+  );
+
+  // Restore, just in case.
+  Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL;
+  resProto.setSubstitution("rs-downloader-test", null);
+});
+// Not really needed because the last test doesn't modify the main collection,
+// but added for consistency with other tests tasks around here.
+add_task(clear_state);

services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt

@@ -0,0 +1 @@
+This would be a RS dump.

services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json

@@ -0,0 +1,9 @@
+{
+  "id": "filename-of-dump.txt",
+  "attachment": {
+    "filename": "filename-of-dump.txt",
+    "hash": "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
+    "size": 25
+  },
+  "some_key": "some metadata"
+}

services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.meta.json

@@ -0,0 +1,9 @@
+{
+  "fyi": "This .meta.json file describes an attachment, but that attachment is missing.",
+  "attachment": {
+    "filename": "filename-without-content.txt",
+    "hash": "...",
+    "size": "..."
+  }
+}
+

services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt

@@ -0,0 +1 @@
+The filename-without-meta.txt.meta.json file is missing.