Bug 1559132 - Add ability to download Remote Settings attachments as bytes r=glasserc

Differential Revision: https://phabricator.services.mozilla.com/D53812

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Mathieu Leplatre 2019-11-25 16:13:52 +00:00
Родитель 71a0f9df8f
Коммит a6d949c39b
5 изменённых файлов: 90 добавлений и 15 удалений

Просмотреть файл

@ -133,6 +133,7 @@ Remote files are not downloaded automatically. In order to keep attachments in s
The provided helper will:
- fetch the remote binary content
- write the file in the profile folder
- check the file size
- check the content SHA256 hash
- do nothing if the file is already present and sound locally.
@ -141,6 +142,7 @@ The provided helper will:
The following aspects are not taken care of (yet! help welcome):
- report Telemetry when download fails
- check available disk space
- preserve bandwidth
- resume downloads of large files
@ -151,6 +153,10 @@ The provided helper will:
This will allow us to package attachments as part of a Firefox release (see `Bug 1542177 <https://bugzilla.mozilla.org/show_bug.cgi?id=1542177>`_)
and return them to calling code as ``resource://`` from within a package archive.
.. notes::
A ``downloadAsBytes()`` method returning an ``ArrayBuffer`` is also available, if writing the attachment into the user profile is not necessary.
.. _services/initial-data:

Просмотреть файл

@ -60,7 +60,7 @@ class Downloader {
async download(record, options = {}) {
const { retries = 3 } = options;
const {
attachment: { location, filename, hash, size },
attachment: { filename, size, hash },
} = record;
const localFilePath = OS.Path.join(
OS.Constants.Path.localProfileDir,
@ -72,7 +72,6 @@ class Downloader {
...this.folders,
filename,
].join("/")}`;
const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
await this._makeDirs();
@ -86,7 +85,54 @@ class Downloader {
throw new Downloader.BadContentError(localFilePath);
}
try {
await this._fetchAttachment(remoteFileUrl, localFilePath);
// Download and write on disk.
const buffer = await this.downloadAsBytes(record, {
checkHash: false, // Hash will be checked on file.
retries: 0, // Already in a retry loop.
});
await OS.File.writeAtomic(localFilePath, buffer, {
tmpPath: `${localFilePath}.tmp`,
});
} catch (e) {
if (retried >= retries) {
throw e;
}
}
retried++;
}
}
/**
* Download the record attachment and return its content as bytes.
*
* @param {Object} record A Remote Settings entry with attachment.
* @param {Object} options Some download options.
* @param {Number} options.retries Number of times download should be retried (default: `3`)
* @param {Number} options.checkHash Check content integrity (default: `true`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @returns {ArrayBuffer} the file content.
*/
async downloadAsBytes(record, options = {}) {
const {
attachment: { location, hash, size },
} = record;
const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
const { retries = 3, checkHash = true } = options;
let retried = 0;
while (true) {
try {
const buffer = await this._fetchAttachment(remoteFileUrl);
if (!checkHash) {
return buffer;
}
if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
return buffer;
}
// Content is corrupted.
throw new Downloader.BadContentError(location);
} catch (e) {
if (retried >= retries) {
throw e;
@ -131,17 +177,14 @@ class Downloader {
return this._cdnURL;
}
async _fetchAttachment(url, destination) {
async _fetchAttachment(url) {
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
const resp = await fetch(url, { headers });
if (!resp.ok) {
throw new Downloader.DownloadError(url, resp);
}
const buffer = await resp.arrayBuffer();
await OS.File.writeAtomic(destination, buffer, {
tmpPath: `${destination}.tmp`,
});
return resp.arrayBuffer();
}
async _makeDirs() {

Просмотреть файл

@ -86,14 +86,25 @@ const Agent = {
// File does not exist.
return false;
}
const buffer = await resp.arrayBuffer();
const bytes = new Uint8Array(buffer);
return this.checkContentHash(bytes, size, hash);
},
/**
* Check that the specified content matches the expected size and SHA-256 hash.
* @param {ArrayBuffer} buffer binary content
* @param {Number} size expected file size
* @param {String} size expected file SHA-256 as hex string
* @returns {boolean}
*/
async checkContentHash(buffer, size, hash) {
const bytes = new Uint8Array(buffer);
// Has expected size? (saves computing hash)
const fileSize = parseInt(resp.headers.get("Content-Length"), 10);
if (fileSize !== size) {
if (bytes.length !== size) {
return false;
}
// Has expected content?
const buffer = await resp.arrayBuffer();
const bytes = new Uint8Array(buffer);
const hashBuffer = await crypto.subtle.digest("SHA-256", bytes);
const hashBytes = new Uint8Array(hashBuffer);
const toHex = b => b.toString(16).padStart(2, "0");

Просмотреть файл

@ -115,6 +115,10 @@ class Worker {
async checkFileHash(filepath, size, hash) {
return this._execute("checkFileHash", [filepath, size, hash]);
}
async checkContentHash(buffer, size, hash) {
return this._execute("checkContentHash", [buffer, size, hash]);
}
}
// Now, first add a shutdown blocker. If that fails, we must have

Просмотреть файл

@ -92,6 +92,14 @@ add_task(async function test_download_writes_file_in_profile() {
});
add_task(clear_state);
add_task(async function test_download_as_bytes() {
const bytes = await downloader.downloadAsBytes(RECORD);
// See *.pem file in tests data.
Assert.ok(bytes.byteLength > 1500, `Wrong bytes size: ${bytes.byteLength}`);
});
add_task(clear_state);
add_task(async function test_file_is_redownloaded_if_size_does_not_match() {
const fileURL = await downloader.download(RECORD);
const localFilePath = pathFromURL(fileURL);
@ -134,9 +142,9 @@ add_task(async function test_download_is_retried_3_times_if_download_fails() {
let called = 0;
const _fetchAttachment = downloader._fetchAttachment;
downloader._fetchAttachment = (url, destination) => {
downloader._fetchAttachment = async url => {
called++;
return _fetchAttachment(url, destination);
return _fetchAttachment(url);
};
let error;
@ -159,7 +167,10 @@ add_task(async function test_download_is_retried_3_times_if_content_fails() {
},
};
let called = 0;
downloader._fetchAttachment = () => called++;
downloader._fetchAttachment = async () => {
called++;
return new ArrayBuffer();
};
let error;
try {