Backed out changeset 7e1942ace2b2 (bug 1501214) for failing test_attachments_downloader.js on a CLOSED TREE

This commit is contained in:
Andreea Pavel 2019-05-08 21:12:06 +03:00
Родитель eec25e57b2
Коммит 67007aabfa
9 изменённых файлов: 14 добавлений и 422 удалений

Просмотреть файл

@ -57,10 +57,10 @@ Options
.. code-block:: js
const subset = await RemoteSettings("a-key").get({
filters: {
property: "value"
},
order: "-weight"
filters: {
"property": "value"
},
order: "-weight"
});
* ``syncIfEmpty``: implicit synchronization if local data is empty (default: ``true``).
@ -98,56 +98,19 @@ The ``sync`` event allows to be notified when the remote settings are changed on
File attachments
----------------
When an entry has a file attached to it, it has an ``attachment`` attribute, which contains the file related information (url, hash, size, mimetype, etc.).
Remote files are not downloaded automatically. In order to keep attachments in sync, the provided helper can be leveraged like this:
When an entry has a file attached to it, it has an ``attachment`` attribute, which contains the file related information (url, hash, size, mimetype, etc.). Remote files are not downloaded automatically.
.. code-block:: js
const client = RemoteSettings("a-key");
client.on("sync", async ({ data: { created, updated, deleted } }) => {
const toDelete = deleted.filter(d => d.attachment);
const toDownload = created
.concat(updated.map(u => u.new))
.filter(d => d.attachment);
// Remove local files of deleted records
await Promise.all(toDelete.map(entry => client.attachments.delete(entry)));
// Download attachments
const fileURLs = await Promise.all(
toDownload.map(entry => client.attachments.download(entry, { retries: 2 }))
);
// Open downloaded files...
const fileContents = await Promise.all(
fileURLs.map(async url => {
const r = await fetch(url);
return r.blob();
})
);
});
The provided helper will:
- fetch the remote binary content
- check the file size
- check the content SHA256 hash
- do nothing if the file is already present and sound locally.
.. important::
The following aspects are not taken care of (yet! help welcome):
- check available disk space
- preserve bandwidth
- resume downloads of large files
.. notes::
The ``download()`` method does not return a file path but instead a ``file://`` URL which points to the locally-downloaded file.
This will allow us to package attachments as part of a Firefox release (see `Bug 1542177 <https://bugzilla.mozilla.org/show_bug.cgi?id=1542177>`_)
and return them to calling code as ``resource://`` from within a package archive.
const data = await RemoteSettings("a-key").get();
data.filter(d => d.attachment)
.forEach(async ({ attachment: { url, filename, size } }) => {
if (size < OS.freeDiskSpace) {
// Planned feature, see Bug 1501214
await downloadLocally(url, filename);
}
});
.. _services/initial-data:
@ -370,3 +333,4 @@ Then, in order to access a specific client instance, the bucket must be specifie
const collection = await RemoteSettings("addons", { bucketName: "blocklists" }).openCollection();
And in the storage inspector, the IndexedDB internal store will be prefixed with ``blocklists`` instead of ``main`` (eg. ``blocklists/addons``).

Просмотреть файл

@ -1,130 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
var EXPORTED_SYMBOLS = [
"Downloader",
];
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { XPCOMUtils } = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
ChromeUtils.defineModuleGetter(this, "RemoteSettingsWorker",
"resource://services-settings/RemoteSettingsWorker.jsm");
ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
class DownloadError extends Error {
constructor(url, resp) {
super(`Could not download ${url}`);
this.name = "DownloadError";
this.resp = resp;
}
}
class BadContentError extends Error {
constructor(path) {
super(`${path} content does not match server hash`);
this.name = "BadContentError";
}
}
class Downloader {
static get DownloadError() { return DownloadError; }
static get BadContentError() { return BadContentError; }
constructor(...folders) {
this.folders = ["settings", ...folders];
this._cdnURL = null;
}
/**
* Download the record attachment into the local profile directory
* and return a file:// URL that points to the local path.
*
* No-op if the file was already downloaded and not corrupted.
*
* @param {Object} record A Remote Settings entry with attachment.
* @param {Object} options Some download options.
* @param {Number} options.retries Number of times download should be retried (default: `3`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded file integrity is not valid.
* @returns {String} the absolute file path to the downloaded attachment.
*/
async download(record, options = {}) {
const { retries = 3 } = options;
const { attachment: { location, filename, hash, size } } = record;
const localFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, ...this.folders, filename);
const localFileUrl = `file://${localFilePath}`;
const remoteFileUrl = (await this._baseAttachmentsURL()) + location;
await this._makeDirs();
let retried = 0;
while (true) {
if (await RemoteSettingsWorker.checkFileHash(localFileUrl, size, hash)) {
return localFileUrl;
}
// File does not exist or is corrupted.
if (retried > retries) {
throw new Downloader.BadContentError(localFilePath);
}
try {
await this._fetchAttachment(remoteFileUrl, localFilePath);
} catch (e) {
if (retried >= retries) {
throw e;
}
}
retried++;
}
}
/**
* Delete the record attachment downloaded locally.
* No-op if the related file does not exist.
*
* @param record A Remote Settings entry with attachment.
*/
async delete(record) {
const { attachment: { filename } } = record;
const path = OS.Path.join(OS.Constants.Path.localProfileDir, ...this.folders, filename);
await OS.File.remove(path, { ignoreAbsent: true });
await this._rmDirs();
}
async _baseAttachmentsURL() {
if (!this._cdnURL) {
const server = Services.prefs.getCharPref("services.settings.server");
const serverInfo = await (await fetch(`${server}/`)).json();
// Server capabilities expose attachments configuration.
const { capabilities: { attachments: { base_url } } } = serverInfo;
// Make sure the URL always has a trailing slash.
this._cdnURL = base_url + (base_url.endsWith("/") ? "" : "/");
}
return this._cdnURL;
}
async _fetchAttachment(url, destination) {
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
const resp = await fetch(url, { headers });
if (!resp.ok) {
throw new Downloader.DownloadError(url, resp);
}
const buffer = await resp.arrayBuffer();
await OS.File.writeAtomic(destination, buffer, { tmpPath: `${destination}.tmp` });
}
async _makeDirs() {
const dirPath = OS.Path.join(OS.Constants.Path.localProfileDir, ...this.folders);
await OS.File.makeDir(dirPath, { from: OS.Constants.Path.localProfileDir });
}
async _rmDirs() {
for (let i = this.folders.length; i > 0; i--) {
const dirPath = OS.Path.join(OS.Constants.Path.localProfileDir, ...this.folders.slice(0, i));
await OS.File.removeEmptyDir(dirPath, { ignoreAbsent: true });
}
}
}

Просмотреть файл

@ -23,8 +23,6 @@ ChromeUtils.defineModuleGetter(this, "RemoteSettingsWorker",
"resource://services-settings/RemoteSettingsWorker.jsm");
ChromeUtils.defineModuleGetter(this, "Utils",
"resource://services-settings/Utils.jsm");
ChromeUtils.defineModuleGetter(this, "Downloader",
"resource://services-settings/Attachments.jsm");
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
@ -203,8 +201,6 @@ class RemoteSettingsClient extends EventEmitter {
adapter: Kinto.adapters.IDB,
adapterOptions: { dbName: DB_NAME, migrateOldData: false },
}));
XPCOMUtils.defineLazyGetter(this, "attachments", () => new Downloader(this.bucketName, collectionName));
}
get identifier() {

Просмотреть файл

@ -68,36 +68,6 @@ const Agent = {
await importDumpIDB(bucket, collection, records);
return records.length;
},
/**
* Check that the specified file matches the expected size and SHA-256 hash.
* @param {String} fileUrl file URL to read from
* @param {Number} size expected file size
* @param {String} size expected file SHA-256 as hex string
* @returns {boolean}
*/
async checkFileHash(fileUrl, size, hash) {
let resp;
try {
resp = await fetch(fileUrl);
} catch (e) {
// File does not exist.
return false;
}
// Has expected size? (saves computing hash)
const fileSize = parseInt(resp.headers.get("Content-Length"), 10);
if (fileSize !== size) {
return false;
}
// Has expected content?
const buffer = await resp.arrayBuffer();
const bytes = new Uint8Array(buffer);
const hashBuffer = await crypto.subtle.digest("SHA-256", bytes);
const hashBytes = new Uint8Array(hashBuffer);
const toHex = b => b.toString(16).padStart(2, "0");
const hashStr = Array.from(hashBytes, toHex).join("");
return hashStr == hash;
},
};
/**

Просмотреть файл

@ -47,10 +47,6 @@ class Worker {
async importJSONDump(bucket, collection) {
return this._execute("importJSONDump", [bucket, collection]);
}
async checkFileHash(filepath, size, hash) {
return this._execute("checkFileHash", [filepath, size, hash]);
}
}
var RemoteSettingsWorker = new Worker("resource://services-settings/RemoteSettingsWorker.js");

Просмотреть файл

@ -14,7 +14,6 @@ EXTRA_COMPONENTS += [
]
EXTRA_JS_MODULES['services-settings'] += [
'Attachments.jsm',
'remote-settings.js',
'RemoteSettingsClient.jsm',
'RemoteSettingsComponents.jsm',

Просмотреть файл

@ -1,175 +0,0 @@
/* import-globals-from ../../../common/tests/unit/head_helpers.js */
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js");
const { Downloader } = ChromeUtils.import("resource://services-settings/Attachments.jsm");
const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm");
const RECORD = {
attachment: {
hash: "f41ed47d0f43325c9f089d03415c972ce1d3f1ecab6e4d6260665baf3db3ccee",
size: 1597,
filename: "test_file.pem",
location: "main-workspace/some-collection/65650a0f-7c22-4c10-9744-2d67e301f5f4.pem",
mimetype: "application/x-pem-file",
},
};
let downloader;
let server;
function pathFromURL(url) {
const uri = Services.io.newURI(url);
const file = uri.QueryInterface(Ci.nsIFileURL).file;
return file.path;
}
function run_test() {
server = new HttpServer();
server.start(-1);
registerCleanupFunction(() => server.stop(() => {}));
server.registerDirectory("/cdn/main-workspace/some-collection/", do_get_file("test_attachments_downloader"));
server.registerPathHandler("/v1/", (request, response) => {
response.write(JSON.stringify({
capabilities: {
attachments: {
base_url: `http://localhost:${server.identity.primaryPort}/cdn/`,
},
},
}));
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setStatusLine(null, 200, "OK");
});
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
run_next_test();
}
async function clear_state() {
downloader = new Downloader("main", "some-collection");
await downloader.delete(RECORD);
}
add_task(clear_state);
add_task(async function test_download_writes_file_in_profile() {
const fileURL = await downloader.download(RECORD);
const localFilePath = pathFromURL(fileURL);
Assert.equal(fileURL, "file://" + OS.Path.join(OS.Constants.Path.localProfileDir,
"settings/main/some-collection/test_file.pem"));
Assert.ok(await OS.File.exists(localFilePath));
const stat = await OS.File.stat(localFilePath);
Assert.equal(stat.size, 1597);
});
add_task(clear_state);
add_task(async function test_file_is_redownloaded_if_size_does_not_match() {
const fileURL = await downloader.download(RECORD);
const localFilePath = pathFromURL(fileURL);
await OS.File.writeAtomic(localFilePath, "bad-content", { encoding: "utf-8" });
let stat = await OS.File.stat(localFilePath);
Assert.notEqual(stat.size, 1597);
await downloader.download(RECORD);
stat = await OS.File.stat(localFilePath);
Assert.equal(stat.size, 1597);
});
add_task(clear_state);
add_task(async function test_file_is_redownloaded_if_corrupted() {
const fileURL = await downloader.download(RECORD);
const localFilePath = pathFromURL(fileURL);
const byteArray = await OS.File.read(localFilePath);
byteArray[0] = 42;
await OS.File.writeAtomic(localFilePath, byteArray);
let content = await OS.File.read(localFilePath, { encoding: "utf-8" });
Assert.notEqual(content.slice(0, 5), "-----");
await downloader.download(RECORD);
content = await OS.File.read(localFilePath, { encoding: "utf-8" });
Assert.equal(content.slice(0, 5), "-----");
});
add_task(clear_state);
add_task(async function test_download_is_retried_3_times_if_download_fails() {
const record = {
attachment: {
...RECORD.attachment,
location: "404-error.pem",
},
};
let called = 0;
const _fetchAttachment = downloader._fetchAttachment;
downloader._fetchAttachment = (url, destination) => {
called++;
return _fetchAttachment(url, destination);
};
let error;
try {
await downloader.download(record);
} catch (e) {
error = e;
}
Assert.equal(called, 4); // 1 + 3 retries
Assert.ok(error instanceof Downloader.DownloadError);
});
add_task(clear_state);
add_task(async function test_download_is_retried_3_times_if_content_fails() {
const record = {
attachment: {
...RECORD.attachment,
hash: "always-wrong",
},
};
let called = 0;
downloader._fetchAttachment = () => called++;
let error;
try {
await downloader.download(record);
} catch (e) {
error = e;
}
Assert.equal(called, 4); // 1 + 3 retries
Assert.ok(error instanceof Downloader.BadContentError);
});
add_task(clear_state);
add_task(async function test_delete_removes_local_file() {
const fileURL = await downloader.download(RECORD);
const localFilePath = pathFromURL(fileURL);
Assert.ok(await OS.File.exists(localFilePath));
downloader.delete(RECORD);
Assert.ok(!await OS.File.exists(localFilePath));
Assert.ok(!await OS.File.exists(downloader.baseFolder));
});
add_task(clear_state);
add_task(async function test_downloader_is_accessible_via_client() {
const client = RemoteSettings("some-collection");
const fileURL = await client.attachments.download(RECORD);
Assert.equal(fileURL, "file://" + OS.Path.join(
OS.Constants.Path.localProfileDir,
"settings",
client.bucketName,
client.collectionName,
RECORD.attachment.filename
));
});
add_task(clear_state);

Просмотреть файл

@ -1,26 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIEbjCCA1agAwIBAgIQBg3WwdBnkBtUdfz/wp4xNzANBgkqhkiG9w0BAQsFADBa
MQswCQYDVQQGEwJJRTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJl
clRydXN0MSIwIAYDVQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTE1
MTAxNDEyMDAwMFoXDTIwMTAxNDEyMDAwMFowbzELMAkGA1UEBhMCVVMxCzAJBgNV
BAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRkwFwYDVQQKExBDbG91ZEZs
YXJlLCBJbmMuMSAwHgYDVQQDExdDbG91ZEZsYXJlIEluYyBSU0EgQ0EtMTCCASIw
DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJGiNOIE4s0M4wdhDeV9aMfAYY9l
yG9cfGQqt7a5UgrRA81bi4istCyhzfzRWUW+NAmf6X2HEnA3xLI1M+pH/xEbk9pw
jc8/1CPy9jUjBwb89zt5PWh2I1KxZVg/Bnx2yYdVcKTUMKt0GLDXfZXN+RYZHJQo
lDlzjH5xV0IpDMv/FsMEZWcfx1JorBf08bRnRVkl9RY00y2ujVr+492ze+zYQ9s7
HcidpR+7ret3jzLSvojsaA5+fOaCG0ctVJcLfnkQ5lWR95ByBdO1NapfqZ1+kmCL
3baVSeUpYQriBwznxfLuGs8POo4QdviYVtSPBWjOEfb+o1c6Mbo8p4noFzUCAwEA
AaOCARkwggEVMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDQG
CCsGAQUFBwEBBCgwJjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu
Y29tMDoGA1UdHwQzMDEwL6AtoCuGKWh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9P
bW5pcm9vdDIwMjUuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIB
FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBSRBYrfTCLG
bYuUTBZFfu5vAvu3wDAfBgNVHSMEGDAWgBTlnVkwgkdYzKz6CFQ2hns6tQRN8DAN
BgkqhkiG9w0BAQsFAAOCAQEAVJle3ar9NSnTrLAhgfkcpClIY6/kabDIEa8cOnu1
SOXf4vbtZakSmmIbFbmYDUGIU5XwwVdF/FKNzNBRf9G4EL/S0NXytBKj4A34UGQA
InaV+DgVLzCifN9cAHi8EFEAfbglUvPvLPFXF0bwffElYm7QBSiHYSZmfOKLCyiv
3zlQsf7ozNBAxfbmnRMRSUBcIhRwnaFoFgDs7yU6R1Yk4pO7eMgWpdPGhymDTIvv
RnauKStzKsAli9i5hQ4nTDITUpMAmeJoXodgwRkC3Civw32UR2rxObIyxPpbfODb
sZKNGO9K5Sjj6turB1zwbd2wI8MhtUCY9tGmSYhe7G6Bkw==
-----END CERTIFICATE-----

Просмотреть файл

@ -3,8 +3,6 @@ head = ../../../common/tests/unit/head_global.js ../../../common/tests/unit/head
firefox-appdir = browser
tags = remote-settings
[test_attachments_downloader.js]
support-files = test_attachments_downloader/**
[test_remote_settings.js]
[test_remote_settings_poll.js]
[test_remote_settings_worker.js]