mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1666511 - Fetch changes from /changeset endpoint r=mythmon 

Differential Revision: https://phabricator.services.mozilla.com/D96203
This commit is contained in:
Mathieu Leplatre 2021-03-02 18:05:14 +00:00
Родитель 43ca6c351c
Коммит 01bfe89af8
3 изменённых файлов: 90 добавлений и 103 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

115
services/settings/Utils.jsm
Просмотреть файл

@ -64,7 +64,7 @@ var Utils = {
: gServerURL;
},
CHANGES_PATH: "/buckets/monitor/collections/changes/records",
CHANGES_PATH: "/buckets/monitor/collections/changes/changeset",
/**
* Logger instance.
@ -124,6 +124,21 @@ var Utils = {
/**
* Fetch the list of remote collections and their timestamp.
* ```
* {
* "timestamp": 1486545678,
* "changes":[
* {
* "host":"kinto-ota.dev.mozaws.net",
* "last_modified":1450717104423,
* "bucket":"blocklists",
* "collection":"certificates"
* },
* ...
* ],
* "metadata": {}
* }
* ```
* @param {String} serverUrl The server URL (eg. `https://server.org/v1`)
* @param {int} expectedTimestamp The timestamp that the server is supposed to return.
* We obtained it from the Megaphone notification payload,
@ -135,29 +150,14 @@ var Utils = {
async fetchLatestChanges(serverUrl, options = {}) {
const { expectedTimestamp, lastEtag = "", filters = {} } = options;
//
// Fetch the list of changes objects from the server that looks like:
// {"data":[
// {
// "host":"kinto-ota.dev.mozaws.net",
// "last_modified":1450717104423,
// "bucket":"blocklists",
// "collection":"certificates"
// }]}
let url = serverUrl + Utils.CHANGES_PATH;
// Use ETag to obtain a `304 Not modified` when no change occurred,
// and `?_since` parameter to only keep entries that weren't processed yet.
const headers = {};
const params = { ...filters };
const params = {
...filters,
_expected: expectedTimestamp ?? 0,
};
if (lastEtag != "") {
headers["If-None-Match"] = lastEtag;
params._since = lastEtag;
}
if (expectedTimestamp) {
params._expected = expectedTimestamp;
}
if (params) {
url +=
"?" +
@ -165,52 +165,43 @@ var Utils = {
.map(([k, v]) => `${k}=${encodeURIComponent(v)}`)
.join("&");
}
const response = await fetch(url, { headers });
const response = await fetch(url);
let changes = [];
// If no changes since last time, go on with empty list of changes.
if (response.status != 304) {
if (response.status >= 500) {
if (response.status >= 500) {
throw new Error(`Server error ${response.status} ${response.statusText}`);
}
const is404FromCustomServer =
response.status == 404 &&
Services.prefs.prefHasUserValue("services.settings.server");
const ct = response.headers.get("Content-Type");
if (!is404FromCustomServer && (!ct || !ct.includes("application/json"))) {
throw new Error(`Unexpected content-type "${ct}"`);
}
let payload;
try {
payload = await response.json();
} catch (e) {
payload = e.message;
}
if (!payload.hasOwnProperty("changes")) {
// If the server is failing, the JSON response might not contain the
// expected data. For example, real server errors (Bug 1259145)
// or dummy local server for tests (Bug 1481348)
if (!is404FromCustomServer) {
throw new Error(
`Server error ${response.status} ${response.statusText}`
`Server error ${url} ${response.status} ${
response.statusText
}: ${JSON.stringify(payload)}`
);
}
const is404FromCustomServer =
response.status == 404 &&
Services.prefs.prefHasUserValue("services.settings.server");
const ct = response.headers.get("Content-Type");
if (!is404FromCustomServer && (!ct || !ct.includes("application/json"))) {
throw new Error(`Unexpected content-type "${ct}"`);
}
let payload;
try {
payload = await response.json();
} catch (e) {
payload = e.message;
}
if (!payload.hasOwnProperty("data")) {
// If the server is failing, the JSON response might not contain the
// expected data. For example, real server errors (Bug 1259145)
// or dummy local server for tests (Bug 1481348)
if (!is404FromCustomServer) {
throw new Error(
`Server error ${response.status} ${
response.statusText
}: ${JSON.stringify(payload)}`
);
}
} else {
changes = payload.data;
}
}
// The server should always return ETag. But we've had situations where the CDN
// was interfering.
const currentEtag = response.headers.has("ETag")
? response.headers.get("ETag")
: undefined;
const { changes = [], timestamp } = payload;
let serverTimeMillis = Date.parse(response.headers.get("Date"));
// Since the response is served via a CDN, the Date header value could have been cached.
const cacheAgeSeconds = response.headers.has("Age")
@ -233,7 +224,7 @@ var Utils = {
return {
changes,
currentEtag,
currentEtag: `"${timestamp}"`,
serverTimeMillis,
backoffSeconds,
ageSeconds,

14
services/settings/test/unit/test_remote_settings.js
Просмотреть файл

@ -74,7 +74,7 @@ function run_test() {
server.registerPathHandler("/v1/", handleResponse);
server.registerPathHandler(
"/v1/buckets/monitor/collections/changes/records",
"/v1/buckets/monitor/collections/changes/changeset",
handleResponse
);
server.registerPathHandler(
@ -1069,7 +1069,7 @@ function getSampleResponse(req, port) {
hello: "kinto",
},
},
"GET:/v1/buckets/monitor/collections/changes/records": {
"GET:/v1/buckets/monitor/collections/changes/changeset": {
sampleHeaders: [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
@ -1080,7 +1080,8 @@ function getSampleResponse(req, port) {
],
status: { status: 200, statusText: "OK" },
responseBody: {
data: [
timestamp: 5000,
changes: [
{
id: "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
bucket: "main",
@ -1263,7 +1264,7 @@ wNuvFqc=
error: "Service Unavailable",
},
},
"GET:/v1/buckets/monitor/collections/changes/records?collection=password-fields&bucket=main": {
"GET:/v1/buckets/monitor/collections/changes/changeset?collection=password-fields&bucket=main&_expected=0": {
sampleHeaders: [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
@ -1274,7 +1275,8 @@ wNuvFqc=
],
status: { status: 200, statusText: "OK" },
responseBody: {
data: [
timestamp: 1338,
changes: [
{
id: "fe5758d0-c67a-42d0-bb4f-8f2d75106b65",
bucket: "main",
@ -1398,7 +1400,7 @@ wNuvFqc=
],
},
},
"GET:/v1/buckets/monitor/collections/changes/records?collection=no-mocked-responses&bucket=main": {
"GET:/v1/buckets/monitor/collections/changes/changeset?collection=no-mocked-responses&bucket=main&_expected=0": {
sampleHeaders: [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",

64
services/settings/test/unit/test_remote_settings_poll.js
Просмотреть файл

@ -61,12 +61,12 @@ function serveChangesEntries(serverTime, entries) {
response.setStatusLine(null, 200, "OK");
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", new Date(serverTime).toUTCString());
const latest = entries[0]?.last_modified ?? 42;
if (entries.length) {
const latest = entries[0].last_modified;
response.setHeader("ETag", `"${latest}"`);
response.setHeader("Last-Modified", new Date(latest).toGMTString());
}
response.write(JSON.stringify({ data: entries }));
response.write(JSON.stringify({ timestamp: latest, changes: entries }));
};
}
@ -86,7 +86,7 @@ add_task(clear_state);
add_task(async function test_an_event_is_sent_on_start() {
server.registerPathHandler(CHANGES_PATH, (request, response) => {
response.write(JSON.stringify({ data: [] }));
response.write(JSON.stringify({ timestamp: 42, changes: [] }));
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("ETag", '"42"');
response.setHeader("Date", new Date().toUTCString());
@ -263,16 +263,7 @@ add_task(async function test_check_up_to_date() {
);
const serverTime = 4000;
function server304(request, response) {
if (
request.hasHeader("if-none-match") &&
request.getHeader("if-none-match") == '"1100"'
) {
response.setHeader("Date", new Date(serverTime).toUTCString());
response.setStatusLine(null, 304, "Service Not Modified");
}
}
server.registerPathHandler(CHANGES_PATH, server304);
server.registerPathHandler(CHANGES_PATH, serveChangesEntries(serverTime, []));
Services.prefs.setCharPref(PREF_LAST_ETAG, '"1100"');
@ -286,7 +277,7 @@ add_task(async function test_check_up_to_date() {
};
Services.obs.addObserver(observer, "remote-settings:changes-poll-end");
// If server has no change, a 304 is received, maybeSync() is not called.
// If server has no change, maybeSync() is not called.
let maybeSyncCalled = false;
const c = RemoteSettings("test-collection", {
bucketName: "test-bucket",
@ -322,10 +313,13 @@ add_task(async function test_expected_timestamp() {
collection: "with-cache-busting",
},
];
if (request.queryString == `_expected=${encodeURIComponent('"42"')}`) {
if (
request.queryString.includes(`_expected=${encodeURIComponent('"42"')}`)
) {
response.write(
JSON.stringify({
data: entries,
timestamp: 1110,
changes: entries,
})
);
}
@ -352,7 +346,8 @@ add_task(async function test_client_last_check_is_saved() {
server.registerPathHandler(CHANGES_PATH, (request, response) => {
response.write(
JSON.stringify({
data: [
timestamp: 42,
changes: [
{
id: "695c2407-de79-4408-91c7-70720dd59d78",
last_modified: 1100,
@ -512,20 +507,20 @@ add_task(async function test_success_with_partial_list() {
collection: "poll-test-collection",
},
];
if (request.queryString == `_since=${encodeURIComponent('"42"')}`) {
if (request.queryString.includes(`_since=${encodeURIComponent('"42"')}`)) {
response.write(
JSON.stringify({
data: entries.slice(0, 1),
timestamp: 43,
changes: entries.slice(0, 1),
})
);
response.setHeader("ETag", '"43"');
} else {
response.write(
JSON.stringify({
data: entries,
timestamp: 42,
changes: entries,
})
);
response.setHeader("ETag", '"42"');
}
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", new Date().toUTCString());
@ -785,7 +780,7 @@ add_task(async function test_check_clockskew_is_updated() {
function serverResponse(request, response) {
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", new Date(serverTime).toUTCString());
response.write(JSON.stringify({ data: [] }));
response.write(JSON.stringify({ timestamp: 42, changes: [] }));
response.setStatusLine(null, 200, "OK");
}
server.registerPathHandler(CHANGES_PATH, serverResponse);
@ -828,7 +823,7 @@ add_task(async function test_check_clockskew_takes_age_into_account() {
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", new Date(serverTime).toUTCString());
response.setHeader("Age", `${ageCDNSeconds}`);
response.write(JSON.stringify({ data: [] }));
response.write(JSON.stringify({ timestamp: 42, changes: [] }));
response.setStatusLine(null, 200, "OK");
}
server.registerPathHandler(CHANGES_PATH, serverResponse);
@ -848,7 +843,7 @@ add_task(async function test_backoff() {
function simulateBackoffResponse(request, response) {
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Backoff", "10");
response.write(JSON.stringify({ data: [] }));
response.write(JSON.stringify({ timestamp: 42, changes: [] }));
response.setStatusLine(null, 200, "OK");
}
server.registerPathHandler(CHANGES_PATH, simulateBackoffResponse);
@ -1023,7 +1018,7 @@ add_task(async function test_syncs_clients_with_local_dump() {
add_task(clear_state);
add_task(async function test_adding_client_resets_polling() {
function serve200or304(request, response) {
function serve200(request, response) {
const entries = [
{
id: "aa71e6cc-9f37-447a-b6e0-c025e8eabd03",
@ -1033,27 +1028,26 @@ add_task(async function test_adding_client_resets_polling() {
collection: "a-collection",
},
];
if (request.queryString == `_since=${encodeURIComponent('"42"')}`) {
if (request.queryString.includes("_since")) {
response.write(
JSON.stringify({
data: entries.slice(0, 1),
timestamp: 42,
changes: [],
})
);
response.setHeader("ETag", '"42"');
response.setStatusLine(null, 304, "Not Modified");
} else {
response.write(
JSON.stringify({
data: entries,
timestamp: 42,
changes: entries,
})
);
response.setHeader("ETag", '"42"');
response.setStatusLine(null, 200, "OK");
}
response.setStatusLine(null, 200, "OK");
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", new Date().toUTCString());
}
server.registerPathHandler(CHANGES_PATH, serve200or304);
server.registerPathHandler(CHANGES_PATH, serve200);
// Poll once, without any client for "a-collection"
await RemoteSettings.pollChanges();
@ -1095,7 +1089,7 @@ add_task(
];
response.write(
JSON.stringify({
data: entries,
changes: entries,
})
);
response.setHeader("ETag", '"42"');