diff --git a/services/common/kinto-http-client.js b/services/common/kinto-http-client.js index deda29e80d24..d03559237073 100644 --- a/services/common/kinto-http-client.js +++ b/services/common/kinto-http-client.js @@ -2500,7 +2500,7 @@ XPCOMUtils.defineLazyGlobalGetters(global, ["fetch"]); { headers: options.headers ? options.headers : {}, path: path + "?" + querystring, - }, + }, // N.B. This doesn't use _getRetry, because all calls to // `paginatedList` are assumed to come from calls that already // used `_getRetry` at e.g. the bucket or collection level. diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm index 74ba0d9efa3c..138ef470a84c 100644 --- a/services/settings/RemoteSettingsClient.jsm +++ b/services/settings/RemoteSettingsClient.jsm @@ -765,24 +765,17 @@ class RemoteSettingsClient extends EventEmitter { options = {} ) { const { retry = false } = options; + const since = retry || !localTimestamp ? undefined : `${localTimestamp}`; - // Fetch collection metadata and list of changes from server - // (or all records on retry). - const client = this.httpClient(); - const [ + // Fetch collection metadata and list of changes from server. + console.debug( + `Fetch changes from server (expected=${expectedTimestamp}, since=${since})` + ); + const { metadata, - { data: remoteRecords, last_modified: remoteTimestamp }, - ] = await Promise.all([ - client.getData({ - query: { _expected: expectedTimestamp }, - }), - client.listRecords({ - filters: { - _expected: expectedTimestamp, - }, - since: retry || !localTimestamp ? undefined : `${localTimestamp}`, - }), - ]); + remoteTimestamp, + remoteRecords, + } = await this._fetchChangeset(expectedTimestamp, since); // We build a sync result, based on remote changes. const syncResult = { @@ -893,6 +886,36 @@ class RemoteSettingsClient extends EventEmitter { return syncResult; } + /** + * Fetch information from changeset endpoint. + * + * @param expectedTimestamp cache busting value + * @param since timestamp of last sync (optional) + */ + async _fetchChangeset(expectedTimestamp, since) { + const client = this.httpClient(); + const { + metadata, + timestamp: remoteTimestamp, + changes: remoteRecords, + } = await client.execute( + { + path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`, + }, + { + query: { + _expected: expectedTimestamp, + _since: since, + }, + } + ); + return { + remoteTimestamp, + metadata, + remoteRecords, + }; + } + /** * Use the filter func to filter the lists of changes obtained from synchronization, * and return them along with the filtered list of local records. diff --git a/services/settings/test/unit/test_remote_settings.js b/services/settings/test/unit/test_remote_settings.js index 2dec180e6170..0c6c94af16d9 100644 --- a/services/settings/test/unit/test_remote_settings.js +++ b/services/settings/test/unit/test_remote_settings.js @@ -69,11 +69,7 @@ function run_test() { handleResponse ); server.registerPathHandler( - "/v1/buckets/main/collections/password-fields", - handleResponse - ); - server.registerPathHandler( - "/v1/buckets/main/collections/password-fields/records", + "/v1/buckets/main/collections/password-fields/changeset", handleResponse ); server.registerPathHandler( @@ -81,15 +77,11 @@ function run_test() { handleResponse ); server.registerPathHandler( - "/v1/buckets/main/collections/language-dictionaries/records", + "/v1/buckets/main/collections/language-dictionaries/changeset", handleResponse ); server.registerPathHandler( - "/v1/buckets/main/collections/with-local-fields", - handleResponse - ); - server.registerPathHandler( - "/v1/buckets/main/collections/with-local-fields/records", + "/v1/buckets/main/collections/with-local-fields/changeset", handleResponse ); server.registerPathHandler("/fake-x5u", handleResponse); @@ -910,26 +902,6 @@ function getSampleResponse(req, port) { ], }, }, - "GET:/v1/buckets/main/collections/password-fields": { - sampleHeaders: [ - "Access-Control-Allow-Origin: *", - "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", - "Content-Type: application/json; charset=UTF-8", - "Server: waitress", - 'Etag: "1234"', - ], - status: { status: 200, statusText: "OK" }, - responseBody: JSON.stringify({ - data: { - id: "password-fields", - last_modified: 1234, - signature: { - signature: "abcdef", - x5u: `http://localhost:${port}/fake-x5u`, - }, - }, - }), - }, "GET:/fake-x5u": { sampleHeaders: ["Content-Type: application/octet-stream"], status: { status: 200, statusText: "OK" }, @@ -940,7 +912,7 @@ ZARKjbu1TuYQHf0fs+GwID8zeLc2zJL7UzcHFwwQ6Nda9OJN4uPAuC/BKaIpxCLL wNuvFqc= -----END CERTIFICATE-----`, }, - "GET:/v1/buckets/main/collections/password-fields/records?_expected=2000&_sort=-last_modified": { + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=2000": { sampleHeaders: [ "Access-Control-Allow-Origin: *", "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", @@ -950,7 +922,16 @@ wNuvFqc= ], status: { status: 200, statusText: "OK" }, responseBody: { - data: [ + timestamp: 3000, + metadata: { + id: "password-fields", + last_modified: 1234, + signature: { + signature: "abcdef", + x5u: `http://localhost:${port}/fake-x5u`, + }, + }, + changes: [ { id: "9d500963-d80e-3a91-6e74-66f3811b99cc", last_modified: 3000, @@ -960,7 +941,7 @@ wNuvFqc= ], }, }, - "GET:/v1/buckets/main/collections/password-fields/records?_expected=3001&_sort=-last_modified&_since=3000": { + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=3001&_since=3000": { sampleHeaders: [ "Access-Control-Allow-Origin: *", "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", @@ -970,7 +951,9 @@ wNuvFqc= ], status: { status: 200, statusText: "OK" }, responseBody: { - data: [ + metadata: {}, + timestamp: 4000, + changes: [ { id: "aabad965-e556-ffe7-4191-074f5dee3df3", last_modified: 4000, @@ -986,7 +969,7 @@ wNuvFqc= ], }, }, - "GET:/v1/buckets/main/collections/password-fields/records?_expected=4001&_sort=-last_modified&_since=4000": { + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=4001&_since=4000": { sampleHeaders: [ "Access-Control-Allow-Origin: *", "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", @@ -996,7 +979,9 @@ wNuvFqc= ], status: { status: 200, statusText: "OK" }, responseBody: { - data: [ + metadata: {}, + timestamp: 5000, + changes: [ { id: "aabad965-e556-ffe7-4191-074f5dee3df3", deleted: true, @@ -1004,7 +989,7 @@ wNuvFqc= ], }, }, - "GET:/v1/buckets/main/collections/password-fields/records?_expected=10000&_sort=-last_modified&_since=9999": { + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=10000&_since=9999": { sampleHeaders: [ "Access-Control-Allow-Origin: *", "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", @@ -1018,7 +1003,7 @@ wNuvFqc= error: "Service Unavailable", }, }, - "GET:/v1/buckets/main/collections/password-fields/records?_expected=10001&_sort=-last_modified&_since=10000": { + "GET:/v1/buckets/main/collections/password-fields/changeset?_expected=10001&_since=10000": { sampleHeaders: [ "Access-Control-Allow-Origin: *", "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", @@ -1029,7 +1014,7 @@ wNuvFqc= status: { status: 200, statusText: "OK" }, responseBody: " [RECORD2, RECORD3] @@ -484,38 +446,42 @@ add_task(async function test_check_synchronization_with_signatures() { 'ETag: "4000"', ], status: { status: 200, statusText: "OK" }, - responseBody: JSON.stringify({ data: [RECORD2, RECORD3] }), + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: THREE_ITEMS_SIG, + }, + }, + changes: [RECORD2, RECORD3], + }), }; - // Prepare a (deliberately) bad signature to check the collection state is - // reset if something is inconsistent - const RESPONSE_BODY_META_BAD_SIG = makeMetaResponseBody( - 4000, - "aW52YWxpZCBzaWduYXR1cmUK" - ); - const RESPONSE_META_BAD_SIG = makeMetaResponse( - 4000, - RESPONSE_BODY_META_BAD_SIG, - "RESPONSE_META_BAD_SIG" - ); + const RESPONSE_EMPTY_NO_UPDATE_BAD_SIG = { + ...RESPONSE_EMPTY_NO_UPDATE, + responseBody: JSON.stringify({ + timestamp: 4000, + metadata: { + signature: { + x5u, + signature: "aW52YWxpZCBzaWduYXR1cmUK", + }, + }, + changes: [], + }), + }; const badSigGoodSigResponses = { - // In this test, we deliberately serve a bad signature initially. The - // subsequent signature returned is a valid one for the three item - // collection. - "GET:/v1/buckets/main/collections/signed?_expected=5000": [ - RESPONSE_META_BAD_SIG, - RESPONSE_META_THREE_ITEMS_SIG, - ], // The first collection state is the three item collection (since - // there's a sync with no updates) - but, since the signature is wrong, + // there was sync with no updates before) - but, since the signature is wrong, // another request will be made... - "GET:/v1/buckets/main/collections/signed/records?_expected=5000&_sort=-last_modified&_since=4000": [ - RESPONSE_EMPTY_NO_UPDATE, + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=4000": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG, ], - // The next request is for the full collection. This will be checked against the valid signature - // - so the sync should succeed. - "GET:/v1/buckets/main/collections/signed/records?_expected=5000&_sort=-last_modified": [ + // Subsequent signature returned is a valid one for the three item + // collection. + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ RESPONSE_COMPLETE_INITIAL, ], }; @@ -558,23 +524,16 @@ add_task(async function test_check_synchronization_with_signatures() { // - Sync will be no-op since local is equal to server, no changes to emit. const badSigGoodOldResponses = { - // In this test, we deliberately serve a bad signature initially. The - // subsequent sitnature returned is a valid one for the three item - // collection. - "GET:/v1/buckets/main/collections/signed?_expected=5000": [ - RESPONSE_META_BAD_SIG, - RESPONSE_META_EMPTY_SIG, - ], // The first collection state is the current state (since there's no update // - but, since the signature is wrong, another request will be made) - "GET:/v1/buckets/main/collections/signed/records?_expected=5000&_sort=-last_modified&_since=4000": [ - RESPONSE_EMPTY_NO_UPDATE, + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000&_since=4000": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG, ], // The next request is for the full collection. This will be // checked against the valid signature and last_modified times will be // compared. Sync should be a no-op, even though the signature is good, // because the local collection is newer. - "GET:/v1/buckets/main/collections/signed/records?_expected=5000&_sort=-last_modified": [ + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ RESPONSE_EMPTY_INITIAL, ], }; @@ -603,17 +562,23 @@ add_task(async function test_check_synchronization_with_signatures() { // Check that a tampered local DB will be overwritten and // sync event contain the appropriate data. + const RESPONSE_COMPLETE_BAD_SIG = { + ...RESPONSE_EMPTY_NO_UPDATE, + responseBody: JSON.stringify({ + timestamp: 5000, + metadata: { + signature: { + x5u, + signature: "aW52YWxpZCBzaWduYXR1cmUK", + }, + }, + changes: [RECORD2, RECORD3], + }), + }; + const badLocalContentGoodSigResponses = { - // In this test, we deliberately serve a bad signature initially. The - // subsequent signature returned is a valid one for the three item - // collection. - "GET:/v1/buckets/main/collections/signed?_expected=5000": [ - RESPONSE_META_BAD_SIG, - RESPONSE_META_THREE_ITEMS_SIG, - ], - // The next request is for the full collection. This will be checked - // against the valid signature - so the sync should succeed. - "GET:/v1/buckets/main/collections/signed/records?_expected=5000&_sort=-last_modified": [ + "GET:/v1/buckets/main/collections/signed/changeset?_expected=5000": [ + RESPONSE_COMPLETE_BAD_SIG, RESPONSE_COMPLETE_INITIAL, ], }; @@ -659,7 +624,7 @@ add_task(async function test_check_synchronization_with_signatures() { // Check that a failing signature throws after retry, and that sync changes // are not applied. - const RESPONSE_ONLY_RECORD4 = { + const RESPONSE_ONLY_RECORD4_BAD_SIG = { comment: "Delete RECORD3, create RECORD4", sampleHeaders: [ "Content-Type: application/json; charset=UTF-8", @@ -667,7 +632,14 @@ add_task(async function test_check_synchronization_with_signatures() { ], status: { status: 200, statusText: "OK" }, responseBody: JSON.stringify({ - data: [ + timestamp: 6000, + metadata: { + signature: { + x5u, + signature: "wrong-sig-here-too", + }, + }, + changes: [ { id: "f765df30-b2f1-42f6-9803-7bd5a07b5098", last_modified: 6000, @@ -676,19 +648,11 @@ add_task(async function test_check_synchronization_with_signatures() { }), }; const allBadSigResponses = { - // In this test, we deliberately serve only a bad signature. - "GET:/v1/buckets/main/collections/signed?_expected=6000": [ - RESPONSE_META_BAD_SIG, + "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000&_since=4000": [ + RESPONSE_EMPTY_NO_UPDATE_BAD_SIG, ], - // The first collection state is the three item collection (since - // there's a sync with no updates) - but, since the signature is wrong, - // another request will be made... - "GET:/v1/buckets/main/collections/signed/records?_expected=6000&_sort=-last_modified&_since=4000": [ - RESPONSE_EMPTY_NO_UPDATE, - ], - // The next request is for the full collection. - "GET:/v1/buckets/main/collections/signed/records?_expected=6000&_sort=-last_modified": [ - RESPONSE_ONLY_RECORD4, + "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000": [ + RESPONSE_ONLY_RECORD4_BAD_SIG, ], }; @@ -747,24 +711,26 @@ add_task(async function test_check_synchronization_with_signatures() { // // Check that we don't apply changes when signature is missing in remote. - const RESPONSE_META_NO_SIG = { + const RESPONSE_NO_SIG = { sampleHeaders: [ "Content-Type: application/json; charset=UTF-8", `ETag: \"123456\"`, ], status: { status: 200, statusText: "OK" }, responseBody: JSON.stringify({ - data: { + metadata: { last_modified: 123456, }, + changes: [], + timestamp: 123456, }), }; const missingSigResponses = { // In this test, we deliberately serve metadata without the signature attribute. // As if the collection was not signed. - "GET:/v1/buckets/main/collections/signed?_expected=6000": [ - RESPONSE_META_NO_SIG, + "GET:/v1/buckets/main/collections/signed/changeset?_expected=6000": [ + RESPONSE_NO_SIG, ], };