Bug 1547995 - Upgrade kinto-offline-client.js to v12.4.0 r=glasserc

Differential Revision: https://phabricator.services.mozilla.com/D30356

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Mathieu Leplatre 2019-05-09 16:38:56 +00:00
Родитель 8308eb7ea1
Коммит 62f3958c34
11 изменённых файлов: 356 добавлений и 34 удалений

Просмотреть файл

@ -92,6 +92,7 @@ function setupKintoPreloadServer(certGenerator, options = {
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
const configPath = "/v1/";
const metadataPath = "/v1/buckets/security-state/collections/intermediates";
const recordsPath = "/v1/buckets/security-state/collections/intermediates/records";
const attachmentsPath = "/attachments/";
@ -111,7 +112,7 @@ function setupKintoPreloadServer(certGenerator, options = {
}
// Basic server information, all static
server.registerPathHandler(configPath, (request, response) => {
const handler = (request, response) => {
try {
const respData = getResponseData(request, server.identity.primaryPort);
if (!respData) {
@ -126,7 +127,9 @@ function setupKintoPreloadServer(certGenerator, options = {
} catch (e) {
info(e);
}
});
};
server.registerPathHandler(configPath, handler);
server.registerPathHandler(metadataPath, handler);
// Lists of certs
server.registerPathHandler(recordsPath, (request, response) => {
@ -505,6 +508,22 @@ function getResponseData(req, port) {
},
}),
},
"GET:/v1/buckets/security-state/collections/intermediates?": {
"responseHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1234\"",
],
"status": { status: 200, statusText: "OK" },
"responseBody": JSON.stringify({
"data": {
"id": "intermediates",
"last_modified": 1234,
},
}),
},
};
let result = cannedResponses[`${req.method}:${req.path}?${req.queryString}`] ||
cannedResponses[`${req.method}:${req.path}`] ||

Просмотреть файл

@ -33,7 +33,7 @@ const global = this;
var EXPORTED_SYMBOLS = ["Kinto"];
/*
* Version 12.3.0 - f7a9e81
* Version 12.4.0 - 896d337
*/
(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Kinto = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
@ -69,7 +69,9 @@ var _utils = require("../src/utils");
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
ChromeUtils.import("resource://gre/modules/Timer.jsm", global);
const {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
const {
XPCOMUtils
} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyGlobalGetters(global, ["fetch", "indexedDB"]);
ChromeUtils.defineModuleGetter(global, "EventEmitter", "resource://gre/modules/EventEmitter.jsm"); // Use standalone kinto-http module landed in FFx.
@ -502,6 +504,15 @@ function createListRequest(cid, store, filters, done) {
if (!indexField) {
// Iterate on all records for this collection (ie. cid)
const isSubQuery = Object.keys(filters).some(key => key.includes(".")); // (ie. filters: {"article.title": "hello"})
if (isSubQuery) {
const newFilter = (0, _utils.transformSubObjectFilters)(filters);
const request = store.index("cid").openCursor(IDBKeyRange.only(cid));
request.onsuccess = cursorHandlers.all(newFilter, done);
return request;
}
const request = store.index("cid").openCursor(IDBKeyRange.only(cid));
request.onsuccess = cursorHandlers.all(filters, done);
return request;
@ -588,24 +599,34 @@ class IDB extends _base.default {
const dataToMigrate = this._options.migrateOldData ? await migrationRequired(this.cid) : null;
this._db = await open(this.dbName, {
version: 1,
version: 2,
onupgradeneeded: event => {
const db = event.target.result; // Records store
const db = event.target.result;
const recordsStore = db.createObjectStore("records", {
keyPath: ["_cid", "id"]
}); // An index to obtain all the records in a collection.
if (event.oldVersion < 1) {
// Records store
const recordsStore = db.createObjectStore("records", {
keyPath: ["_cid", "id"]
}); // An index to obtain all the records in a collection.
recordsStore.createIndex("cid", "_cid"); // Here we create indices for every known field in records by collection.
// Local record status ("synced", "created", "updated", "deleted")
recordsStore.createIndex("cid", "_cid"); // Here we create indices for every known field in records by collection.
// Local record status ("synced", "created", "updated", "deleted")
recordsStore.createIndex("_status", ["_cid", "_status"]); // Last modified field
recordsStore.createIndex("_status", ["_cid", "_status"]); // Last modified field
recordsStore.createIndex("last_modified", ["_cid", "last_modified"]); // Timestamps store
recordsStore.createIndex("last_modified", ["_cid", "last_modified"]); // Timestamps store
db.createObjectStore("timestamps", {
keyPath: "cid"
});
db.createObjectStore("timestamps", {
keyPath: "cid"
});
}
if (event.oldVersion < 2) {
// Collections store
db.createObjectStore("collections", {
keyPath: "cid"
});
}
}
});
@ -937,6 +958,32 @@ class IDB extends _base.default {
}
}
async saveMetadata(metadata) {
try {
await this.prepare("collections", store => store.put({
cid: this.cid,
metadata
}), {
mode: "readwrite"
});
return metadata;
} catch (e) {
this._handleError("saveMetadata", e);
}
}
async getMetadata() {
try {
let entry = null;
await this.prepare("collections", store => {
store.get(this.cid).onsuccess = e => entry = e.target.result;
});
return entry ? entry.metadata : null;
} catch (e) {
this._handleError("getMetadata", e);
}
}
}
/**
* IDB transaction proxy.
@ -1155,6 +1202,14 @@ class BaseAdapter {
throw new Error("Not Implemented.");
}
saveMetadata(metadata) {
throw new Error("Not Implemented.");
}
getMetadata() {
throw new Error("Not Implemented.");
}
}
exports.default = BaseAdapter;
@ -1166,6 +1221,7 @@ Object.defineProperty(exports, "__esModule", {
value: true
});
exports.recordsEqual = recordsEqual;
exports.createKeyValueStoreIdSchema = createKeyValueStoreIdSchema;
exports.CollectionTransaction = exports.default = exports.ServerWasFlushedError = exports.SyncResultObject = void 0;
var _base = _interopRequireDefault(require("./adapters/base"));
@ -1317,6 +1373,30 @@ function createUUIDSchema() {
};
}
/**
* IDSchema for when using kinto.js as a key-value store.
* Using this IDSchema requires you to set a property as the id.
* This will be the property used to retrieve this record.
*
* @example
* const exampleCollection = db.collection("example", { idSchema: createKeyValueStoreIdSchema() })
* await exampleCollection.create({ title: "How to tie a tie", favoriteColor: "blue", id: "user123" }, { useRecordId: true })
* await exampleCollection.getAny("user123")
*/
function createKeyValueStoreIdSchema() {
return {
generate() {
throw new Error("createKeyValueStoreIdSchema() does not generate an id");
},
validate() {
return true;
}
};
}
function markStatus(record, status) {
return { ...record,
@ -1675,6 +1755,7 @@ class Collection {
async clear() {
await this.db.clear();
await this.db.saveMetadata(null);
await this.db.saveLastModified(null);
return {
data: [],
@ -2556,7 +2637,9 @@ class Collection {
const result = new SyncResultObject();
try {
// Fetch last changes from the server.
// Fetch collection metadata.
await this.pullMetadata(client, options); // Fetch last changes from the server.
await this.pullChanges(client, result, options);
const {
lastModified
@ -2680,6 +2763,23 @@ class Collection {
return await this.db.importBulk(newRecords.map(markSynced));
}
async pullMetadata(client, options = {}) {
const {
expectedTimestamp
} = options;
const query = expectedTimestamp ? {
query: {
_expected: expectedTimestamp
}
} : undefined;
const metadata = await client.getData(query);
return this.db.saveMetadata(metadata);
}
async metadata() {
return this.db.getMetadata();
}
}
/**
* A Collection-oriented wrapper for an adapter's transaction.
@ -3058,6 +3158,7 @@ exports.waterfall = waterfall;
exports.deepEqual = deepEqual;
exports.omitKeys = omitKeys;
exports.arrayEqual = arrayEqual;
exports.transformSubObjectFilters = transformSubObjectFilters;
exports.RE_RECORD_ID = void 0;
const RE_RECORD_ID = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
/**
@ -3115,6 +3216,11 @@ function filterObject(filters, entry) {
if (Array.isArray(value)) {
return value.some(candidate => candidate === entry[filter]);
} else if (typeof value === "object") {
return filterObject(value, entry[filter]);
} else if (!entry.hasOwnProperty(filter)) {
console.error(`The property ${filter} does not exist`);
return false;
}
return entry[filter] === value;
@ -3222,6 +3328,31 @@ function arrayEqual(a, b) {
return true;
}
function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
const last = arr.length - 1;
return arr.reduce((acc, cv, i) => {
if (i === last) {
return acc[cv] = val;
} else if (acc.hasOwnProperty(cv)) {
return acc[cv];
} else {
return acc[cv] = {};
}
}, nestedFiltersObj);
}
function transformSubObjectFilters(filtersObj) {
const transformedFilters = {};
for (const key in filtersObj) {
const keysArr = key.split(".");
const val = filtersObj[key];
makeNestedObjectFromArr(keysArr, val, transformedFilters);
}
return transformedFilters;
}
},{}]},{},[1])(1)
});

Просмотреть файл

@ -110,7 +110,8 @@ const statements = {
"createCollectionMetadata": `
CREATE TABLE collection_metadata (
collection_name TEXT PRIMARY KEY,
last_modified INTEGER
last_modified INTEGER,
metadata TEXT
) WITHOUT ROWID;`,
"createCollectionDataRecordIdIndex": `
@ -135,14 +136,25 @@ const statements = {
AND record_id = :record_id;`,
"saveLastModified": `
REPLACE INTO collection_metadata (collection_name, last_modified)
VALUES (:collection_name, :last_modified);`,
INSERT INTO collection_metadata(collection_name, last_modified)
VALUES(:collection_name, :last_modified)
ON CONFLICT(collection_name) DO UPDATE SET last_modified = :last_modified`,
"getLastModified": `
SELECT last_modified
FROM collection_metadata
WHERE collection_name = :collection_name;`,
"saveMetadata": `
INSERT INTO collection_metadata(collection_name, metadata)
VALUES(:collection_name, :metadata)
ON CONFLICT(collection_name) DO UPDATE SET metadata = :metadata`,
"getMetadata": `
SELECT metadata
FROM collection_metadata
WHERE collection_name = :collection_name;`,
"getRecord": `
SELECT record
FROM collection_data
@ -174,6 +186,10 @@ const statements = {
SELECT collection_name, SUM(LENGTH(record)) as size, COUNT(record) as num_records
FROM collection_data
GROUP BY collection_name;`,
"addMetadataColumn": `
ALTER TABLE collection_metadata
ADD COLUMN metadata TEXT;`,
};
const createStatements = [
@ -182,7 +198,7 @@ const createStatements = [
"createCollectionDataRecordIdIndex",
];
const currentSchemaVersion = 1;
const currentSchemaVersion = 2;
/**
* Firefox adapter.
@ -216,9 +232,11 @@ class FirefoxAdapter extends Kinto.adapters.BaseAdapter {
for (let statementName of createStatements) {
await connection.execute(statements[statementName]);
}
await connection.setSchemaVersion(currentSchemaVersion);
} else if (schema != 1) {
} else if (schema == 1) {
await connection.execute(statements.addMetadataColumn);
await connection.setSchemaVersion(currentSchemaVersion);
} else if (schema != 2) {
throw new Error("Unknown database schema: " + schema);
}
});
@ -401,6 +419,26 @@ class FirefoxAdapter extends Kinto.adapters.BaseAdapter {
});
}
async saveMetadata(metadata) {
const params = {
collection_name: this.collection,
metadata: JSON.stringify(metadata),
};
await this._executeStatement(statements.saveMetadata, params);
return metadata;
}
async getMetadata() {
const params = {
collection_name: this.collection,
};
const result = await this._executeStatement(statements.getMetadata, params);
if (result.length == 0) {
return null;
}
return JSON.parse(result[0].getResultByName("metadata"));
}
calculateStorage() {
return this._executeStatement(statements.calculateStorage, {})
.then(result => {

Просмотреть файл

@ -13,9 +13,6 @@ let server;
// are more tests for core Kinto.js (and its storage adapter) in the
// xpcshell tests under /services/common
add_task(async function test_something() {
const configPath = "/v1/";
const recordsPath = "/v1/buckets/security-state/collections/onecrl/records";
const dummyServerURL = `http://localhost:${server.identity.primaryPort}/v1`;
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
@ -43,8 +40,9 @@ add_task(async function test_something() {
info(e);
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
server.registerPathHandler("/v1/", handleResponse);
server.registerPathHandler("/v1/buckets/security-state/collections/onecrl", handleResponse);
server.registerPathHandler("/v1/buckets/security-state/collections/onecrl/records", handleResponse);
// Test an empty db populates from JSON dump.
await OneCRLBlocklistClient.maybeSync(42);
@ -148,6 +146,22 @@ function getSampleResponse(req, port) {
"hello": "kinto",
}),
},
"GET:/v1/buckets/security-state/collections/onecrl": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1234\"",
],
"status": { status: 200, statusText: "OK" },
"responseBody": JSON.stringify({
"data": {
"id": "onecrl",
"last_modified": 1234,
},
}),
},
"GET:/v1/buckets/security-state/collections/onecrl/records?_expected=2000&_sort=-last_modified&_since=1000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
@ -213,5 +227,6 @@ function getSampleResponse(req, port) {
},
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[`${req.method}:${req.path}`] ||
responses[req.method];
}

Просмотреть файл

@ -31,9 +31,6 @@ add_task(async function test_something() {
PinningBlocklistClient: PinningPreloadClient,
} = BlocklistClients.initialize({ verifySignature: false });
const configPath = "/v1/";
const recordsPath = "/v1/buckets/pinning/collections/pins/records";
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
@ -59,8 +56,9 @@ add_task(async function test_something() {
info(e);
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
server.registerPathHandler("/v1/", handleResponse);
server.registerPathHandler("/v1/buckets/pinning/collections/pins", handleResponse);
server.registerPathHandler("/v1/buckets/pinning/collections/pins/records", handleResponse);
let sss = Cc["@mozilla.org/ssservice;1"]
.getService(Ci.nsISiteSecurityService);
@ -184,6 +182,22 @@ function getSampleResponse(req, port) {
"hello": "kinto",
}),
},
"GET:/v1/buckets/pinning/collections/pins": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1234\"",
],
"status": { status: 200, statusText: "OK" },
"responseBody": JSON.stringify({
"data": {
"id": "pins",
"last_modified": 1234,
},
}),
},
"GET:/v1/buckets/pinning/collections/pins/records?_expected=2000&_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
@ -298,5 +312,6 @@ function getSampleResponse(req, port) {
},
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[`${req.method}:${req.path}`] ||
responses[req.method];
}

Просмотреть файл

@ -285,6 +285,7 @@ add_task(clear_collection);
// kinto.js), more making sure things are basically working as expected.
add_task(async function test_kinto_sync() {
const configPath = "/v1/";
const metadataPath = "/v1/buckets/default/collections/test_collection";
const recordsPath = "/v1/buckets/default/collections/test_collection/records";
// register a handler
function handleResponse(request, response) {
@ -309,6 +310,7 @@ add_task(async function test_kinto_sync() {
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(metadataPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
// create an empty collection, sync to populate
@ -392,6 +394,22 @@ function getSampleResponse(req, port) {
"hello": "kinto",
}),
},
"GET:/v1/buckets/default/collections/test_collection": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1234\"",
],
"status": { status: 200, statusText: "OK" },
"responseBody": JSON.stringify({
"data": {
"id": "test_collection",
"last_modified": 1234,
},
}),
},
"GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
@ -453,5 +471,6 @@ function getSampleResponse(req, port) {
},
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[`${req.method}:${req.path}`] ||
responses[req.method];
}

Просмотреть файл

@ -1,6 +1,7 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
const {Sqlite} = ChromeUtils.import("resource://gre/modules/Sqlite.jsm");
const {FirefoxAdapter} = ChromeUtils.import("resource://services-common/kinto-storage-adapter.js");
// set up what we need to make storage adapters
@ -221,6 +222,19 @@ function test_collection_operations() {
Assert.equal(lastModified, 1458796543);
await sqliteHandle.close();
});
add_task(async function test_save_metadata_preserves_lastModified() {
let sqliteHandle = await do_get_kinto_connection();
let adapter = do_get_kinto_adapter(sqliteHandle);
await adapter.saveLastModified(42);
await adapter.saveMetadata({id: "col"});
let lastModified = await adapter.getLastModified();
Assert.equal(lastModified, 42);
await sqliteHandle.close();
});
}
// test kinto db setup and operations in various scenarios
@ -257,3 +271,35 @@ add_test(function test_creation_from_empty_db() {
cleanup_kinto();
run_next_test();
});
// test schema version upgrade at v2
add_test(function test_migration_from_v1_to_v2() {
add_test(function test_migrate_from_v1_to_v2() {
// place an empty kinto db file in the profile
let profile = do_get_profile();
let v1DB = do_get_file("test_storage_adapter/v1.sqlite");
v1DB.copyTo(profile, kintoFilename);
run_next_test();
});
add_test(async function schema_is_update_from_1_to_2() {
// The `v1.sqlite` has schema version 1.
let sqliteHandle = await Sqlite.openConnection({ path: kintoFilename });
Assert.equal(await sqliteHandle.getSchemaVersion(), 1);
await sqliteHandle.close();
// The `.openConnection()` migrates it to version 2.
sqliteHandle = await FirefoxAdapter.openConnection({ path: kintoFilename });
Assert.equal(await sqliteHandle.getSchemaVersion(), 2);
await sqliteHandle.close();
run_next_test();
});
test_collection_operations();
cleanup_kinto();
run_next_test();
});

Двоичные данные
services/common/tests/unit/test_storage_adapter/v1.sqlite Normal file

Двоичный файл не отображается.

Просмотреть файл

@ -15,7 +15,7 @@ importScripts("resource://gre/modules/workers/require.js",
"resource://gre/modules/third_party/jsesc/jsesc.js");
const IDB_NAME = "remote-settings";
const IDB_VERSION = 1;
const IDB_VERSION = 2;
const IDB_RECORDS_STORE = "records";
const IDB_TIMESTAMPS_STORE = "timestamps";

Просмотреть файл

@ -78,9 +78,11 @@ function run_test() {
}
const configPath = "/v1/";
const changesPath = "/v1/buckets/monitor/collections/changes/records";
const metadataPath = "/v1/buckets/main/collections/password-fields";
const recordsPath = "/v1/buckets/main/collections/password-fields/records";
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(changesPath, handleResponse);
server.registerPathHandler(metadataPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
run_next_test();
@ -520,6 +522,26 @@ function getSampleResponse(req, port) {
}],
},
},
"GET:/v1/buckets/main/collections/password-fields": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1234\"",
],
"status": { status: 200, statusText: "OK" },
"responseBody": JSON.stringify({
"data": {
"id": "password-fields",
"last_modified": 1234,
"signature": {
"signature": "abcdef",
"x5u": "http://localhost/dummy",
},
},
}),
},
"GET:/v1/buckets/main/collections/password-fields/records?_expected=2000&_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",

Просмотреть файл

@ -34,6 +34,10 @@ function handleCannedResponse(cannedResponse, request, response) {
response.write(cannedResponse.responseBody);
}
function collectionPath(collectionId) {
return `/buckets/default/collections/${collectionId}`;
}
function collectionRecordsPath(collectionId) {
return `/buckets/default/collections/${collectionId}/records`;
}
@ -256,10 +260,23 @@ class KintoServer {
return;
}
this.collections.add(collectionId);
const remoteCollectionPath = "/v1" + collectionPath(encodeURIComponent(collectionId));
this.httpServer.registerPathHandler(remoteCollectionPath, this.handleGetCollection.bind(this, collectionId));
const remoteRecordsPath = "/v1" + collectionRecordsPath(encodeURIComponent(collectionId));
this.httpServer.registerPathHandler(remoteRecordsPath, this.handleGetRecords.bind(this, collectionId));
}
handleGetCollection(collectionId, request, response) {
response.setStatusLine(null, 200, "OK");
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.setHeader("Date", (new Date()).toUTCString());
response.write(JSON.stringify({
data: {
id: collectionId,
},
}));
}
handleGetRecords(collectionId, request, response) {
if (this.checkAuth(request, response)) {
return;