From e49684e56dd667e52aa437e39bb343586e2388e2 Mon Sep 17 00:00:00 2001 From: Mark Striemer Date: Mon, 27 Mar 2017 18:11:30 -0500 Subject: [PATCH] Update script to download and import firefox schemas (fixes #1210) --- bin/firefox-schema-import | 49 +++++++- package.json | 5 +- src/schema/firefox-schemas-import.js | 71 +++++++++-- tests/schema/test.firefox-schemas-import.js | 130 ++++++++++++++++++-- 4 files changed, 233 insertions(+), 22 deletions(-) diff --git a/bin/firefox-schema-import b/bin/firefox-schema-import index e1cc3a3e..8e22bff4 100755 --- a/bin/firefox-schema-import +++ b/bin/firefox-schema-import @@ -3,5 +3,50 @@ require('babel-register'); require('babel-polyfill'); -require('../src/schema/firefox-schemas-import') - .importSchemas('src/schema/firefox', 'src/schema/updates'); +const fs = require('fs'); +const path = require('path'); + +const schemaImport = require('../src/schema/firefox-schemas-import'); + +const firefoxDir = 'tmp/firefox'; +const importedDir = 'src/schema/imported'; +const updatesDir = 'src/schema/updates'; + +const version = process.argv[2]; + +if (!/^[0-9]+$/.test(version)) { + console.error(`Usage: ${process.argv[1]} version`); + process.exit(1); +} + +function emptyDir(dir) { + fs.readdirSync(dir).forEach((file) => { + fs.unlinkSync(path.join(dir, file)); + }); +} + +try { + fs.mkdirSync(firefoxDir); +} catch (e) { + // The folder already existed, remove any old schema files. + emptyDir(firefoxDir); +} + +// Remove the old schema files. +emptyDir(importedDir); + +schemaImport.fetchSchemas(version, firefoxDir) + .then(() => { + schemaImport.importSchemas(firefoxDir, updatesDir, importedDir); + }) + .then(() => { + emptyDir(firefoxDir); + fs.rmdirSync(firefoxDir); + }) + .catch((error) => { + /* eslint-disable no-console */ + console.error(error.toString()); + console.error(error.stack); + /* eslint-enable no-console */ + process.exit(2); + }); diff --git a/package.json b/package.json index 3d6c955a..eea52826 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "chai": "3.5.0", "comment-json": "1.1.3", "coveralls": "2.13.0", - "deepmerge": "^1.3.2", + "deepmerge": "1.3.2", + "fstream": "1.0.11", "gfm.css": "1.1.1", "grunt": "1.0.1", "grunt-contrib-clean": "1.0.0", @@ -59,8 +60,10 @@ "markdown-it-emoji": "1.3.0", "mocha": "3.1.2", "mocha-multi": "0.10.0", + "request": "2.81.0", "shelljs": "0.7.7", "sinon": "2.1.0", + "tar": "2.2.1", "webpack": "1.14.0", "webpack-dev-server": "1.16.2" }, diff --git a/src/schema/firefox-schemas-import.js b/src/schema/firefox-schemas-import.js index 431659bc..1231f273 100644 --- a/src/schema/firefox-schemas-import.js +++ b/src/schema/firefox-schemas-import.js @@ -1,8 +1,11 @@ import fs from 'fs'; import path from 'path'; +import zlib from 'zlib'; import commentJson from 'comment-json'; import merge from 'deepmerge'; +import request from 'request'; +import tar from 'tar'; const FLAG_PATTERN_REGEX = /^\(\?[im]*\)(.*)/; const UNRECOGNIZED_PROPERTY_REFS = [ @@ -10,6 +13,16 @@ const UNRECOGNIZED_PROPERTY_REFS = [ 'manifest#/types/UnrecognizedProperty', ]; +const schemaRegexes = [ + new RegExp('browser/components/extensions/schemas/.*\.json'), + new RegExp('toolkit/components/extensions/schemas/.*\.json'), +]; + +export const refMap = { + ExtensionURL: 'manifest#/types/ExtensionURL', + HttpURL: 'manifest#/types/HttpURL', +}; + // Reference some functions on inner so they can be stubbed in tests. export const inner = {}; @@ -88,6 +101,8 @@ export function rewriteValue(key, value) { } else if (key === '$ref') { if (value.includes('#/types')) { return value; + } else if (value in refMap) { + return refMap[value]; } let path = value; let schemaId = ''; @@ -208,7 +223,7 @@ export function rewriteExtend(schemas, schemaId) { const extendId = extendSchema.namespace; const extendDefinitions = {}; const extendTypes = {}; - extendSchema.types.forEach((type) => { + (extendSchema.types || []).forEach((type) => { const { $extend, id, ...rest } = type; if ($extend) { // Move the $extend into definitions. @@ -235,13 +250,21 @@ export function rewriteExtend(schemas, schemaId) { return { definitions, refs, types }; } -inner.normalizeSchema = (schemas) => { +inner.normalizeSchema = (schemas, file) => { let extendSchemas; let primarySchema; if (schemas.length === 1) { - primarySchema = schemas[0]; - extendSchemas = []; + // If there is only a manifest namespace then this just extends the manifest. + if (schemas[0].namespace === 'manifest' && file !== 'manifest.json') { + primarySchema = { + namespace: file.slice(0, file.indexOf('.')), + }; + extendSchemas = [schemas[0]]; + } else { + primarySchema = schemas[0]; + extendSchemas = []; + } } else { extendSchemas = schemas.slice(0, schemas.length - 1); primarySchema = schemas[schemas.length - 1]; @@ -258,8 +281,8 @@ inner.normalizeSchema = (schemas) => { }; }; -inner.loadSchema = (schema) => { - const { id, ...rest } = inner.normalizeSchema(schema); +inner.loadSchema = (schema, file) => { + const { id, ...rest } = inner.normalizeSchema(schema, file); const newSchema = { id, ...inner.rewriteObject(rest) }; if (id === 'manifest') { newSchema.$ref = '#/types/WebExtensionManifest'; @@ -271,7 +294,7 @@ export function processSchemas(schemas, ourSchemas) { const loadedSchemas = {}; schemas.forEach(({ file, schema }) => { // Convert the Firefox schema to more standard JSON schema. - const loadedSchema = inner.loadSchema(schema); + const loadedSchema = inner.loadSchema(schema, file); loadedSchemas[loadedSchema.id] = { file, schema: loadedSchema }; }); // Now that everything is loaded, we can finish mapping the non-standard @@ -303,11 +326,11 @@ function schemaFiles(basePath) { return fs.readdirSync(basePath); } -function writeSchemasToFile(basePath, loadedSchemas) { +function writeSchemasToFile(basePath, importedPath, loadedSchemas) { // Write out the schemas. Object.keys(loadedSchemas).forEach((id) => { const { file, schema } = loadedSchemas[id]; - writeSchema(path.join(basePath, '..', 'imported'), file, schema); + writeSchema(importedPath, file, schema); }); } @@ -324,9 +347,35 @@ function loadSchemasFromFile(basePath) { return schemas; } -export function importSchemas(firefoxPath, ourPath) { +export function importSchemas(firefoxPath, ourPath, importedPath) { const rawSchemas = loadSchemasFromFile(firefoxPath); const ourSchemas = readSchema(ourPath, 'manifest.json'); const processedSchemas = processSchemas(rawSchemas, ourSchemas); - writeSchemasToFile(firefoxPath, processedSchemas); + writeSchemasToFile(firefoxPath, importedPath, processedSchemas); +} + +function downloadUrl(version) { + return `https://hg.mozilla.org/mozilla-central/archive/FIREFOX_AURORA_${version}_BASE.tar.gz`; +} + +inner.isBrowserSchema = (path) => { + return schemaRegexes.some((re) => re.test(path)); +}; + +export function fetchSchemas(version, outputPath) { + return new Promise((resolve) => { + request.get(downloadUrl(version)) + .pipe(zlib.createGunzip()) + // eslint-disable-next-line new-cap + .pipe(tar.Parse()) + .on('entry', (entry) => { + if (inner.isBrowserSchema(entry.path)) { + const filePath = path.join(outputPath, path.basename(entry.path)); + entry.pipe(fs.createWriteStream(filePath)); + } + }) + .on('end', () => { + resolve(); + }); + }); } diff --git a/tests/schema/test.firefox-schemas-import.js b/tests/schema/test.firefox-schemas-import.js index 2561b4ba..0798cd61 100644 --- a/tests/schema/test.firefox-schemas-import.js +++ b/tests/schema/test.firefox-schemas-import.js @@ -1,11 +1,18 @@ import fs from 'fs'; +import fstream from 'fstream'; import path from 'path'; +import zlib from 'zlib'; + +import request from 'request'; +import tar from 'tar'; import { + fetchSchemas, + importSchemas, inner, loadTypes, - importSchemas, processSchemas, + refMap, rewriteExtend, rewriteKey, rewriteOptionalToRequired, @@ -15,6 +22,16 @@ import { describe('firefox schema import', () => { let sandbox; + function createDir(dirPath) { + fs.mkdirSync(dirPath); + } + + function removeDir(dirPath) { + fs.readdirSync(dirPath).forEach( + (file) => fs.unlinkSync(path.join(dirPath, file))); + fs.rmdirSync(dirPath); + } + beforeEach(() => { sandbox = sinon.sandbox.create(); }); @@ -188,6 +205,17 @@ describe('firefox schema import', () => { rewriteValue('additionalProperties', { $ref: 'UnrecognizedProperty' }), undefined); }); + + describe('known refs that are not specific', () => { + beforeEach(() => { refMap.SomeType = 'manifest#/types/SomeType'; }); + afterEach(() => { delete refMap.SomeType; }); + + it('get rewritten to good paths', () => { + assert.equal( + rewriteValue('$ref', 'SomeType'), + 'manifest#/types/SomeType'); + }); + }); }); describe('rewriteKey', () => { @@ -275,7 +303,7 @@ describe('firefox schema import', () => { }); }); - it('handles a single schema in the array', () => { + it('handles the manifest schema', () => { const schemas = [ { namespace: 'manifest', @@ -283,7 +311,7 @@ describe('firefox schema import', () => { }, ]; assert.deepEqual( - inner.normalizeSchema(schemas), + inner.normalizeSchema(schemas, 'manifest.json'), { id: 'manifest', types: { Permission: { id: 'Permission', type: 'string' } }, @@ -291,6 +319,41 @@ describe('firefox schema import', () => { refs: {}, }); }); + + it('handles manifest extensions without a schema', () => { + const schemas = [ + { + namespace: 'manifest', + types: [{ + $extend: 'WebExtensionManifest', + properties: { + chrome_url_overrides: { + type: 'object', + }, + }, + }], + }, + ]; + assert.deepEqual( + inner.normalizeSchema(schemas, 'url_overrides.json'), + { + id: 'url_overrides', + types: {}, + definitions: { + WebExtensionManifest: { + properties: { + chrome_url_overrides: { type: 'object' }, + }, + }, + }, + refs: { + 'url_overrides#/definitions/WebExtensionManifest': { + namespace: 'manifest', + type: 'WebExtensionManifest', + }, + }, + }); + }); }); describe('loadSchema', () => { @@ -823,17 +886,15 @@ describe('firefox schema import', () => { const expectedPath = 'tests/schema/expected'; beforeEach(() => { - fs.mkdirSync(outputPath); + createDir(outputPath); }); afterEach(() => { - schemaFiles.forEach( - (file) => fs.unlinkSync(path.join(outputPath, file))); - fs.rmdirSync(outputPath); + removeDir(outputPath); }); it('imports schemas from filesystem', () => { - importSchemas(firefoxPath, ourPath); + importSchemas(firefoxPath, ourPath, outputPath); schemaFiles.forEach((file) => { assert.deepEqual( JSON.parse(fs.readFileSync(path.join(outputPath, file))), @@ -841,4 +902,57 @@ describe('firefox schema import', () => { }); }); }); + + describe('fetchSchemas', () => { + const outputPath = 'tests/schema/imported'; + + beforeEach(() => { + createDir(outputPath); + }); + + afterEach(() => { + removeDir(outputPath); + }); + + it('downloads the firefox source and extracts the schemas', () => { + // eslint-disable-next-line new-cap + const packer = tar.Pack({ noProprietary: true }); + const schemaPath = 'tests/schema/firefox'; + // eslint-disable-next-line new-cap + const tarball = fstream.Reader({ path: schemaPath, type: 'Directory' }) + .pipe(packer) + .pipe(zlib.createGzip()); + sandbox + .stub(inner, 'isBrowserSchema') + .withArgs('firefox/cookies.json') + .returns(false) + .withArgs('firefox/manifest.json') + .returns(true); + sandbox + .stub(request, 'get') + .withArgs('https://hg.mozilla.org/mozilla-central/archive/FIREFOX_AURORA_54_BASE.tar.gz') + .returns(tarball); + assert.deepEqual(fs.readdirSync(outputPath), []); + return fetchSchemas(54, outputPath) + .then(() => { + assert.deepEqual(fs.readdirSync(outputPath), ['manifest.json']); + }); + }); + }); + + describe('isBrowserSchema', () => { + it('pulls in browser and toolkit schemas', () => { + const files = [ + 'moz/browser/components/extensions/schemas/bookmarks.json', + 'moz/toolkit/components/extensions/schemas/manifest.json', + 'moz/toolkit/components/extensions/schemas/Schemas.jsm', + ]; + assert.deepEqual( + files.filter((f) => inner.isBrowserSchema(f)), + [ + 'moz/browser/components/extensions/schemas/bookmarks.json', + 'moz/toolkit/components/extensions/schemas/manifest.json', + ]); + }); + }); });