Update script to download and import firefox schemas (fixes #1210)

bin/firefox-schema-import

@@ -3,5 +3,50 @@
 require('babel-register');
 require('babel-polyfill');
 
-require('../src/schema/firefox-schemas-import')
-  .importSchemas('src/schema/firefox', 'src/schema/updates');
+const fs = require('fs');
+const path = require('path');
+
+const schemaImport = require('../src/schema/firefox-schemas-import');
+
+const firefoxDir = 'tmp/firefox';
+const importedDir = 'src/schema/imported';
+const updatesDir = 'src/schema/updates';
+
+const version = process.argv[2];
+
+if (!/^[0-9]+$/.test(version)) {
+  console.error(`Usage: ${process.argv[1]} version`);
+  process.exit(1);
+}
+
+function emptyDir(dir) {
+  fs.readdirSync(dir).forEach((file) => {
+    fs.unlinkSync(path.join(dir, file));
+  });
+}
+
+try {
+  fs.mkdirSync(firefoxDir);
+} catch (e) {
+  // The folder already existed, remove any old schema files.
+  emptyDir(firefoxDir);
+}
+
+// Remove the old schema files.
+emptyDir(importedDir);
+
+schemaImport.fetchSchemas(version, firefoxDir)
+  .then(() => {
+    schemaImport.importSchemas(firefoxDir, updatesDir, importedDir);
+  })
+  .then(() => {
+    emptyDir(firefoxDir);
+    fs.rmdirSync(firefoxDir);
+  })
+  .catch((error) => {
+    /* eslint-disable no-console */
+    console.error(error.toString());
+    console.error(error.stack);
+    /* eslint-enable no-console */
+    process.exit(2);
+  });

package.json

@@ -37,7 +37,8 @@
     "chai": "3.5.0",
     "comment-json": "1.1.3",
     "coveralls": "2.13.0",
-    "deepmerge": "^1.3.2",
+    "deepmerge": "1.3.2",
+    "fstream": "1.0.11",
     "gfm.css": "1.1.1",
     "grunt": "1.0.1",
     "grunt-contrib-clean": "1.0.0",
@@ -59,8 +60,10 @@
     "markdown-it-emoji": "1.3.0",
     "mocha": "3.1.2",
     "mocha-multi": "0.10.0",
+    "request": "2.81.0",
     "shelljs": "0.7.7",
     "sinon": "2.1.0",
+    "tar": "2.2.1",
     "webpack": "1.14.0",
     "webpack-dev-server": "1.16.2"
   },

src/schema/firefox-schemas-import.js

@@ -1,8 +1,11 @@
 import fs from 'fs';
 import path from 'path';
+import zlib from 'zlib';
 
 import commentJson from 'comment-json';
 import merge from 'deepmerge';
+import request from 'request';
+import tar from 'tar';
 
 const FLAG_PATTERN_REGEX = /^\(\?[im]*\)(.*)/;
 const UNRECOGNIZED_PROPERTY_REFS = [
@@ -10,6 +13,16 @@ const UNRECOGNIZED_PROPERTY_REFS = [
   'manifest#/types/UnrecognizedProperty',
 ];
 
+const schemaRegexes = [
+  new RegExp('browser/components/extensions/schemas/.*\.json'),
+  new RegExp('toolkit/components/extensions/schemas/.*\.json'),
+];
+
+export const refMap = {
+  ExtensionURL: 'manifest#/types/ExtensionURL',
+  HttpURL: 'manifest#/types/HttpURL',
+};
+
 // Reference some functions on inner so they can be stubbed in tests.
 export const inner = {};
 
@@ -88,6 +101,8 @@ export function rewriteValue(key, value) {
   } else if (key === '$ref') {
     if (value.includes('#/types')) {
       return value;
+    } else if (value in refMap) {
+      return refMap[value];
     }
     let path = value;
     let schemaId = '';
@@ -208,7 +223,7 @@ export function rewriteExtend(schemas, schemaId) {
     const extendId = extendSchema.namespace;
     const extendDefinitions = {};
     const extendTypes = {};
-    extendSchema.types.forEach((type) => {
+    (extendSchema.types || []).forEach((type) => {
       const { $extend, id, ...rest } = type;
       if ($extend) {
         // Move the $extend into definitions.
@@ -235,13 +250,21 @@ export function rewriteExtend(schemas, schemaId) {
   return { definitions, refs, types };
 }
 
-inner.normalizeSchema = (schemas) => {
+inner.normalizeSchema = (schemas, file) => {
   let extendSchemas;
   let primarySchema;
 
   if (schemas.length === 1) {
-    primarySchema = schemas[0];
-    extendSchemas = [];
+    // If there is only a manifest namespace then this just extends the manifest.
+    if (schemas[0].namespace === 'manifest' && file !== 'manifest.json') {
+      primarySchema = {
+        namespace: file.slice(0, file.indexOf('.')),
+      };
+      extendSchemas = [schemas[0]];
+    } else {
+      primarySchema = schemas[0];
+      extendSchemas = [];
+    }
   } else {
     extendSchemas = schemas.slice(0, schemas.length - 1);
     primarySchema = schemas[schemas.length - 1];
@@ -258,8 +281,8 @@ inner.normalizeSchema = (schemas) => {
   };
 };
 
-inner.loadSchema = (schema) => {
-  const { id, ...rest } = inner.normalizeSchema(schema);
+inner.loadSchema = (schema, file) => {
+  const { id, ...rest } = inner.normalizeSchema(schema, file);
   const newSchema = { id, ...inner.rewriteObject(rest) };
   if (id === 'manifest') {
     newSchema.$ref = '#/types/WebExtensionManifest';
@@ -271,7 +294,7 @@ export function processSchemas(schemas, ourSchemas) {
   const loadedSchemas = {};
   schemas.forEach(({ file, schema }) => {
     // Convert the Firefox schema to more standard JSON schema.
-    const loadedSchema = inner.loadSchema(schema);
+    const loadedSchema = inner.loadSchema(schema, file);
     loadedSchemas[loadedSchema.id] = { file, schema: loadedSchema };
   });
   // Now that everything is loaded, we can finish mapping the non-standard
@@ -303,11 +326,11 @@ function schemaFiles(basePath) {
   return fs.readdirSync(basePath);
 }
 
-function writeSchemasToFile(basePath, loadedSchemas) {
+function writeSchemasToFile(basePath, importedPath, loadedSchemas) {
   // Write out the schemas.
   Object.keys(loadedSchemas).forEach((id) => {
     const { file, schema } = loadedSchemas[id];
-    writeSchema(path.join(basePath, '..', 'imported'), file, schema);
+    writeSchema(importedPath, file, schema);
   });
 }
 
@@ -324,9 +347,35 @@ function loadSchemasFromFile(basePath) {
   return schemas;
 }
 
-export function importSchemas(firefoxPath, ourPath) {
+export function importSchemas(firefoxPath, ourPath, importedPath) {
   const rawSchemas = loadSchemasFromFile(firefoxPath);
   const ourSchemas = readSchema(ourPath, 'manifest.json');
   const processedSchemas = processSchemas(rawSchemas, ourSchemas);
-  writeSchemasToFile(firefoxPath, processedSchemas);
+  writeSchemasToFile(firefoxPath, importedPath, processedSchemas);
+}
+
+function downloadUrl(version) {
+  return `https://hg.mozilla.org/mozilla-central/archive/FIREFOX_AURORA_${version}_BASE.tar.gz`;
+}
+
+inner.isBrowserSchema = (path) => {
+  return schemaRegexes.some((re) => re.test(path));
+};
+
+export function fetchSchemas(version, outputPath) {
+  return new Promise((resolve) => {
+    request.get(downloadUrl(version))
+      .pipe(zlib.createGunzip())
+      // eslint-disable-next-line new-cap
+      .pipe(tar.Parse())
+      .on('entry', (entry) => {
+        if (inner.isBrowserSchema(entry.path)) {
+          const filePath = path.join(outputPath, path.basename(entry.path));
+          entry.pipe(fs.createWriteStream(filePath));
+        }
+      })
+      .on('end', () => {
+        resolve();
+      });
+  });
 }

tests/schema/test.firefox-schemas-import.js

@@ -1,11 +1,18 @@
 import fs from 'fs';
+import fstream from 'fstream';
 import path from 'path';
+import zlib from 'zlib';
+
+import request from 'request';
+import tar from 'tar';
 
 import {
+  fetchSchemas,
+  importSchemas,
   inner,
   loadTypes,
-  importSchemas,
   processSchemas,
+  refMap,
   rewriteExtend,
   rewriteKey,
   rewriteOptionalToRequired,
@@ -15,6 +22,16 @@ import {
 describe('firefox schema import', () => {
   let sandbox;
 
+  function createDir(dirPath) {
+    fs.mkdirSync(dirPath);
+  }
+
+  function removeDir(dirPath) {
+    fs.readdirSync(dirPath).forEach(
+      (file) => fs.unlinkSync(path.join(dirPath, file)));
+    fs.rmdirSync(dirPath);
+  }
+
   beforeEach(() => {
     sandbox = sinon.sandbox.create();
   });
@@ -188,6 +205,17 @@ describe('firefox schema import', () => {
         rewriteValue('additionalProperties', { $ref: 'UnrecognizedProperty' }),
         undefined);
     });
+
+    describe('known refs that are not specific', () => {
+      beforeEach(() => { refMap.SomeType = 'manifest#/types/SomeType'; });
+      afterEach(() => { delete refMap.SomeType; });
+
+      it('get rewritten to good paths', () => {
+        assert.equal(
+          rewriteValue('$ref', 'SomeType'),
+          'manifest#/types/SomeType');
+      });
+    });
   });
 
   describe('rewriteKey', () => {
@@ -275,7 +303,7 @@ describe('firefox schema import', () => {
         });
     });
 
-    it('handles a single schema in the array', () => {
+    it('handles the manifest schema', () => {
       const schemas = [
         {
           namespace: 'manifest',
@@ -283,7 +311,7 @@ describe('firefox schema import', () => {
         },
       ];
       assert.deepEqual(
-        inner.normalizeSchema(schemas),
+        inner.normalizeSchema(schemas, 'manifest.json'),
         {
           id: 'manifest',
           types: { Permission: { id: 'Permission', type: 'string' } },
@@ -291,6 +319,41 @@ describe('firefox schema import', () => {
           refs: {},
         });
     });
+
+    it('handles manifest extensions without a schema', () => {
+      const schemas = [
+        {
+          namespace: 'manifest',
+          types: [{
+            $extend: 'WebExtensionManifest',
+            properties: {
+              chrome_url_overrides: {
+                type: 'object',
+              },
+            },
+          }],
+        },
+      ];
+      assert.deepEqual(
+        inner.normalizeSchema(schemas, 'url_overrides.json'),
+        {
+          id: 'url_overrides',
+          types: {},
+          definitions: {
+            WebExtensionManifest: {
+              properties: {
+                chrome_url_overrides: { type: 'object' },
+              },
+            },
+          },
+          refs: {
+            'url_overrides#/definitions/WebExtensionManifest': {
+              namespace: 'manifest',
+              type:  'WebExtensionManifest',
+            },
+          },
+        });
+    });
   });
 
   describe('loadSchema', () => {
@@ -823,17 +886,15 @@ describe('firefox schema import', () => {
     const expectedPath = 'tests/schema/expected';
 
     beforeEach(() => {
-      fs.mkdirSync(outputPath);
+      createDir(outputPath);
     });
 
     afterEach(() => {
-      schemaFiles.forEach(
-        (file) => fs.unlinkSync(path.join(outputPath, file)));
-      fs.rmdirSync(outputPath);
+      removeDir(outputPath);
     });
 
     it('imports schemas from filesystem', () => {
-      importSchemas(firefoxPath, ourPath);
+      importSchemas(firefoxPath, ourPath, outputPath);
       schemaFiles.forEach((file) => {
         assert.deepEqual(
           JSON.parse(fs.readFileSync(path.join(outputPath, file))),
@@ -841,4 +902,57 @@ describe('firefox schema import', () => {
       });
     });
   });
+
+  describe('fetchSchemas', () => {
+    const outputPath = 'tests/schema/imported';
+
+    beforeEach(() => {
+      createDir(outputPath);
+    });
+
+    afterEach(() => {
+      removeDir(outputPath);
+    });
+
+    it('downloads the firefox source and extracts the schemas', () => {
+      // eslint-disable-next-line new-cap
+      const packer = tar.Pack({ noProprietary: true });
+      const schemaPath = 'tests/schema/firefox';
+      // eslint-disable-next-line new-cap
+      const tarball = fstream.Reader({ path: schemaPath, type: 'Directory' })
+        .pipe(packer)
+        .pipe(zlib.createGzip());
+      sandbox
+        .stub(inner, 'isBrowserSchema')
+        .withArgs('firefox/cookies.json')
+        .returns(false)
+        .withArgs('firefox/manifest.json')
+        .returns(true);
+      sandbox
+        .stub(request, 'get')
+        .withArgs('https://hg.mozilla.org/mozilla-central/archive/FIREFOX_AURORA_54_BASE.tar.gz')
+        .returns(tarball);
+      assert.deepEqual(fs.readdirSync(outputPath), []);
+      return fetchSchemas(54, outputPath)
+        .then(() => {
+          assert.deepEqual(fs.readdirSync(outputPath), ['manifest.json']);
+        });
+    });
+  });
+
+  describe('isBrowserSchema', () => {
+    it('pulls in browser and toolkit schemas', () => {
+      const files = [
+        'moz/browser/components/extensions/schemas/bookmarks.json',
+        'moz/toolkit/components/extensions/schemas/manifest.json',
+        'moz/toolkit/components/extensions/schemas/Schemas.jsm',
+      ];
+      assert.deepEqual(
+        files.filter((f) => inner.isBrowserSchema(f)),
+        [
+          'moz/browser/components/extensions/schemas/bookmarks.json',
+          'moz/toolkit/components/extensions/schemas/manifest.json',
+        ]);
+    });
+  });
 });