Support folding multiple matching schemas into one (fixes #1209)

Adds support for importing schemas from a local .tgz file.

bin/firefox-schema-import

@@ -12,16 +12,31 @@ const firefoxDir = 'tmp/firefox';
 const importedDir = 'src/schema/imported';
 const updatesDir = 'src/schema/updates';
 
-const version = process.argv[2];
+const arg = process.argv[2];
+var version;
+var filePath;
 
-if (!/^[0-9]+$/.test(version)) {
+try {
-  console.error(`Usage: ${process.argv[1]} version`);
+  fs.statSync(arg);
+  filePath = arg;
+} catch (e) {
+  if (/^[0-9]+$/.test(arg)) {
+    version = arg;
+  }
+}
+
+
+if (!(filePath || version)) {
+  // eslint-disable-next-line no-console
+  console.error(`Usage: ${process.argv[1]} version|filePath`);
   process.exit(1);
 }
 
-function emptyDir(dir) {
+function emptyDir(dir, matching) {
   fs.readdirSync(dir).forEach((file) => {
-    fs.unlinkSync(path.join(dir, file));
+    if (!matching || matching.test(file)) {
+      fs.unlinkSync(path.join(dir, file));
+    }
   });
 }
 
@@ -33,9 +48,10 @@ try {
 }
 
 // Remove the old schema files.
-emptyDir(importedDir);
+emptyDir(importedDir, /.*.json$/);
 
-schemaImport.fetchSchemas(version, firefoxDir)
+schemaImport.fetchSchemas(
+    { inputPath: filePath, outputPath: firefoxDir, version })
   .then(() => {
     schemaImport.importSchemas(firefoxDir, updatesDir, importedDir);
   })

src/schema/firefox-schemas-import.js

@@ -26,6 +26,11 @@ export const refMap = {
 // Reference some functions on inner so they can be stubbed in tests.
 export const inner = {};
 
+// Consider moving this to a Set if you add more schema namespaces.
+// Some schemas aren't actually exposed to add-ons, or are for internal
+// use in Firefox only. We shouldn't import these schemas.
+export const ignoredSchemas = ['omnibox_internal'];
+
 function stripFlagsFromPattern(value) {
   // TODO: Fix these patterns and remove this code.
   const matches = FLAG_PATTERN_REGEX.exec(value);
@@ -250,24 +255,108 @@ export function rewriteExtend(schemas, schemaId) {
   return { definitions, refs, types };
 }
 
+export function filterSchemas(schemas) {
+  return schemas.filter((schema) => {
+    return !ignoredSchemas.includes(schema.namespace);
+  });
+}
+
+/**
+ * Merge multiple schemas into one if they are properties of each other.
+ *
+ * Example:
+ *
+ *  [{ namespace: "privacy", permissions: ["privacy"] },
+ *   { namespace: "privacy.network", properties: { networkPredictionEnabled: {} } }]
+ *
+ *  becomes
+ *
+ *  [{ namespace: "privacy",
+ *     permissions: ["privacy"],
+ *     properties: {
+ *       network: {
+ *         properties: {
+ *           networkPredictionEnabled: {}
+ *  }}}}]
+ */
+export function foldSchemas(schemas) {
+  // Map the schemas by prefix.
+  const schemasByPrefix = {};
+  schemas.forEach((schema) => {
+    const [prefix, property, more] = schema.namespace.split('.', 3);
+    if (more) {
+      throw new Error('namespace may only have one level of nesting');
+    }
+    if (!(prefix in schemasByPrefix)) {
+      schemasByPrefix[prefix] = {};
+    }
+    let namespace = property ? property : 'baseNamespace';
+    if (schemasByPrefix[prefix][namespace]) {
+      throw new Error('matching namespaces are not allowed');
+    } else {
+      schemasByPrefix[prefix][namespace] = schema;
+    }
+  });
+  // If there aren't any matching prefixes then there's no folding to do.
+  const hasMatchingPrefixes = Object.keys(schemasByPrefix).some((prefix) => {
+    const prefixedSchemas = schemasByPrefix[prefix];
+    // Continue if there are multiple properties (baseNamespace and something
+    // else) or there is one property that isn't baseNamespace.
+    return Object.keys(prefixedSchemas).length > 1
+      || !('baseNamespace' in prefixedSchemas);
+  });
+  if (!hasMatchingPrefixes) {
+    return schemas;
+  }
+
+  // There is folding to do, join the matching schemas.
+  const foldedSchemas = [];
+
+  // The order of the schemas will be maintained since they were inserted in
+  // the order of schemas.
+  Object.keys(schemasByPrefix).forEach((namespace) => {
+    const { baseNamespace = {}, ...nestedSchemas } = schemasByPrefix[namespace];
+    foldedSchemas.push(baseNamespace);
+    // Ensure the base namespace is set.
+    baseNamespace.namespace = namespace;
+    if (Object.keys(nestedSchemas).length > 0 && !baseNamespace.properties) {
+      baseNamespace.properties = {};
+    }
+    Object.keys(nestedSchemas).forEach((property) => {
+      const schema = nestedSchemas[property];
+      delete schema.namespace;
+      if (schema.types) {
+        baseNamespace.types = baseNamespace.types || [];
+        baseNamespace.types = baseNamespace.types.concat(schema.types);
+        delete schema.types;
+      }
+      baseNamespace.properties[property] = schema;
+    });
+  });
+
+  return foldedSchemas;
+}
+
 inner.normalizeSchema = (schemas, file) => {
+  const filteredSchemas = foldSchemas(filterSchemas(schemas));
   let extendSchemas;
   let primarySchema;
 
-  if (schemas.length === 1) {
+  if (filteredSchemas.length === 1) {
     // If there is only a manifest namespace then this just extends the manifest.
-    if (schemas[0].namespace === 'manifest' && file !== 'manifest.json') {
+    if (filteredSchemas[0].namespace === 'manifest'
+        && file !== 'manifest.json') {
       primarySchema = {
         namespace: file.slice(0, file.indexOf('.')),
       };
-      extendSchemas = [schemas[0]];
+      extendSchemas = [filteredSchemas[0]];
     } else {
-      primarySchema = schemas[0];
+      primarySchema = filteredSchemas[0];
       extendSchemas = [];
     }
   } else {
-    extendSchemas = schemas.slice(0, schemas.length - 1);
+    extendSchemas = filteredSchemas.slice(0, filteredSchemas.length - 1);
-    primarySchema = schemas[schemas.length - 1];
+    primarySchema = filteredSchemas[filteredSchemas.length - 1];
   }
   const { namespace, types, ...rest } = primarySchema;
   const { types: extendTypes, ...extendRest } = rewriteExtend(
@@ -290,18 +379,38 @@ inner.loadSchema = (schema, file) => {
   return newSchema;
 };
 
-export function processSchemas(schemas, ourSchemas) {
+inner.mergeSchemas = (schemaLists) => {
-  const loadedSchemas = {};
+  const schemas = {};
+  Object.keys(schemaLists).forEach((namespace) => {
+    const namespaceSchemas = schemaLists[namespace];
+    if (namespaceSchemas.length === 1) {
+      schemas[namespace] = namespaceSchemas[0];
+    } else {
+      const file = `${namespace}.json`;
+      const merged = namespaceSchemas.reduce((memo, { schema }) => {
+        return merge(memo, schema);
+      }, {});
+      schemas[namespace] = { file, schema: merged };
+    }
+  });
+  return schemas;
+};
+
+export function processSchemas(schemas) {
+  const schemaListsByNamespace = {};
   schemas.forEach(({ file, schema }) => {
     // Convert the Firefox schema to more standard JSON schema.
     const loadedSchema = inner.loadSchema(schema, file);
-    loadedSchemas[loadedSchema.id] = { file, schema: loadedSchema };
+    const { id } = loadedSchema;
+    if (!(id in schemaListsByNamespace)) {
+      schemaListsByNamespace[id] = [];
+    }
+    schemaListsByNamespace[id].push({ file, schema: loadedSchema });
   });
+  const mergedSchemasByNamespace = inner.mergeSchemas(schemaListsByNamespace);
   // Now that everything is loaded, we can finish mapping the non-standard
   // $extend to $ref.
-  const extendedSchemas = inner.mapExtendToRef(loadedSchemas);
+  return inner.mapExtendToRef(mergedSchemasByNamespace);
-  // Update the Firefox schemas with some missing validations, defaults and descriptions.
-  return inner.updateWithAddonsLinterData(extendedSchemas, ourSchemas);
 }
 
 const SKIP_SCHEMAS = [
@@ -350,8 +459,10 @@ function loadSchemasFromFile(basePath) {
 export function importSchemas(firefoxPath, ourPath, importedPath) {
   const rawSchemas = loadSchemasFromFile(firefoxPath);
   const ourSchemas = readSchema(ourPath, 'manifest.json');
-  const processedSchemas = processSchemas(rawSchemas, ourSchemas);
+  const processedSchemas = processSchemas(rawSchemas);
-  writeSchemasToFile(firefoxPath, importedPath, processedSchemas);
+  const updatedSchemas = inner.updateWithAddonsLinterData(
+    processedSchemas, ourSchemas);
+  writeSchemasToFile(firefoxPath, importedPath, updatedSchemas);
 }
 
 function downloadUrl(version) {
@@ -362,9 +473,15 @@ inner.isBrowserSchema = (path) => {
   return schemaRegexes.some((re) => re.test(path));
 };
 
-export function fetchSchemas(version, outputPath) {
+export function fetchSchemas({ inputPath, outputPath, version }) {
   return new Promise((resolve) => {
-    request.get(downloadUrl(version))
+    let tarball;
+    if (inputPath) {
+      tarball = fs.createReadStream(inputPath);
+    } else if (version) {
+      tarball = request.get(downloadUrl(version));
+    }
+    tarball
       .pipe(zlib.createGunzip())
       // eslint-disable-next-line new-cap
       .pipe(tar.Parse())

tests/rules/javascript/test.unsupported_browser_api.js

@@ -44,4 +44,29 @@ describe('unsupported browser APIs', () => {
         assert.equal(validationMessages.length, 0);
       });
   });
+
+  it('does not flag on 3 levels of nesting', () => {
+    const code =
+      'browser.privacy.websites.thirdPartyCookiesAllowed.get({}, () => {})';
+    const jsScanner = new JavaScriptScanner(code, 'goodcode.js', {
+      addonMetadata: { id: '@supported-api' },
+    });
+    return jsScanner.scan()
+      .then((validationMessages) => {
+        assert.equal(validationMessages.length, 0);
+      });
+  });
+
+  // We only test the first two levels for now.
+  it.skip('flags when 3 levels of nesting is unsupported', () => {
+    const code =
+      'browser.privacy.websites.unsupportedSetting.get({}, () => {})';
+    const jsScanner = new JavaScriptScanner(code, 'badcode.js', {
+      addonMetadata: { id: '@unsupported-api' },
+    });
+    return jsScanner.scan()
+      .then((validationMessages) => {
+        assert.equal(validationMessages.length, 1);
+      });
+  });
 });

tests/schema/test.firefox-schemas-import.js

@@ -8,6 +8,9 @@ import tar from 'tar';
 
 import {
   fetchSchemas,
+  filterSchemas,
+  foldSchemas,
+  ignoredSchemas,
   importSchemas,
   inner,
   loadTypes,
@@ -281,7 +284,7 @@ describe('firefox schema import', () => {
         },
       ];
       assert.deepEqual(
-        inner.normalizeSchema(schemas),
+        inner.normalizeSchema(schemas, 'cookies.json'),
         {
           id: 'cookies',
           types: {
@@ -397,22 +400,76 @@ describe('firefox schema import', () => {
       loadSchema.withArgs(firstSchema).returns({ id: 'manifest', schema: 1 });
       loadSchema.withArgs(secondSchema).returns({ id: 'cookies', schema: 2 });
       sandbox
-        .stub(inner, 'mapExtendToRef')
+        .stub(inner, 'mergeSchemas')
         .withArgs({
-          manifest: { file: 'one', schema: { id: 'manifest', schema: 1 } },
+          manifest: [{ file: 'one', schema: { id: 'manifest', schema: 1 } }],
-          cookies: { file: 'two', schema: { id: 'cookies', schema: 2 } },
+          cookies: [{ file: 'two', schema: { id: 'cookies', schema: 2 } }],
         })
-        .returns({ mapExtendToRef: 'done' });
+        .returns({ mergeSchemas: 'done' });
       sandbox
-        .stub(inner, 'updateWithAddonsLinterData')
+        .stub(inner, 'mapExtendToRef')
-        .withArgs({ mapExtendToRef: 'done' })
+        .withArgs({ mergeSchemas: 'done' })
-        .returns({ updateWithAddonsLinterData: 'done' });
+        .returns({ mapExtendToRef: 'done' });
       assert.deepEqual(
         processSchemas([
           { file: 'one', schema: firstSchema },
           { file: 'two', schema: secondSchema },
         ]),
-        { updateWithAddonsLinterData: 'done' });
+        { mapExtendToRef: 'done' });
+    });
+  });
+
+  describe('mergeSchemas', () => {
+    it('merges schemas with the same namespace', () => {
+      const schemas = [{
+        file: 'foo_foo.json',
+        schema: [{
+          namespace: 'foo',
+          types: [{ id: 'Foo', type: 'string' }],
+        }],
+      }, {
+        file: 'foo_bar.json',
+        schema: [{
+          namespace: 'foo.bar',
+          types: [{ id: 'FooBar', type: 'number' }],
+          properties: { thing: {} },
+        }],
+      }, {
+        file: 'bar.json',
+        schema: [{
+          namespace: 'bar',
+          types: [{ id: 'Bar', type: 'string' }],
+        }],
+      }];
+
+      assert.deepEqual(
+        processSchemas(schemas),
+        {
+          foo: {
+            file: 'foo.json',
+            schema: {
+              id: 'foo',
+              definitions: {},
+              refs: {},
+              types: {
+                Foo: { type: 'string' },
+                FooBar: { type: 'number' },
+              },
+              properties: {
+                bar: { properties: { thing: {} }, required: ['thing'] },
+              },
+            },
+          },
+          bar: {
+            file: 'bar.json',
+            schema: {
+              id: 'bar',
+              definitions: {},
+              refs: {},
+              types: { Bar: { type: 'string' } },
+            },
+          },
+        });
     });
   });
 
@@ -933,7 +990,32 @@ describe('firefox schema import', () => {
         .withArgs('https://hg.mozilla.org/mozilla-central/archive/FIREFOX_AURORA_54_BASE.tar.gz')
         .returns(tarball);
       assert.deepEqual(fs.readdirSync(outputPath), []);
-      return fetchSchemas(54, outputPath)
+      return fetchSchemas({ version: 54, outputPath })
+        .then(() => {
+          assert.deepEqual(fs.readdirSync(outputPath), ['manifest.json']);
+        });
+    });
+
+    it('extracts the schemas from a local file', () => {
+      // eslint-disable-next-line new-cap
+      const packer = tar.Pack({ noProprietary: true });
+      const schemaPath = 'tests/schema/firefox';
+      // eslint-disable-next-line new-cap
+      const tarball = fstream.Reader({ path: schemaPath, type: 'Directory' })
+        .pipe(packer)
+        .pipe(zlib.createGzip());
+      sandbox
+        .stub(inner, 'isBrowserSchema')
+        .withArgs('firefox/cookies.json')
+        .returns(false)
+        .withArgs('firefox/manifest.json')
+        .returns(true);
+      sandbox
+        .stub(fs, 'createReadStream')
+        .withArgs('mozilla-central.tgz')
+        .returns(tarball);
+      assert.deepEqual(fs.readdirSync(outputPath), []);
+      return fetchSchemas({ inputPath: 'mozilla-central.tgz', outputPath })
         .then(() => {
           assert.deepEqual(fs.readdirSync(outputPath), ['manifest.json']);
         });
@@ -955,4 +1037,188 @@ describe('firefox schema import', () => {
         ]);
     });
   });
+
+  describe('foldSchemas', () => {
+    it('does not fold non-matching schemas', () => {
+      const schemas = [
+        { namespace: 'manifest' },
+        { namespace: 'omnibox' },
+      ];
+      // Copy the schemas so we can verify they're unchanged and un-mutated.
+      const expectedSchemas = schemas.map((schema) => ({ ...schema }));
+      assert.deepEqual(foldSchemas(schemas), expectedSchemas);
+    });
+
+    it('folds matching schemas, maintaining types at top-level', () => {
+      const schemas = [
+        { namespace: 'manifest' },
+        { namespace: 'privacy.network',
+          properties: { networkPredictionEnabled: {} },
+          types: [{
+            id: 'IPHandlingPolicy',
+            type: 'string',
+            enum: ['default', 'disable_non_proxied_udp'],
+          }],
+        },
+        { namespace: 'privacy',
+          permissions: ['privacy'],
+          properties: { foo: {} },
+          types: [{
+            $extend: 'permission',
+            choices: [{ type: 'string', enum: ['privacy'] }],
+          }],
+        },
+        { namespace: 'privacy.websites',
+          properties: { thirdPartyCookiesAllowed: {} } },
+      ];
+      assert.deepEqual(foldSchemas(schemas), [
+        { namespace: 'manifest' },
+        { namespace: 'privacy',
+          permissions: ['privacy'],
+          properties: {
+            foo: {},
+            network: {
+              properties: { networkPredictionEnabled: {} },
+            },
+            websites: {
+              properties: { thirdPartyCookiesAllowed: {} },
+            },
+          },
+          types: [{
+            $extend: 'permission',
+            choices: [{ type: 'string', enum: ['privacy'] }],
+          }, {
+            id: 'IPHandlingPolicy',
+            type: 'string',
+            enum: ['default', 'disable_non_proxied_udp'],
+          }],
+        },
+      ]);
+    });
+
+    it('handles a base schema without properties', () => {
+      const schemas = [
+        { namespace: 'manifest' },
+        { namespace: 'privacy.network',
+          properties: { networkPredictionEnabled: {} } },
+        { namespace: 'privacy', permissions: ['privacy'] },
+        { namespace: 'privacy.websites',
+          properties: { thirdPartyCookiesAllowed: {} } },
+      ];
+      assert.deepEqual(foldSchemas(schemas), [
+        { namespace: 'manifest' },
+        { namespace: 'privacy',
+          permissions: ['privacy'],
+          properties: {
+            network: {
+              properties: { networkPredictionEnabled: {} },
+            },
+            websites: {
+              properties: { thirdPartyCookiesAllowed: {} },
+            },
+          },
+        },
+      ]);
+    });
+
+    it('handles matching schemas without a base schema', () => {
+      const schemas = [
+        { namespace: 'manifest' },
+        { namespace: 'privacy.network',
+          properties: { networkPredictionEnabled: {} } },
+        { namespace: 'privacy.websites',
+          properties: { thirdPartyCookiesAllowed: {} } },
+      ];
+      assert.deepEqual(foldSchemas(schemas), [
+        { namespace: 'manifest' },
+        { namespace: 'privacy',
+          properties: {
+            network: {
+              properties: { networkPredictionEnabled: {} },
+            },
+            websites: {
+              properties: { thirdPartyCookiesAllowed: {} },
+            },
+          },
+        },
+      ]);
+    });
+
+    it('handles a single schema', () => {
+      const schemas = [
+        { namespace: 'alarms',
+          permissions: ['alarms'],
+          properties: {} },
+      ];
+      const expectedSchemas = schemas.map((schema) => ({ ...schema }));
+      assert.deepEqual(foldSchemas(schemas), expectedSchemas);
+    });
+
+    it('throws if there is more than two levels of nesting', () => {
+      const schemas = [
+        { namespace: 'devtools.panels.sidebars',
+          properties: { createSidebar: {} } },
+      ];
+      assert.throws(
+        () => foldSchemas(schemas),
+        /may only have one level of nesting/);
+    });
+
+    it('throws if there is more than one matching namespace', () => {
+      const schemas = Object.freeze([
+        Object.freeze({
+          namespace: 'devtools.sidebar',
+          properties: { createSidebar: {} },
+        }),
+        Object.freeze({
+          namespace: 'devtools.sidebar',
+          properties: { createBar: {} },
+        }),
+      ]);
+      assert.throws(() => foldSchemas(schemas), /matching namespaces/);
+    });
+
+    it('throws if there is more than one base namespace', () => {
+      const schemas = Object.freeze([
+        Object.freeze({
+          namespace: 'devtools',
+          properties: { createSidebar: {} },
+        }),
+        Object.freeze({
+          namespace: 'devtools',
+          properties: { createBar: {} },
+        }),
+      ]);
+      assert.throws(() => foldSchemas(schemas), /matching namespaces/);
+    });
+  });
+
+  describe('filterSchemas', () => {
+    before(() => {
+      ignoredSchemas.push('some_namespace');
+    });
+
+    after(() => {
+      ignoredSchemas.pop();
+    });
+
+    it('removes schemas that we want to ignore', () => {
+      const goodSchema = Object.freeze({
+        namespace: 'yay',
+        properties: { yay: 'woo' },
+      });
+      const schemas = [
+        goodSchema,
+        { namespace: 'some_namespace', properties: { foo: {} } },
+      ];
+      assert.deepEqual(filterSchemas(schemas), [goodSchema]);
+    });
+
+    it('does not remove anything if there are no ignored schemas', () => {
+      const schemas = Object.freeze([
+        Object.freeze({ namespace: 'alarms', permissions: ['alarms'] }),
+      ]);
+      assert.deepEqual(filterSchemas(schemas), schemas);
+    });
+  });
 });