Create an `OutputFn` that can build indexed RAM bundles

Summary: Adds functionality to assemble an indexed source map to the new Buck integration. This implementation supports startup section optimisations. Hooking it up, and grouping optimisations will be in follow-ups.

Reviewed By: jeanlauliac

Differential Revision: D5106985

fbshipit-source-id: cc4c6ac8cfe4e718fc8bb2a8a93cb88914c92e0b

local-cli/bundle/output/unbundle/as-assets.js

@@ -69,6 +69,7 @@ function saveAsAssets(
       buildSourceMapWithMetaData({
         startupModules: startupModules.concat(),
         lazyModules: lazyModules.concat(),
+        fixWrapperOffset: true,
       }),
       sourcemapSourcesRoot
     );

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -19,8 +19,7 @@ const writeSourceMap = require('./write-sourcemap');
 
 const {joinModules} = require('./util');
 
-import type ModuleTransport from '../../../../packager/src//lib/ModuleTransport';
-import type {Bundle, ModuleGroups, OutputOptions} from '../../types.flow';
+import type {Bundle, ModuleGroups, ModuleTransportLike, OutputOptions} from '../../types.flow';
 
 const SIZEOF_UINT32 = 4;
 
@@ -62,6 +61,7 @@ function saveAsIndexedFile(
         startupModules: startupModules.concat(),
         lazyModules: lazyModules.concat(),
         moduleGroups,
+        fixWrapperOffset: true,
       }),
       sourcemapSourcesRoot
     );
@@ -104,7 +104,7 @@ function entryOffset(n) {
   return (2 + n * 2) * SIZEOF_UINT32;
 }
 
-function buildModuleTable(startupCode, buffers, moduleGroups) {
+function buildModuleTable(startupCode, moduleBuffers, moduleGroups) {
   // table format:
   // - num_entries:      uint_32  number of entries
   // - startup_code_len: uint_32  length of the startup section
@@ -127,7 +127,7 @@ function buildModuleTable(startupCode, buffers, moduleGroups) {
 
   // entries
   let codeOffset = startupCode.length;
-  buffers.forEach(({id, buffer}) => {
+  moduleBuffers.forEach(({id, buffer}) => {
     const group = moduleGroups.groups.get(id);
     const idsInGroup = group ? [id].concat(Array.from(group)) : [id];
 
@@ -170,7 +170,12 @@ function buildModuleBuffers(modules, moduleGroups, encoding) {
     ));
 }
 
-function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
+function buildTableAndContents(
+  startupCode: string,
+  modules: $ReadOnlyArray<ModuleTransportLike>,
+  moduleGroups: ModuleGroups,
+  encoding?: 'utf8' | 'utf16le' | 'ascii',
+) {
   // file contents layout:
   // - magic number      char[4]  0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
   // - offset table      table    see `buildModuleTables`
@@ -190,7 +195,7 @@ function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
 
 function createModuleGroups(
   groups: Map<number, Set<number>>,
-  modules: Array<ModuleTransport>,
+  modules: $ReadOnlyArray<ModuleTransportLike>,
 ): ModuleGroups {
   return {
     groups,
@@ -205,4 +210,6 @@ function * concat(iterators) {
   }
 }
 
-module.exports = saveAsIndexedFile;
+exports.save = saveAsIndexedFile;
+exports.buildTableAndContents = buildTableAndContents;
+exports.createModuleGroups = createModuleGroups;

local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js

@@ -10,25 +10,31 @@
  */
 'use strict';
 
-const {combineSourceMaps, joinModules} = require('./util');
+const {combineSourceMaps, combineSourceMapsAddingOffsets, joinModules} = require('./util');
 
 import type {ModuleGroups, ModuleTransportLike} from '../../types.flow';
 
 type Params = {
+  fixWrapperOffset: boolean,
   lazyModules: Array<ModuleTransportLike>,
   moduleGroups?: ModuleGroups,
   startupModules: Array<ModuleTransportLike>,
 };
 
-module.exports = ({startupModules, lazyModules, moduleGroups}: Params) => {
+module.exports = ({fixWrapperOffset, lazyModules, moduleGroups, startupModules}: Params) => {
+  const options = fixWrapperOffset ? {fixWrapperOffset: true} : undefined;
   const startupModule: ModuleTransportLike = {
     code: joinModules(startupModules),
     id:  Number.MIN_SAFE_INTEGER,
-    map: combineSourceMaps({modules: startupModules}),
+    map: combineSourceMaps(startupModules, undefined, options),
+    sourcePath: '',
   };
-  return combineSourceMaps({
-    modules: [startupModule].concat(lazyModules),
+
+  const map = combineSourceMapsAddingOffsets(
+    [startupModule].concat(lazyModules),
     moduleGroups,
-    withCustomOffsets: true,
-  });
+    options,
+  );
+  delete map.x_facebook_offsets[Number.MIN_SAFE_INTEGER];
+  return map;
 };

local-cli/bundle/output/unbundle/index.js

@@ -14,7 +14,7 @@
 const Server = require('../../../../packager/src/Server');
 
 const asAssets = require('./as-assets');
-const asIndexedFile = require('./as-indexed-file');
+const asIndexedFile = require('./as-indexed-file').save;
 
 import type Bundle from '../../../../packager/src//Bundler/Bundle';
 import type {OutputOptions, RequestOptions} from '../../types.flow';

local-cli/bundle/output/unbundle/util.js

@@ -12,7 +12,7 @@
 
 const invariant = require('fbjs/lib/invariant');
 
-import type {IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap';
+import type {FBIndexMap, IndexMap, MappingsMap, SourceMap} from '../../../../packager/src/lib/SourceMap';
 import type {ModuleGroups, ModuleTransportLike} from '../../types.flow';
 
 const newline = /\r\n?|\n|\u2028|\u2029/g;
@@ -43,39 +43,45 @@ const Section =
   (line: number, column: number, map: SourceMap) =>
     ({map, offset: {line, column}});
 
-type CombineSourceMapsOptions = {
-  moduleGroups?: ModuleGroups,
-  modules: Array<ModuleTransportLike>,
-  withCustomOffsets?: boolean,
-};
+type CombineOptions = {fixWrapperOffset: boolean};
 
-function combineSourceMaps({
-  moduleGroups,
-  modules,
-  withCustomOffsets,
-}: CombineSourceMapsOptions): IndexMap {
-  const offsets = [];
+function combineSourceMaps(
+  modules: Array<ModuleTransportLike>,
+  moduleGroups?: ModuleGroups,
+  options?: ?CombineOptions,
+): IndexMap {
+  const sections = combineMaps(modules, null, moduleGroups, options);
+  return {sections, version: 3};
+}
+
+function combineSourceMapsAddingOffsets(
+  modules: Array<ModuleTransportLike>,
+  moduleGroups?: ModuleGroups,
+  options?: ?CombineOptions,
+): FBIndexMap {
+  const x_facebook_offsets = [];
+  const sections = combineMaps(modules, x_facebook_offsets, moduleGroups, options);
+  return {sections, version: 3, x_facebook_offsets};
+}
+
+function combineMaps(modules, offsets: ?Array<number>, moduleGroups, options) {
   const sections = [];
-  const sourceMap: IndexMap = withCustomOffsets
-    ? {sections, version: 3, x_facebook_offsets: offsets}
-    : {sections, version: 3};
 
   let line = 0;
   modules.forEach(moduleTransport => {
     const {code, id, name} = moduleTransport;
     let column = 0;
-    let hasOffset = false;
     let group;
     let groupLines = 0;
     let {map} = moduleTransport;
 
-    if (withCustomOffsets) {
-      if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
-        // this is a module appended to another module
-        return;
-      }
+    if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
+      // this is a module appended to another module
+      return;
+    }
 
 
+    if (offsets != null) {
       group = moduleGroups && moduleGroups.groups.get(id);
       if (group && moduleGroups) {
         const {modulesById} = moduleGroups;
@@ -86,13 +92,10 @@ function combineSourceMaps({
         otherModules.forEach(m => {
           groupLines += countLines(m.code);
         });
-        map = combineSourceMaps({
-          modules: [moduleTransport].concat(otherModules),
-        });
+        map = combineSourceMaps([moduleTransport].concat(otherModules));
       }
 
-      hasOffset = id != null;
-      column = wrapperEnd(code);
+      column = options && options.fixWrapperOffset ? wrapperEnd(code) : 0;
     }
 
     invariant(
@@ -100,7 +103,7 @@ function combineSourceMaps({
       'Random Access Bundle source maps cannot be built from raw mappings',
     );
     sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
-    if (hasOffset) {
+    if (offsets != null && id != null) {
       offsets[id] = line;
       for (const moduleId of group || []) {
         offsets[moduleId] = line;
@@ -109,7 +112,7 @@ function combineSourceMaps({
     line += countLines(code) + groupLines;
   });
 
-  return sourceMap;
+  return sections;
 }
 
 const joinModules =
@@ -117,8 +120,9 @@ const joinModules =
     modules.map(m => m.code).join('\n');
 
 module.exports = {
-  countLines,
-  lineToLineSourceMap,
   combineSourceMaps,
+  combineSourceMapsAddingOffsets,
+  countLines,
   joinModules,
+  lineToLineSourceMap,
 };

local-cli/bundle/types.flow.js

@@ -19,7 +19,7 @@ export type {Bundle, FBSourceMap, ModuleTransport, SourceMap, Unbundle};
 
 export type ModuleGroups = {|
   groups: Map<number, Set<number>>,
-  modulesById: Map<number, ModuleTransport>,
+  modulesById: Map<number, ModuleTransportLike>,
   modulesInGroups: Set<number>,
 |};
 
@@ -28,6 +28,7 @@ export type ModuleTransportLike = {
   id: number,
   map?: $PropertyType<ModuleTransport, 'map'>,
   +name?: string,
+  sourcePath: string,
 };
 
 export type OutputOptions = {

packager/src/ModuleGraph/output/__tests__/as-indexed-ram-bundle-test.js

@@ -0,0 +1,231 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+declare var jest: any;
+jest.disableAutomock();
+
+const indexedRamBundle = require('../indexed-ram-bundle');
+
+declare var describe: any;
+declare var expect: any;
+declare var it: (string, () => ?Promise<any>) => void;
+declare var beforeAll: (() => ?Promise<any>) => void;
+
+let code: Buffer;
+let map;
+let ids, modules, requireCall;
+const idForPath = ({path}) => getId(path);
+beforeAll(() => {
+  modules = [
+    makeModule('a', 'script'),
+    makeModule('b'),
+    makeModule('c'),
+    makeModule('d'),
+    makeModule('e'),
+    makeModule('f'),
+  ];
+  requireCall = makeModule('r', 'script', 'require(1);');
+
+  ids = new Map(modules.map(({file}, i) => [file.path, i]));
+  ({code, map} = createRamBundle());
+});
+
+it('starts the bundle file with the magic number', () => {
+  expect(code.readUInt32LE(0)).toBe(0xFB0BD1E5);
+});
+
+it('contains the number of modules in the module table', () => {
+  expect(code.readUInt32LE(SIZEOF_INT32)).toBe(modules.length);
+});
+
+it('has the length correct of the startup section', () => {
+  expect(code.readUInt32LE(SIZEOF_INT32 * 2))
+    .toBe(requireCall.file.code.length + 1);
+});
+
+it('contains the code after the offset table', () => {
+  const {codeOffset, startupSectionLength, table} = parseOffsetTable(code);
+
+  const startupSection =
+    code.slice(codeOffset, codeOffset + startupSectionLength - 1);
+  expect(startupSection.toString()).toBe(requireCall.file.code);
+
+  table.forEach(([offset, length], i) => {
+    const moduleCode =
+      code.slice(codeOffset + offset, codeOffset + offset + length - 1);
+    expect(moduleCode.toString()).toBe(modules[i].file.code);
+  });
+});
+
+it('creates a source map', () => {
+  let line = countLines(requireCall);
+  expect(map.sections.slice(1)).toEqual(modules.map(m => {
+    const section = {
+      map: m.file.map || lineByLineMap(m.file.path),
+      offset: {column: 0, line},
+    };
+    line += countLines(m);
+    return section;
+  }));
+  expect(map.x_facebook_offsets).toEqual([1, 2, 3, 4, 5, 6]);
+});
+
+describe('Optimization:', () => {
+  let last, preloaded;
+  beforeAll(() => {
+    last = modules[modules.length - 1];
+    preloaded = [modules[2], modules[3], last];
+    ({code, map} = createRamBundle(new Set(preloaded.map(getPath))));
+  });
+
+  it('supports additional modules in the startup section', () => {
+    const {codeOffset, startupSectionLength, table} = parseOffsetTable(code);
+
+    const startupSection =
+      code.slice(codeOffset, codeOffset + startupSectionLength - 1);
+    expect(startupSection.toString())
+      .toBe(preloaded.concat([requireCall]).map(getCode).join('\n'));
+
+
+    preloaded.forEach(m => {
+      const idx = idForPath(m.file);
+      expect(table[idx]).toEqual(m === last ? undefined : [0, 0]);
+    });
+
+    table.forEach(([offset, length], i) => {
+      if (offset !== 0 && length !== 0) {
+        const moduleCode =
+          code.slice(codeOffset + offset, codeOffset + offset + length - 1);
+        expect(moduleCode.toString()).toBe(modules[i].file.code);
+      }
+    });
+  });
+
+  it('reflects additional sources in the startup section in the source map', () => {
+    let line = preloaded.reduce(
+      (l, m) => l + countLines(m),
+      countLines(requireCall),
+    );
+
+    expect(map.x_facebook_offsets).toEqual([4, 5,,, 6]); // eslint-disable-line no-sparse-arrays
+
+    expect(map.sections.slice(1)).toEqual(
+      modules
+        .filter(not(Set.prototype.has), new Set(preloaded))
+        .map(m => {
+          const section = {
+            map: m.file.map || lineByLineMap(m.file.path),
+            offset: {column: 0, line},
+          };
+          line += countLines(m);
+          return section;
+        }
+    ));
+
+  });
+});
+
+function createRamBundle(preloadedModules = new Set()) {
+  const build = indexedRamBundle.createBuilder(preloadedModules);
+  const result = build({
+    filename: 'arbitrary/filename.js',
+    idForPath,
+    modules,
+    requireCalls: [requireCall],
+  });
+
+  if (typeof result.code === 'string') {
+    throw new Error('Expected a buffer, not a string');
+  }
+  return {code: result.code, map: result.map};
+}
+
+function makeModule(name, type = 'module', moduleCode = `var ${name};`) {
+  const path = `/${name}.js`;
+  return {
+    dependencies: [],
+    file: {
+      code: type === 'module' ? makeModuleCode(moduleCode) : moduleCode,
+      map: type !== 'module'
+        ? null
+        : makeModuleMap(name, path),
+      path,
+      type,
+    },
+  };
+}
+
+function makeModuleMap(name, path) {
+  return {
+    version: 3,
+    mappings: Array(parseInt(name, 36) + 1).join(','),
+    names: [name],
+    sources: [path],
+  };
+}
+
+function makeModuleCode(moduleCode) {
+  return `__d(() => {${moduleCode}})`;
+}
+
+function getId(path) {
+  if (path === requireCall.file.path) {
+    return -1;
+  }
+
+  const id = ids.get(path);
+  if (id == null) {
+    throw new Error(`Unknown file: ${path}`);
+  }
+  return id;
+}
+
+function getCode(module) {
+  return module.file.code;
+}
+
+function getPath(module) {
+  return module.file.path;
+}
+
+const SIZEOF_INT32 = 4;
+function parseOffsetTable(buffer) {
+  const n = buffer.readUInt32LE(SIZEOF_INT32);
+  const startupSectionLength = buffer.readUInt32LE(SIZEOF_INT32 * 2);
+  const baseOffset = SIZEOF_INT32 * 3;
+  const table = Array(n);
+  for (let i = 0; i < n; ++i) {
+    const offset = baseOffset + i * 2 * SIZEOF_INT32;
+    table[i] = [buffer.readUInt32LE(offset), buffer.readUInt32LE(offset + SIZEOF_INT32)];
+  }
+  return {
+    codeOffset: baseOffset + n * 2 * SIZEOF_INT32,
+    startupSectionLength,
+    table,
+  };
+}
+
+function countLines(module) {
+  return module.file.code.split('\n').length;
+}
+
+function lineByLineMap(file) {
+  return {
+    file: file,
+    mappings: 'AAAA;',
+    names: [],
+    sources: [file],
+    version: 3,
+  };
+}
+
+const not = fn => function() { return !fn.apply(this, arguments); };

packager/src/ModuleGraph/output/indexed-ram-bundle.js

@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * @flow
+ */
+'use strict';
+
+const buildSourceMapWithMetaData = require('../../../../local-cli/bundle/output/unbundle/build-unbundle-sourcemap-with-metadata.js');
+
+const {buildTableAndContents, createModuleGroups} = require('../../../../local-cli/bundle/output/unbundle/as-indexed-file');
+const {concat} = require('./util');
+
+import type {FBIndexMap} from '../../lib/SourceMap.js';
+import type {OutputFn} from '../types.flow';
+
+function asIndexedRamBundle({
+  filename,
+  idForPath,
+  modules,
+  preloadedModules,
+  requireCalls,
+}) {
+  const [startup, deferred] = partition(modules, preloadedModules);
+  const startupModules = Array.from(concat(startup, requireCalls));
+  const deferredModules = deferred.map(m => toModuleTransport(m.file, idForPath));
+  const moduleGroups = createModuleGroups(new Map(), deferredModules);
+
+  const tableAndContents = buildTableAndContents(
+    startupModules.map(getModuleCode).join('\n'),
+    deferredModules,
+    moduleGroups,
+    'utf8',
+  );
+
+  return {
+    code: Buffer.concat(tableAndContents),
+    map: buildSourceMapWithMetaData({
+      fixWrapperOffset: false,
+      startupModules: startupModules.map(m => toModuleTransport(m.file, idForPath)),
+      lazyModules: deferredModules,
+    }),
+  };
+}
+
+function toModuleTransport(file, idForPath) {
+  return {
+    code: file.code,
+    id: idForPath(file),
+    map: file.map,
+    name: file.path,
+    sourcePath: file.path,
+  };
+}
+
+function getModuleCode(module) {
+  return module.file.code;
+}
+
+function partition(modules, preloadedModules) {
+  const startup = [];
+  const deferred = [];
+  for (const module of modules) {
+    (preloadedModules.has(module.file.path) ? startup : deferred).push(module);
+  }
+
+  return [startup, deferred];
+}
+
+function createBuilder(preloadedModules: Set<string>): OutputFn<FBIndexMap> {
+  return x => asIndexedRamBundle({preloadedModules, ...x});
+}
+exports.createBuilder = createBuilder;

packager/src/ModuleGraph/output/plain-bundle.js

@@ -13,7 +13,7 @@
 const meta = require('../../../../local-cli/bundle/output/meta');
 
 const {createIndexMap} = require('./source-map');
-const {addModuleIdsToModuleWrapper} = require('./util');
+const {addModuleIdsToModuleWrapper, concat} = require('./util');
 
 import type {OutputFn} from '../types.flow';
 
@@ -55,16 +55,10 @@ function asPlainBundle({
   };
 }
 
-module.exports = (asPlainBundle: OutputFn);
+module.exports = (asPlainBundle: OutputFn<>);
 
 const reLine = /^/gm;
 function countLines(string: string): number {
   //$FlowFixMe This regular expression always matches
   return string.match(reLine).length;
 }
-
-function* concat<T>(...iterables: Array<Iterable<T>>): Iterable<T> {
-  for (const it of iterables) {
-    yield* it;
-  }
-}

packager/src/ModuleGraph/output/util.js

@@ -44,6 +44,14 @@ exports.addModuleIdsToModuleWrapper = (
   );
 };
 
+exports.concat = function* concat<T>(
+  ...iterables: Array<Iterable<T>>
+): Iterable<T> {
+  for (const it of iterables) {
+    yield* it;
+  }
+};
+
 // Creates an idempotent function that returns numeric IDs for objects based
 // on their `path` property.
 exports.createIdForPathFn = (): ({path: string} => number) => {

packager/src/ModuleGraph/types.flow.js

@@ -10,7 +10,7 @@
  */
 'use strict';
 
-import type {MappingsMap, SourceMap} from '../lib/SourceMap';
+import type {FBSourceMap, MappingsMap, SourceMap} from '../lib/SourceMap';
 import type {Ast} from 'babel-core';
 import type {Console} from 'console';
 export type {Transformer} from '../JSTransformer/worker/worker.js';
@@ -26,7 +26,7 @@ type Dependency = {|
 
 export type File = {|
   code: string,
-  map?: ?Object,
+  map?: ?MappingsMap,
   path: string,
   type: CodeFileTypes,
 |};
@@ -75,18 +75,18 @@ export type PostProcessModules = (
   entryPoints: Array<string>,
 ) => Iterable<Module>;
 
-export type OutputFn = ({|
+export type OutputFn<M: FBSourceMap | SourceMap = FBSourceMap | SourceMap> = ({|
   filename: string,
   idForPath: IdForPathFn,
   modules: Iterable<Module>,
   requireCalls: Iterable<Module>,
   sourceMapPath?: string,
-|}) => OutputResult;
+|}) => OutputResult<M>;
 
-type OutputResult = {|
+type OutputResult<M: FBSourceMap | SourceMap> = {|
   code: string | Buffer,
   extraFiles?: Iterable<[string, string | Buffer]>,
-  map: SourceMap,
+  map: M,
 |};
 
 export type PackageData = {|

packager/src/lib/SourceMap.js

@@ -28,8 +28,9 @@ export type IndexMap = {
   version: number,
 };
 
+export type FBIndexMap = IndexMap & FBExtensions;
 export type SourceMap = IndexMap | MappingsMap;
-export type FBSourceMap = (IndexMap & FBExtensions) | (MappingsMap & FBExtensions);
+export type FBSourceMap = FBIndexMap | (MappingsMap & FBExtensions);
 
 function isMappingsMap(map: SourceMap)/*: %checks*/ {
   return map.mappings !== undefined;