From 39f83c4a12b8350e331c3a6c575a2cab64950d7a Mon Sep 17 00:00:00 2001 From: Adam Miskiewicz Date: Thu, 14 Sep 2017 04:20:26 -0700 Subject: [PATCH] Refactor how symlinks are discovered in local-cli, support scoped modules Summary: This PR refactors the symlink finding logic in local-cli in order to support nested symlinked modules as well as symlinked scoped NPM modules. Run tests, or try project with `npm link`'ed or `yarn link`'ed dependencies. Closes https://github.com/facebook/react-native/pull/15776 Reviewed By: cpojer Differential Revision: D5823008 Pulled By: jeanlauliac fbshipit-source-id: f2daeceef37bed2f8a136a0a5918730f9832884c --- local-cli/util/Config.js | 19 +- local-cli/util/__mocks__/fs.js | 368 ++++++++++++++++++ .../__tests__/findSymlinkedModules-test.js | 354 +++++++++++++++++ local-cli/util/findSymlinkedModules.js | 110 ++++++ 4 files changed, 842 insertions(+), 9 deletions(-) create mode 100644 local-cli/util/__mocks__/fs.js create mode 100644 local-cli/util/__tests__/findSymlinkedModules-test.js create mode 100644 local-cli/util/findSymlinkedModules.js diff --git a/local-cli/util/Config.js b/local-cli/util/Config.js index a17645a27c..ecde7953eb 100644 --- a/local-cli/util/Config.js +++ b/local-cli/util/Config.js @@ -14,7 +14,7 @@ * found when Flow v0.54 was deployed. To see the error delete this comment and * run Flow. */ const blacklist = require('metro-bundler/src/blacklist'); -const findSymlinksPaths = require('./findSymlinksPaths'); +const findSymlinkedModules = require('./findSymlinkedModules'); const fs = require('fs'); const getPolyfills = require('../../rn-get-polyfills'); const invariant = require('fbjs/lib/invariant'); @@ -168,14 +168,15 @@ function getProjectPath() { return path.resolve(__dirname, '../..'); } -const resolveSymlink = (roots) => - roots.concat( - findSymlinksPaths( - path.join(getProjectPath(), 'node_modules'), - roots - ) +const resolveSymlinksForRoots = roots => + roots.reduce( + (arr, rootPath) => arr.concat( + findSymlinkedModules(rootPath, roots) + ), + [...roots] ); + /** * Module capable of getting the configuration out of a given file. * @@ -195,9 +196,9 @@ const Config = { getProjectRoots: () => { const root = process.env.REACT_NATIVE_APP_ROOT; if (root) { - return resolveSymlink([path.resolve(root)]); + return resolveSymlinksForRoots([path.resolve(root)]); } - return resolveSymlink([getProjectPath()]); + return resolveSymlinksForRoots([getProjectPath()]); }, getProvidesModuleNodeModules: () => providesModuleNodeModules.slice(), getSourceExts: () => [], diff --git a/local-cli/util/__mocks__/fs.js b/local-cli/util/__mocks__/fs.js new file mode 100644 index 0000000000..f540602843 --- /dev/null +++ b/local-cli/util/__mocks__/fs.js @@ -0,0 +1,368 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @format + */ + +'use strict'; + +const {EventEmitter} = require('events'); +const {dirname} = require.requireActual('path'); +const fs = jest.genMockFromModule('fs'); +const path = require('path'); +const stream = require.requireActual('stream'); + +const noop = () => {}; + +function asyncCallback(cb) { + return function() { + setImmediate(() => cb.apply(this, arguments)); + }; +} + +const mtime = { + getTime: () => Math.ceil(Math.random() * 10000000), +}; + +fs.realpath.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let node; + try { + node = getToNode(filepath); + } catch (e) { + return callback(e); + } + if (node && typeof node === 'object' && node.SYMLINK != null) { + return callback(null, node.SYMLINK); + } + return callback(null, filepath); +}); + +fs.readdirSync.mockImplementation(filepath => Object.keys(getToNode(filepath))); + +fs.readdir.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let node; + try { + node = getToNode(filepath); + if (node && typeof node === 'object' && node.SYMLINK != null) { + node = getToNode(node.SYMLINK); + } + } catch (e) { + return callback(e); + } + + if (!(node && typeof node === 'object' && node.SYMLINK == null)) { + return callback(new Error(filepath + ' is not a directory.')); + } + + return callback(null, Object.keys(node)); +}); + +fs.readFile.mockImplementation(function(filepath, encoding, callback) { + callback = asyncCallback(callback); + if (arguments.length === 2) { + callback = encoding; + encoding = null; + } + + let node; + try { + node = getToNode(filepath); + // dir check + if (node && typeof node === 'object' && node.SYMLINK == null) { + callback(new Error('Error readFile a dir: ' + filepath)); + } + if (node == null) { + return callback(Error('No such file: ' + filepath)); + } else { + return callback(null, node); + } + } catch (e) { + return callback(e); + } +}); + +fs.readFileSync.mockImplementation(function(filepath, encoding) { + const node = getToNode(filepath); + // dir check + if (node && typeof node === 'object' && node.SYMLINK == null) { + throw new Error('Error readFileSync a dir: ' + filepath); + } + return node; +}); + +function readlinkSync(filepath) { + const node = getToNode(filepath); + if (node !== null && typeof node === 'object' && !!node.SYMLINK) { + return node.SYMLINK; + } else { + throw new Error(`EINVAL: invalid argument, readlink '${filepath}'`); + } +} + +fs.readlink.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let result; + try { + result = readlinkSync(filepath); + } catch (e) { + callback(e); + return; + } + callback(null, result); +}); + +fs.readlinkSync.mockImplementation(readlinkSync); + +function existsSync(filepath) { + try { + const node = getToNode(filepath); + return node !== null; + } catch (e) { + return false; + } +} + +fs.exists.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let result; + try { + result = existsSync(filepath); + } catch (e) { + callback(e); + return; + } + callback(null, result); +}); + +fs.existsSync.mockImplementation(existsSync); + +function makeStatResult(node) { + const isSymlink = node != null && node.SYMLINK != null; + return { + isBlockDevice: () => false, + isCharacterDevice: () => false, + isDirectory: () => node != null && typeof node === 'object' && !isSymlink, + isFIFO: () => false, + isFile: () => node != null && typeof node === 'string', + isSocket: () => false, + isSymbolicLink: () => isSymlink, + mtime, + }; +} + +function statSync(filepath) { + const node = getToNode(filepath); + if (node != null && node.SYMLINK) { + return statSync(node.SYMLINK); + } + return makeStatResult(node); +} + +fs.stat.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let result; + try { + result = statSync(filepath); + } catch (e) { + callback(e); + return; + } + callback(null, result); +}); + +fs.statSync.mockImplementation(statSync); + +function lstatSync(filepath) { + const node = getToNode(filepath); + return makeStatResult(node); +} + +fs.lstat.mockImplementation((filepath, callback) => { + callback = asyncCallback(callback); + let result; + try { + result = lstatSync(filepath); + } catch (e) { + callback(e); + return; + } + callback(null, result); +}); + +fs.lstatSync.mockImplementation(lstatSync); + +fs.open.mockImplementation(function(filepath) { + const callback = arguments[arguments.length - 1] || noop; + let data, error, fd; + try { + data = getToNode(filepath); + } catch (e) { + error = e; + } + + if (error || data == null) { + error = Error(`ENOENT: no such file or directory, open ${filepath}`); + } + if (data != null) { + /* global Buffer: true */ + fd = {buffer: new Buffer(data, 'utf8'), position: 0}; + } + + callback(error, fd); +}); + +fs.read.mockImplementation( + (fd, buffer, writeOffset, length, position, callback = noop) => { + let bytesWritten; + try { + if (position == null || position < 0) { + ({position} = fd); + } + bytesWritten = fd.buffer.copy( + buffer, + writeOffset, + position, + position + length, + ); + fd.position = position + bytesWritten; + } catch (e) { + callback(Error('invalid argument')); + return; + } + callback(null, bytesWritten, buffer); + }, +); + +fs.close.mockImplementation((fd, callback = noop) => { + try { + fd.buffer = fs.position = undefined; + } catch (e) { + callback(Error('invalid argument')); + return; + } + callback(null); +}); + +let filesystem; + +fs.createReadStream.mockImplementation(filepath => { + if (!filepath.startsWith('/')) { + throw Error('Cannot open file ' + filepath); + } + + const parts = filepath.split('/').slice(1); + let file = filesystem; + + for (const part of parts) { + file = file[part]; + if (!file) { + break; + } + } + + if (typeof file !== 'string') { + throw Error('Cannot open file ' + filepath); + } + + return new stream.Readable({ + read() { + this.push(file, 'utf8'); + this.push(null); + }, + }); +}); + +fs.createWriteStream.mockImplementation(file => { + let node; + try { + node = getToNode(dirname(file)); + } finally { + if (typeof node === 'object') { + const writeStream = new stream.Writable({ + write(chunk) { + this.__chunks.push(chunk); + }, + }); + writeStream.__file = file; + writeStream.__chunks = []; + writeStream.end = jest.fn(writeStream.end); + fs.createWriteStream.mock.returned.push(writeStream); + return writeStream; + } else { + throw new Error('Cannot open file ' + file); + } + } +}); +fs.createWriteStream.mock.returned = []; + +fs.__setMockFilesystem = object => (filesystem = object); + +const watcherListByPath = new Map(); + +fs.watch.mockImplementation((filename, options, listener) => { + if (options.recursive) { + throw new Error('recursive watch not implemented'); + } + let watcherList = watcherListByPath.get(filename); + if (watcherList == null) { + watcherList = []; + watcherListByPath.set(filename, watcherList); + } + const fsWatcher = new EventEmitter(); + fsWatcher.on('change', listener); + fsWatcher.close = () => { + watcherList.splice(watcherList.indexOf(fsWatcher), 1); + fsWatcher.close = () => { + throw new Error('FSWatcher is already closed'); + }; + }; + watcherList.push(fsWatcher); +}); + +fs.__triggerWatchEvent = (eventType, filename) => { + const directWatchers = watcherListByPath.get(filename) || []; + directWatchers.forEach(wtc => wtc.emit('change', eventType)); + const dirPath = path.dirname(filename); + const dirWatchers = watcherListByPath.get(dirPath) || []; + dirWatchers.forEach(wtc => + wtc.emit('change', eventType, path.relative(dirPath, filename)), + ); +}; + +function getToNode(filepath) { + // Ignore the drive for Windows paths. + if (filepath.match(/^[a-zA-Z]:\\/)) { + filepath = filepath.substring(2); + } + + if (filepath.endsWith(path.sep)) { + filepath = filepath.slice(0, -1); + } + const parts = filepath.split(/[\/\\]/); + if (parts[0] !== '') { + throw new Error('Make sure all paths are absolute.'); + } + let node = filesystem; + parts.slice(1).forEach(part => { + if (node && node.SYMLINK) { + node = getToNode(node.SYMLINK); + } + node = node[part]; + if (node == null) { + const err = new Error('ENOENT: no such file or directory'); + err.code = 'ENOENT'; + throw err; + } + }); + + return node; +} + +module.exports = fs; diff --git a/local-cli/util/__tests__/findSymlinkedModules-test.js b/local-cli/util/__tests__/findSymlinkedModules-test.js new file mode 100644 index 0000000000..8a1900453a --- /dev/null +++ b/local-cli/util/__tests__/findSymlinkedModules-test.js @@ -0,0 +1,354 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @format + * @emails oncall+javascript_foundation + */ + +jest.mock('fs'); + +const fs = require('fs'); +const findSymlinkedModules = require('../findSymlinkedModules'); + +describe('findSymlinksForProjectRoot', () => { + it('correctly finds normal module symlinks', () => { + fs.__setMockFilesystem({ + root: { + projectA: { + 'package.json': JSON.stringify({ + name: 'projectA', + main: 'main.js', + }), + node_modules: { + depFoo: { + 'package.json': JSON.stringify({ + name: 'depFoo', + main: 'main.js', + }), + }, + projectB: { + SYMLINK: '/root/projectB', + }, + }, + }, + projectB: { + 'package.json': JSON.stringify({ + name: 'projectB', + main: 'main.js', + }), + node_modules: { + depBar: { + 'package.json': JSON.stringify({ + name: 'depBar', + main: 'main.js', + }), + }, + }, + }, + }, + }); + + const symlinkedModules = findSymlinkedModules('/root/projectA', []); + expect(symlinkedModules).toEqual(['/root/projectB']); + }); + + it('correctly finds scoped module symlinks', () => { + fs.__setMockFilesystem({ + root: { + projectA: { + 'package.json': JSON.stringify({ + name: 'projectA', + main: 'main.js', + }), + node_modules: { + depFoo: { + 'package.json': JSON.stringify({ + name: 'depFoo', + main: 'main.js', + }), + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + projectB: { + SYMLINK: '/root/projectB', + }, + }, + }, + projectB: { + 'package.json': JSON.stringify({ + name: 'projectB', + main: 'main.js', + }), + node_modules: { + depBar: { + 'package.json': JSON.stringify({ + name: 'depBar', + main: 'main.js', + }), + }, + }, + }, + '@scoped': { + projectC: { + 'package.json': JSON.stringify({ + name: '@scoped/projectC', + main: 'main.js', + }), + }, + }, + }, + }); + + const symlinkedModules = findSymlinkedModules('/root/projectA', []); + expect(symlinkedModules).toEqual([ + '/root/@scoped/projectC', + '/root/projectB', + ]); + }); + + it('correctly finds module symlinks within other module symlinks', () => { + fs.__setMockFilesystem({ + root: { + projectA: { + 'package.json': JSON.stringify({ + name: 'projectA', + main: 'main.js', + }), + node_modules: { + depFoo: { + 'package.json': JSON.stringify({ + name: 'depFoo', + main: 'main.js', + }), + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + projectB: { + SYMLINK: '/root/projectB', + }, + }, + }, + projectB: { + 'package.json': JSON.stringify({ + name: 'projectB', + main: 'main.js', + }), + node_modules: { + depBar: { + 'package.json': JSON.stringify({ + name: 'depBar', + main: 'main.js', + }), + }, + projectD: { + SYMLINK: '/root/projectD', + }, + }, + }, + '@scoped': { + projectC: { + 'package.json': JSON.stringify({ + name: '@scoped/projectC', + main: 'main.js', + }), + }, + }, + projectD: { + 'package.json': JSON.stringify({ + name: 'projectD', + main: 'main.js', + }), + }, + }, + }); + + const symlinkedModules = findSymlinkedModules('/root/projectA', []); + expect(symlinkedModules).toEqual([ + '/root/@scoped/projectC', + '/root/projectB', + '/root/projectD', + ]); + }); + + it('correctly handles duplicate symlink paths', () => { + // projectA -> + // -> projectC + // -> projectB -> projectC + // Final list should only contain projectC once + fs.__setMockFilesystem({ + root: { + projectA: { + 'package.json': JSON.stringify({ + name: 'projectA', + main: 'main.js', + }), + node_modules: { + depFoo: { + 'package.json': JSON.stringify({ + name: 'depFoo', + main: 'main.js', + }), + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + projectB: { + SYMLINK: '/root/projectB', + }, + }, + }, + projectB: { + 'package.json': JSON.stringify({ + name: 'projectB', + main: 'main.js', + }), + node_modules: { + depBar: { + 'package.json': JSON.stringify({ + name: 'depBar', + main: 'main.js', + }), + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + }, + }, + '@scoped': { + projectC: { + 'package.json': JSON.stringify({ + name: '@scoped/projectC', + main: 'main.js', + }), + }, + }, + }, + }); + + const symlinkedModules = findSymlinkedModules('/root/projectA', []); + expect(symlinkedModules).toEqual([ + '/root/@scoped/projectC', + '/root/projectB', + ]); + }); + + it('correctly handles symlink recursion', () => { + // projectA -> + // -> projectC -> projectD -> projectA + // -> projectB -> projectC -> projectA + // -> projectD -> projectC -> projectA + // Should not infinite loop, should not contain projectA + fs.__setMockFilesystem({ + root: { + projectA: { + 'package.json': JSON.stringify({ + name: 'projectA', + main: 'main.js', + }), + node_modules: { + depFoo: { + 'package.json': JSON.stringify({ + name: 'depFoo', + main: 'main.js', + }), + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + projectB: { + SYMLINK: '/root/projectB', + }, + }, + }, + projectB: { + 'package.json': JSON.stringify({ + name: 'projectB', + main: 'main.js', + }), + node_modules: { + depBar: { + 'package.json': JSON.stringify({ + name: 'depBar', + main: 'main.js', + }), + }, + projectD: { + SYMLINK: '/root/projectD', + }, + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + }, + }, + '@scoped': { + projectC: { + 'package.json': JSON.stringify({ + name: '@scoped/projectC', + main: 'main.js', + }), + node_modules: { + projectA: { + SYMLINK: '/root/projectA', + }, + projectD: { + SYMLINK: '/root/projectD', + }, + projectE: { + SYMLINK: '/root/projectE', + }, + }, + }, + }, + projectD: { + 'package.json': JSON.stringify({ + name: 'projectD', + main: 'main.js', + }), + node_modules: { + '@scoped': { + projectC: { + SYMLINK: '/root/@scoped/projectC', + }, + }, + projectE: { + SYMLINK: '/root/projectE', + }, + }, + }, + projectE: { + 'package.json': JSON.stringify({ + name: 'projectD', + main: 'main.js', + }), + }, + }, + }); + + const symlinkedModules = findSymlinkedModules('/root/projectA'); + expect(symlinkedModules).toEqual([ + '/root/@scoped/projectC', + '/root/projectB', + '/root/projectD', + '/root/projectE', + ]); + }); +}); diff --git a/local-cli/util/findSymlinkedModules.js b/local-cli/util/findSymlinkedModules.js new file mode 100644 index 0000000000..64191b5858 --- /dev/null +++ b/local-cli/util/findSymlinkedModules.js @@ -0,0 +1,110 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * @format + * @flow + */ + +const path = require('path'); +const fs = require('fs'); + +/** + * Find symlinked modules inside "node_modules." + * + * Naively, we could just perform a depth-first search of all folders in + * node_modules, recursing when we find a symlink. + * + * We can be smarter than this due to our knowledge of how npm/Yarn lays out + * "node_modules" / how tools that build on top of npm/Yarn (such as Lerna) + * install dependencies. + * + * Starting from a given root node_modules folder, this algorithm will look at + * both the top level descendants of the node_modules folder or second level + * descendants of folders that start with "@" (which indicates a scoped + * package). If any of those folders is a symlink, it will recurse into the + * link, and perform the same search in the linked folder. + * + * The end result should be a list of all resolved module symlinks for a given + * root. + */ +module.exports = function findSymlinkedModules( + projectRoot: string, + ignoredRoots?: Array = [], +) { + const timeStart = Date.now(); + const nodeModuleRoot = path.join(projectRoot, 'node_modules'); + const resolvedSymlinks = findModuleSymlinks(nodeModuleRoot, [ + ...ignoredRoots, + projectRoot, + ]); + const timeEnd = Date.now(); + + console.log( + `Scanning folders for symlinks in ${nodeModuleRoot} (${timeEnd - + timeStart}ms)`, + ); + + return resolvedSymlinks; +}; + +function findModuleSymlinks( + modulesPath: string, + ignoredPaths: Array = [], +): Array { + if (!fs.existsSync(modulesPath)) { + return []; + } + + // Find module symlinks + const moduleFolders = fs.readdirSync(modulesPath); + const symlinks = moduleFolders.reduce((links, folderName) => { + const folderPath = path.join(modulesPath, folderName); + const maybeSymlinkPaths = []; + if (folderName.startsWith('@')) { + const scopedModuleFolders = fs.readdirSync(folderPath); + maybeSymlinkPaths.push( + ...scopedModuleFolders.map(name => path.join(folderPath, name)), + ); + } else { + maybeSymlinkPaths.push(folderPath); + } + return links.concat(resolveSymlinkPaths(maybeSymlinkPaths, ignoredPaths)); + }, []); + + // For any symlinks found, look in _that_ modules node_modules directory + // and find any symlinked modules + const nestedSymlinks = symlinks.reduce( + (links, symlinkPath) => + links.concat( + // We ignore any found symlinks or anything from the ignored list, + // to prevent infinite recursion + findModuleSymlinks(path.join(symlinkPath, 'node_modules'), [ + ...ignoredPaths, + ...symlinks, + ]), + ), + [], + ); + + return [...new Set([...symlinks, ...nestedSymlinks])]; +} + +function resolveSymlinkPaths(maybeSymlinkPaths, ignoredPaths) { + return maybeSymlinkPaths.reduce((links, maybeSymlinkPath) => { + if (fs.lstatSync(maybeSymlinkPath).isSymbolicLink()) { + const resolved = path.resolve( + path.dirname(maybeSymlinkPath), + fs.readlinkSync(maybeSymlinkPath), + ); + if (ignoredPaths.indexOf(resolved) === -1 && fs.existsSync(resolved)) { + links.push(resolved); + } + } + return links; + }, []); +}