From 38ab1c3559e25382957d608e49e624dc72a4409c Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:35:57 -0800 Subject: [PATCH] feat: add option to merge ASARs (#34) * feat: fuse ASARs * Rename, improve * Rename option * Drop universal from MACHO_MAGIC --- package.json | 4 +- src/asar-utils.ts | 173 ++++++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 23 +++++- yarn.lock | 5 ++ 4 files changed, 202 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 924f3c3..6d82a9c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "apple silicon", "universal" ], - "repository": { + "repository": { "type": "git", "url": "https://github.com/electron/universal.git" }, @@ -33,6 +33,7 @@ "@continuous-auth/semantic-release-npm": "^2.0.0", "@types/debug": "^4.1.5", "@types/fs-extra": "^9.0.4", + "@types/minimatch": "^3.0.5", "@types/node": "^14.14.7", "@types/plist": "^3.0.2", "husky": "^4.3.0", @@ -47,6 +48,7 @@ "debug": "^4.3.1", "dir-compare": "^2.4.0", "fs-extra": "^9.0.1", + "minimatch": "^3.0.4", "plist": "^3.0.4" }, "husky": { diff --git a/src/asar-utils.ts b/src/asar-utils.ts index 90f4cbc..f48d3c9 100644 --- a/src/asar-utils.ts +++ b/src/asar-utils.ts @@ -1,14 +1,38 @@ import * as asar from 'asar'; +import { execFileSync } from 'child_process'; import * as crypto from 'crypto'; import * as fs from 'fs-extra'; import * as path from 'path'; +import * as minimatch from 'minimatch'; +import * as os from 'os'; import { d } from './debug'; +const LIPO = 'lipo'; + export enum AsarMode { NO_ASAR, HAS_ASAR, } +export type MergeASARsOptions = { + x64AsarPath: string; + arm64AsarPath: string; + outputAsarPath: string; + + singleArchFiles?: string; +}; + +// See: https://github.com/apple-opensource-mirror/llvmCore/blob/0c60489d96c87140db9a6a14c6e82b15f5e5d252/include/llvm/Object/MachOFormat.h#L108-L112 +const MACHO_MAGIC = new Set([ + // 32-bit Mach-O + 0xfeedface, + 0xcefaedfe, + + // 64-bit Mach-O + 0xfeedfacf, + 0xcffaedfe, +]); + export const detectAsarMode = async (appPath: string) => { d('checking asar mode of', appPath); const asarPath = path.resolve(appPath, 'Contents', 'Resources', 'app.asar'); @@ -31,3 +55,152 @@ export const generateAsarIntegrity = (asarPath: string) => { .digest('hex'), }; }; + +function toRelativePath(file: string): string { + return file.replace(/^\//, ''); +} + +function isDirectory(a: string, file: string): boolean { + return Boolean('files' in asar.statFile(a, file)); +} + +function checkSingleArch(archive: string, file: string, allowList?: string): void { + if (allowList === undefined || !minimatch(file, allowList, { matchBase: true })) { + throw new Error( + `Detected unique file "${file}" in "${archive}" not covered by ` + + `allowList rule: "${allowList}"`, + ); + } +} + +export const mergeASARs = async ({ + x64AsarPath, + arm64AsarPath, + outputAsarPath, + singleArchFiles, +}: MergeASARsOptions): Promise => { + d(`merging ${x64AsarPath} and ${arm64AsarPath}`); + + const x64Files = new Set(asar.listPackage(x64AsarPath).map(toRelativePath)); + const arm64Files = new Set(asar.listPackage(arm64AsarPath).map(toRelativePath)); + + // + // Build set of unpacked directories and files + // + + const unpackedFiles = new Set(); + + function buildUnpacked(a: string, fileList: Set): void { + for (const file of fileList) { + const stat = asar.statFile(a, file); + + if (!('unpacked' in stat) || !stat.unpacked) { + continue; + } + + if ('files' in stat) { + continue; + } + unpackedFiles.add(file); + } + } + + buildUnpacked(x64AsarPath, x64Files); + buildUnpacked(arm64AsarPath, arm64Files); + + // + // Build list of files/directories unique to each asar + // + + for (const file of x64Files) { + if (!arm64Files.has(file)) { + checkSingleArch(x64AsarPath, file, singleArchFiles); + } + } + const arm64Unique = []; + for (const file of arm64Files) { + if (!x64Files.has(file)) { + checkSingleArch(arm64AsarPath, file, singleArchFiles); + arm64Unique.push(file); + } + } + + // + // Find common bindings with different content + // + + const commonBindings = []; + for (const file of x64Files) { + if (!arm64Files.has(file)) { + continue; + } + + // Skip directories + if (isDirectory(x64AsarPath, file)) { + continue; + } + + const x64Content = asar.extractFile(x64AsarPath, file); + const arm64Content = asar.extractFile(arm64AsarPath, file); + + if (x64Content.compare(arm64Content) === 0) { + continue; + } + + if (!MACHO_MAGIC.has(x64Content.readUInt32LE(0))) { + throw new Error(`Can't reconcile two non-macho files ${file}`); + } + + commonBindings.push(file); + } + + // + // Extract both + // + + const x64Dir = await fs.mkdtemp(path.join(os.tmpdir(), 'x64-')); + const arm64Dir = await fs.mkdtemp(path.join(os.tmpdir(), 'arm64-')); + + try { + d(`extracting ${x64AsarPath} to ${x64Dir}`); + asar.extractAll(x64AsarPath, x64Dir); + + d(`extracting ${arm64AsarPath} to ${arm64Dir}`); + asar.extractAll(arm64AsarPath, arm64Dir); + + for (const file of arm64Unique) { + const source = path.resolve(arm64Dir, file); + const destination = path.resolve(x64Dir, file); + + if (isDirectory(arm64AsarPath, file)) { + d(`creating unique directory: ${file}`); + await fs.mkdirp(destination); + continue; + } + + d(`xopying unique file: ${file}`); + await fs.mkdirp(path.dirname(destination)); + await fs.copy(source, destination); + } + + for (const binding of commonBindings) { + const source = await fs.realpath(path.resolve(arm64Dir, binding)); + const destination = await fs.realpath(path.resolve(x64Dir, binding)); + + d(`merging binding: ${binding}`); + execFileSync(LIPO, [source, destination, '-create', '-output', destination]); + } + + d(`creating archive at ${outputAsarPath}`); + + const resolvedUnpack = Array.from(unpackedFiles).map((file) => path.join(x64Dir, file)); + + await asar.createPackageWithOptions(x64Dir, outputAsarPath, { + unpack: `{${resolvedUnpack.join(',')}}`, + }); + + d('done merging'); + } finally { + await Promise.all([fs.remove(x64Dir), fs.remove(arm64Dir)]); + } +}; diff --git a/src/index.ts b/src/index.ts index dd0839c..ebe4a71 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import * as plist from 'plist'; import * as dircompare from 'dir-compare'; import { AppFile, AppFileType, getAllAppFiles } from './file-utils'; -import { AsarMode, detectAsarMode, generateAsarIntegrity } from './asar-utils'; +import { AsarMode, detectAsarMode, generateAsarIntegrity, mergeASARs } from './asar-utils'; import { sha } from './sha'; import { d } from './debug'; @@ -31,6 +31,14 @@ type MakeUniversalOpts = { * Forcefully overwrite any existing files that are in the way of generating the universal application */ force: boolean; + /** + * Merge x64 and arm64 ASARs into one. + */ + mergeASARs?: boolean; + /** + * Minimatch pattern of paths that are allowed to be present in one of the ASAR files, but not in the other. + */ + singleArchFiles?: string; }; const dupedFiles = (files: AppFile[]) => @@ -186,7 +194,18 @@ export const makeUniversalApp = async (opts: MakeUniversalOpts): Promise = * look at codifying that assumption as actual logic. */ // FIXME: Codify the assumption that app.asar.unpacked only contains native modules - if (x64AsarMode === AsarMode.HAS_ASAR) { + if (x64AsarMode === AsarMode.HAS_ASAR && opts.mergeASARs) { + d('merging x64 and arm64 asars'); + const output = path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar'); + await mergeASARs({ + x64AsarPath: path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar'), + arm64AsarPath: path.resolve(opts.arm64AppPath, 'Contents', 'Resources', 'app.asar'), + outputAsarPath: output, + singleArchFiles: opts.singleArchFiles, + }); + + generatedIntegrity['Resources/app.asar'] = generateAsarIntegrity(output); + } else if (x64AsarMode === AsarMode.HAS_ASAR) { d('checking if the x64 and arm64 asars are identical'); const x64AsarSha = await sha(path.resolve(tmpApp, 'Contents', 'Resources', 'app.asar')); const arm64AsarSha = await sha( diff --git a/yarn.lock b/yarn.lock index fb2a50c..e35d2a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -294,6 +294,11 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/minimatch@^3.0.5": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" + integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== + "@types/minimist@^1.2.0": version "1.2.1" resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.1.tgz#283f669ff76d7b8260df8ab7a4262cc83d988256"