From 7925a51149d8083c17aa00839b8a52d247abae93 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 21 Sep 2020 16:09:11 -0700 Subject: [PATCH] feat: support concurrent installation of browsers (#3929) A few details on locking registry to prohibit concurrent access: - locking is done by creating a `__dirlock` directory in the top-level of our registry. - since `__dirlock` directory does not match any of browser directories, old versions of the installer will ignore it - in case of concurrent access, installation will wait for a lock to be released for 10 minutes, periodically trying to grab the lock. If it fails to do so in 10 minutes, the installation will fail. Fixes #3912 --- package-lock.json | 38 ++++++++++++++++++++++++++++++++++++-- package.json | 2 ++ src/install/installer.ts | 20 +++++++++++++++++--- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 83a70a9a45..894246264b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1328,6 +1328,15 @@ "@types/node": "*" } }, + "@types/proper-lockfile": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.1.tgz", + "integrity": "sha512-HAjVfDa73pFgivViHyDu8HHHcds+W4MgOuZZAdyFJrHS8ngtCXmhl4hc2YXqSOwO6Bsa+iF2Sgxb2+gv874VOQ==", + "dev": true, + "requires": { + "@types/retry": "*" + } + }, "@types/proxy-from-env": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/proxy-from-env/-/proxy-from-env-1.0.1.tgz", @@ -1337,6 +1346,12 @@ "@types/node": "*" } }, + "@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true + }, "@types/rimraf": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.0.tgz", @@ -4226,8 +4241,7 @@ "graceful-fs": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", - "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", - "dev": true + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==" }, "har-schema": { "version": "2.0.0", @@ -6066,6 +6080,16 @@ "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", "dev": true }, + "proper-lockfile": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.1.tgz", + "integrity": "sha512-1w6rxXodisVpn7QYvLk706mzprPTAPCYAqxMvctmPN3ekuRk/kuGkGc82pangZiAt4R3lwSuUzheTTn0/Yb7Zg==", + "requires": { + "graceful-fs": "^4.1.11", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", @@ -6445,6 +6469,11 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, + "retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" + }, "reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -6641,6 +6670,11 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, "slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", diff --git a/package.json b/package.json index 884d309d44..34018dbca1 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "mime": "^2.4.6", "pngjs": "^5.0.0", "progress": "^2.0.3", + "proper-lockfile": "^4.1.1", "proxy-from-env": "^1.1.0", "rimraf": "^3.0.2", "ws": "^7.3.1" @@ -55,6 +56,7 @@ "@types/node": "^10.17.28", "@types/pngjs": "^3.4.2", "@types/progress": "^2.0.3", + "@types/proper-lockfile": "^4.1.1", "@types/proxy-from-env": "^1.0.1", "@types/rimraf": "^3.0.0", "@types/ws": "7.2.6", diff --git a/src/install/installer.ts b/src/install/installer.ts index 23031e566c..08739b0873 100644 --- a/src/install/installer.ts +++ b/src/install/installer.ts @@ -19,6 +19,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as util from 'util'; import * as removeFolder from 'rimraf'; +import * as lockfile from 'proper-lockfile'; import * as browserPaths from '../utils/browserPaths'; import * as browserFetcher from './browserFetcher'; import { getFromENV } from '../utils/utils'; @@ -32,16 +33,29 @@ const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs)); const removeFolderAsync = util.promisify(removeFolder); export async function installBrowsersWithProgressBar(packagePath: string) { - const browsersPath = browserPaths.browsersPath(packagePath); - const linksDir = path.join(browsersPath, '.links'); - if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) { browserFetcher.logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); return false; } + + const browsersPath = browserPaths.browsersPath(packagePath); + await fsMkdirAsync(browsersPath, { recursive: true }); + const releaseLock = await lockfile.lock(browsersPath, { + retries: { + retries: 10, + // Retry 20 times during 10 minutes with + // exponential back-off. + // See documentation at: https://www.npmjs.com/package/retry#retrytimeoutsoptions + factor: 1.27579, + }, + lockfilePath: path.join(browsersPath, '__dirlock'), + }); + const linksDir = path.join(browsersPath, '.links'); + await fsMkdirAsync(linksDir, { recursive: true }); await fsWriteFileAsync(path.join(linksDir, sha1(packagePath)), packagePath); await validateCache(packagePath, browsersPath, linksDir); + await releaseLock(); } async function validateCache(packagePath: string, browsersPath: string, linksDir: string) {