From d38fb68cdc436a6d07c4c3aef59564bc1aa7500b Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Wed, 24 Feb 2021 06:51:51 -0800 Subject: [PATCH] Tweak version extraction --- build/config.json | 7 +++++++ build/src/cgmanifest.js | 35 +++++++++++++++++++---------------- build/src/utils/config.js | 8 ++++++++ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/build/config.json b/build/config.json index 266b605c..fbac8d9e 100644 --- a/build/config.json +++ b/build/config.json @@ -33,6 +33,13 @@ "https://packagecloud.io/github/git-lfs/debian": "https://packagecloud.io/github/git-lfs/gpgkey" }, + "poolUrlFallback": { + "moby-cli": "https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod", + "moby-engine": "https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod", + "yarn": "https://dl.yarnpkg.com/debian", + "git-lfs": "https://packagecloud.io/github/git-lfs/ubuntu" + }, + "githubRepoName": "microsoft/vscode-dev-containers", "containersPathInRepo": "containers", "repoContainersToBuildPath": "repository-containers/images", diff --git a/build/src/cgmanifest.js b/build/src/cgmanifest.js index 97a61c90..f89cd6c4 100644 --- a/build/src/cgmanifest.js +++ b/build/src/cgmanifest.js @@ -12,21 +12,25 @@ const dependencyLookupConfig = { // Command to get package versions: dpkg-query --show -f='${Package}\t${Version}\n' // Output: // Command to get download URLs: apt-get update && apt-get install -y --reinstall --print-uris + // Output: Multi-line output, but each line is '.deb' __.deb namePrefix: 'Debian Package:', listCommand: "dpkg-query --show -f='${Package}\\t${Version}\\n'", lineRegEx: /(.+)\t(.+)/, getUriCommand: 'apt-get update && apt-get install -y --reinstall --print-uris', - uriMatchRegex: "'(.+)'\\s*${PACKAGE}_${VERSION}" + downloadUriMatchRegEx: "'(.+\\.deb)'\\s*${PACKAGE}_.+\\s", + poolUriMatchRegEx: "'(.+)/pool.+\\.deb'\\s*${PACKAGE}_.+\\s" }, ubuntu: { // Command to get package versions: dpkg-query --show -f='${Package}\t${Version}\n' // Output: // Command to get download URLs: apt-get update && apt-get install -y --reinstall --print-uris + // Output: Multi-line output, but each line is '.deb' __.deb namePrefix: 'Ubuntu Package:', listCommand: "dpkg-query --show -f='${Package}\\t${Version}\\n'", lineRegEx: /(.+)\t(.+)/, getUriCommand: 'apt-get update && apt-get install -y --reinstall --print-uris', - uriMatchRegex: "'(.+)'\\s*${PACKAGE}_${VERSION}" + downloadUriMatchRegEx: "'(.+\\.deb)'\\s*${PACKAGE}_.+\\s", + poolUriMatchRegEx: "'(.+)/pool.+\\.deb'\\s*${PACKAGE}_.+\\s" }, alpine: { // Command to get package versions: apk info -e -v @@ -36,8 +40,9 @@ const dependencyLookupConfig = { listCommand: "apk info -e -v", lineRegEx: /(.+)-([0-9].+)/, getUriCommand: 'apk update && apk policy', - uriMatchRegex: '${PACKAGE} policy:\\n.*${VERSION}:\\n.*lib/apk/db/installed\\n\\s*(.+)\\n', - uriSuffix: '/x86_64/${PACKAGE}-${VERSION}.apk' + downloadUriMatchRegEx: '${PACKAGE} policy:\\n.*${VERSION}:\\n.*lib/apk/db/installed\\n\\s*(.+)\\n', + downloadUriSuffix: '/x86_64/${PACKAGE}-${VERSION}.apk', + poolUriMatchRegEx: '${PACKAGE} policy:\\n.*${VERSION}:\\n.*lib/apk/db/installed\\n\\s*(.+)\\n', } } @@ -218,22 +223,20 @@ function getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.'); const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a'); const uriCaptureGroup = new RegExp( - config.uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm') + config.poolUriMatchRegEx.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm') .exec(packageUriCommandOutput); if (!uriCaptureGroup) { - console.log(`(!) No URI found for ${package} ${version}`); + console.log(`(!) No URI found for ${package} ${version}. Attempting to use fallback.`); + const fallbackPoolUrl = configUtils.getFallbackPoolUrl(package); + if (fallbackPoolUrl) { + return fallbackPoolUrl; + } throw new Error('No download URI found for package'); } // Extract URIs - const uriString = uriCaptureGroup ? uriCaptureGroup[1] : ''; - const poolUrlCaptureGroup = /(.+)\/pool\//.exec(uriString); - if (!poolUrlCaptureGroup) { - console.log(`(!) No pool found for ${package} ${version}`); - throw new Error('No pool URL found for package'); - } - return poolUrlCaptureGroup[1]; + return uriCaptureGroup[1]; } /* Generate "Npm" entries. E.g. @@ -261,7 +264,7 @@ async function generateNpmComponentList(packageList, alreadyRegistered) { if (package.indexOf('@') >= 0) { [package, version] = package.split('@'); } else { - const npmInfoRaw = await asyncUtils.spawn('npm', ['info', package, '--json'], { shell: true, stdio: 'pipe' }); + const npmInfoRaw = await asyncUtils.spawn('npm', ['info', package, '--json'], { shell: true, stdio: ['inherit', 'pipe', 'inherit'] }); const npmInfo = JSON.parse(npmInfoRaw); version = npmInfo['dist-tags'].latest; } @@ -369,7 +372,7 @@ async function generateGitComponentList(gitRepoPath, imageTag, alreadyRegistered if (typeof repoPath === 'string') { console.log(`(*) Getting remote and commit for ${repoName} at ${repoPath}...`); // Go to the specified folder, see if the commands have already been run, if not run them and get output - const remoteAndCommitOutput = await getDockerRunCommandOutput(imageTag, `cd \\"${repoPath}\\" && if [ -f \\".git-remote-and-commit\\" ]; then cat .git-remote-and-commit; else git remote get-url origin && echo $(git log -n 1 --pretty=format:%H -- .); fi`); + const remoteAndCommitOutput = await getDockerRunCommandOutput(imageTag, `cd \\"${repoPath}\\" && if [ -f \\".git-remote-and-commit\\" ]; then cat .git-remote-and-commit; else git remote get-url origin && echo $(git log -n 1 --pretty=format:%H -- .); fi`,true); const remoteAndCommit = remoteAndCommitOutput.split('\n'); const gitRemote = remoteAndCommit[0]; const gitCommit = remoteAndCommit[1]; @@ -580,7 +583,7 @@ function filteredManualComponentRegistrations(manualRegistrations, alreadyRegist } async function getDockerRunCommandOutput(imageTag, command, forceRoot) { - const wrappedCommand = `bash -c "set -e && echo ~~~BEGIN~~~ && ${command.replace(/\$/g, '\\\$')} && echo ~~~END~~~"`; + const wrappedCommand = `bash -c "set -e && echo ~~~BEGIN~~~ && ${command} && echo ~~~END~~~"`; const runArgs = ['run','--init', '--privileged', '--rm'].concat(forceRoot ? ['-u', 'root'] : []); runArgs.push(imageTag); runArgs.push(wrappedCommand); diff --git a/build/src/utils/config.js b/build/src/utils/config.js index e7709782..2ad929ce 100644 --- a/build/src/utils/config.js +++ b/build/src/utils/config.js @@ -512,6 +512,13 @@ function getPoolKeyForPoolUrl(poolUrl) { return poolKey; } +function getFallbackPoolUrl(package) { + const poolUrl = config.poolUrlFallback[package]; + console.log (`(*) Fallback pool URL for ${package} is ${poolUrl}`); + return poolUrl; +} + + async function getStagingFolder(release) { if (!stagingFolders[release]) { const stagingFolder = path.join(os.tmpdir(), 'vscode-dev-containers', release); @@ -550,6 +557,7 @@ module.exports = { getLinuxDistroForDefinition: getLinuxDistroForDefinition, getVersionFromRelease: getVersionFromRelease, getTagsForVersion: getTagsForVersion, + getFallbackPoolUrl: getFallbackPoolUrl, getPoolKeyForPoolUrl: getPoolKeyForPoolUrl, getConfig: getConfig, shouldFlattenDefinitionBaseImage: shouldFlattenDefinitionBaseImage