From 8a9c8e9a720d86660d7645c5ad31e9d4331f15e5 Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Tue, 16 Feb 2021 00:15:55 +0000 Subject: [PATCH] Improve CG processing of go, linux packages (CI Ignore) --- build/config.json | 14 +- build/src/cgmanifest.js | 152 +++++++++++------- build/src/utils/config.js | 7 + .../definition-manifest.json | 78 ++------- .../codespaces-linux/definition-manifest.json | 78 ++------- containers/go/definition-manifest.json | 75 ++------- 6 files changed, 151 insertions(+), 253 deletions(-) diff --git a/build/config.json b/build/config.json index 99e30cf9..266b605c 100644 --- a/build/config.json +++ b/build/config.json @@ -11,7 +11,7 @@ "yarn.lock", ".npmignore" ], - + "needsDedicatedPage": [ "codespaces-linux", "codespaces-linux-stretch" @@ -21,6 +21,18 @@ "codespaces-linux-stretch" ], + "poolKeys": { + "https://dl.yarnpkg.com/debian": "https://dl.yarnpkg.com/debian/pubkey.gpg", + "https://packages.microsoft.com/repos/microsoft-debian-stretch-prod": "https://packages.microsoft.com/keys/microsoft.asc", + "https://packages.microsoft.com/repos/microsoft-debian-buster-prod": "https://packages.microsoft.com/keys/microsoft.asc", + "https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod": "https://packages.microsoft.com/keys/microsoft.asc", + "https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod": "https://packages.microsoft.com/keys/microsoft.asc", + "http://archive.ubuntu.com/ubuntu": "http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg", + "http://security.ubuntu.com/ubuntu": "http://security.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg", + "https://packagecloud.io/github/git-lfs/ubuntu": "https://packagecloud.io/github/git-lfs/gpgkey", + "https://packagecloud.io/github/git-lfs/debian": "https://packagecloud.io/github/git-lfs/gpgkey" + }, + "githubRepoName": "microsoft/vscode-dev-containers", "containersPathInRepo": "containers", "repoContainersToBuildPath": "repository-containers/images", diff --git a/build/src/cgmanifest.js b/build/src/cgmanifest.js index 80132110..97a61c90 100644 --- a/build/src/cgmanifest.js +++ b/build/src/cgmanifest.js @@ -3,6 +3,8 @@ const push = require('./push').push; const asyncUtils = require('./utils/async'); const configUtils = require('./utils/config'); +//TODO: Generate markdown of versions per image that can then be stored with the definitions on release as a different output format. + // Example manifest: https://dev.azure.com/mseng/AzureDevOps/_git/Governance.Specs?path=%2Fcgmanifest.json&version=GBusers%2Fcajone%2Fcgmanifest.json // Docker images and native OS libraries need to be registered as "other" while others are scenario dependant const dependencyLookupConfig = { @@ -135,11 +137,25 @@ async function getDefinitionManifest(repo, release, registry, registryPath, defi await generatePipComponentList(dependencies.pip, imageTag, alreadyRegistered), await generatePipComponentList(dependencies.pipx, imageTag, alreadyRegistered, true), await generateGemComponentList(dependencies.gem, imageTag, alreadyRegistered), + await generateGoComponentList(dependencies.go, imageTag, alreadyRegistered), await generateCargoComponentList(dependencies.cargo, imageTag, alreadyRegistered), await generateOtherComponentList(dependencies.other, imageTag, alreadyRegistered), filteredManualComponentRegistrations(dependencies.manual, alreadyRegistered)); } +/* Generate "Linux" entry for linux packages. E.g. +{ + "Component": { + "Type": "linux", + "Linux": { + "Name": "yarn", + "Version": "1.22.5-1", + "Distribution": "Debian", + "Pool-URL": "https://dl.yarnpkg.com/debian", + "Key-URL": "https://dl.yarnpkg.com/debian/pubkey.gpg" + } +} + */ async function generatePackageComponentList(config, packageList, imageTag, alreadyRegistered) { if (!packageList) { return []; @@ -165,74 +181,59 @@ async function generatePackageComponentList(config, packageList, imageTag, alrea const packageVersionList = packageVersionListOutput.split('\n'); packageVersionList.forEach((packageVersion) => { packageVersion = packageVersion.trim(); - if (packageVersion !== '') { const versionCaptureGroup = new RegExp(config.lineRegEx).exec(packageVersion); if (!versionCaptureGroup) { console.log(`(!) Warning: Unable to parse output "${packageVersion}". Likely not from command. Skipping.`); - } else { - const package = versionCaptureGroup[1]; - const version = versionCaptureGroup[2]; - const entry = createEntryForLinuxPackage(packageUriCommandOutput, config.namePrefix, package, version, alreadyRegistered, config.uriMatchRegex, config.uriSuffix) - if (entry) { - componentList.push(entry); - } + return; } + const package = versionCaptureGroup[1]; + const version = versionCaptureGroup[2]; + const uniquePackageName = `${package}-${config.distro}`; + if (alreadyRegistered[uniquePackageName] && alreadyRegistered[uniquePackageName].indexOf(version) > -1) { + return; + } + alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || []; + alreadyRegistered[uniquePackageName].push(version); + const poolUrl = getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version) + componentList.push({ + "Component": { + "Type": "linux", + "Linux": { + "Name": package, + "Version": version, + "Distribution": config.distro, + "Pool-URL": poolUrl, + "Pool-Key": configUtils.getPoolKeyForPoolUrl(poolUrl) + } + } + }); } }); return componentList; } +function getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version) { + // Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu + const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.'); + const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a'); + const uriCaptureGroup = new RegExp( + config.uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm') + .exec(packageUriCommandOutput); -/* Generate "Other" entry for linux packages. E.g. -{ - "Component": { - "Type": "other", - "Other": { - "Name": "Debian Package: apt-transport-https", - "Version": "1.8.2.1", - "DownloadUrl": "http://deb.debian.org/debian/pool/main/a/apt/apt-transport-https_1.8.2.1_all.deb" - } - } -} -*/ -function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, package, version, alreadyRegistered, uriMatchRegex, uriSuffix) { - const uniquePackageName = `${entryNamePrefix} ${package}`; - if (typeof alreadyRegistered[uniquePackageName] === 'undefined' - || alreadyRegistered[uniquePackageName].indexOf(version) < 0) { - - let downloadUrl = 'https://no-url-available' - - // Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu - const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.'); - const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a'); - const uriCaptureGroup = new RegExp( - uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm') - .exec(packageUriCommandOutput); - - if (!uriCaptureGroup) { - console.log(`(!) No URI found for ${package} ${version}`); - } else { - const uriString = uriCaptureGroup ? uriCaptureGroup[1] : ''; - alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || []; - alreadyRegistered[uniquePackageName].push(version); - downloadUrl = `${uriString}${uriSuffix ? uriSuffix.replace('${PACKAGE}', package).replace('${VERSION}', version) : ''}` - } - - - return { - "Component": { - "Type": "other", - "Other": { - "Name": uniquePackageName, - "Version": version, - "DownloadUrl": downloadUrl - } - } - } + if (!uriCaptureGroup) { + console.log(`(!) No URI found for ${package} ${version}`); + throw new Error('No download URI found for package'); } - return null; + // Extract URIs + const uriString = uriCaptureGroup ? uriCaptureGroup[1] : ''; + const poolUrlCaptureGroup = /(.+)\/pool\//.exec(uriString); + if (!poolUrlCaptureGroup) { + console.log(`(!) No pool found for ${package} ${version}`); + throw new Error('No pool URL found for package'); + } + return poolUrlCaptureGroup[1]; } /* Generate "Npm" entries. E.g. @@ -246,6 +247,7 @@ function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, pa } } */ +//TODO: Get version info from inside container - this works for dev images as registered now, but not looking at older ones async function generateNpmComponentList(packageList, alreadyRegistered) { if (!packageList) { return []; @@ -510,7 +512,47 @@ async function generateCargoComponentList(crates, imageTag, alreadyRegistered) { return componentList; } +/* Generate "Go" entries. E.g. +"Component": { + "Type": "go", + "Go": { + "Name": "golang.org/x/tools/gopls", + "Version": "0.6.4" + } +} +*/ +async function generateGoComponentList(packages, imageTag, alreadyRegistered) { + if (!packages) { + return []; + } + const componentList = []; + console.log(`(*) Generating "Go" registrations...`); + + const packageInstallOutput = await getDockerRunCommandOutput(imageTag, "cat /usr/local/etc/vscode-dev-containers/go.log"); + for(let package in packages) { + console.log(`(*) Getting version for ${package}...`); + const versionCommand = packages[package]; + let version; + if(versionCommand) { + version = await getDockerRunCommandOutput(imageTag, versionCommand); + } else { + const versionCaptureGroup = new RegExp(`${package}\\s.*v([0-9]+\\.[0-9]+\\.[0-9]+.*)\\n`,'m').exec(packageInstallOutput); + version = versionCaptureGroup ? versionCaptureGroup[1] : 'latest'; + } + addIfUnique(`${package}-go`, version, componentList, alreadyRegistered, { + "Component": { + "Type": "Go", + "Go": { + "Name": package, + "Version": version, + } + } + }); + } + + return componentList; +} function addIfUnique(uniqueName, uniqueVersion, componentList, alreadyRegistered, componentJson) { if (typeof alreadyRegistered[uniqueName] === 'undefined' || alreadyRegistered[uniqueName].indexOf(uniqueVersion) < 0) { diff --git a/build/src/utils/config.js b/build/src/utils/config.js index 8c6e59e8..e7709782 100644 --- a/build/src/utils/config.js +++ b/build/src/utils/config.js @@ -506,6 +506,12 @@ function getAllDependencies() { return config.definitionDependencies; } +function getPoolKeyForPoolUrl(poolUrl) { + const poolKey = config.poolKeys[poolUrl]; + console.log (`(*) Key for ${poolUrl} is ${poolKey}`); + return poolKey; +} + async function getStagingFolder(release) { if (!stagingFolders[release]) { const stagingFolder = path.join(os.tmpdir(), 'vscode-dev-containers', release); @@ -544,6 +550,7 @@ module.exports = { getLinuxDistroForDefinition: getLinuxDistroForDefinition, getVersionFromRelease: getVersionFromRelease, getTagsForVersion: getTagsForVersion, + getPoolKeyForPoolUrl: getPoolKeyForPoolUrl, getConfig: getConfig, shouldFlattenDefinitionBaseImage: shouldFlattenDefinitionBaseImage }; diff --git a/containers/codespaces-linux-stretch/definition-manifest.json b/containers/codespaces-linux-stretch/definition-manifest.json index 9e99660d..4b4eca39 100644 --- a/containers/codespaces-linux-stretch/definition-manifest.json +++ b/containers/codespaces-linux-stretch/definition-manifest.json @@ -15,7 +15,8 @@ "imageLink": "https://github.com/microsoft/oryx", "debian": [ "git-lfs", - "moby-cli" + "moby-cli", + "powershell" ], "pipx": [ "pylint", @@ -104,70 +105,15 @@ "downloadUrl": "https://www.ruby-lang.org/en/downloads/" } }, - "manual": [{ - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/tools/gopls", - "Version": "0.6.4" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "honnef.co/go/tools", - "Version": "0.1.1" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/lint/golint", - "Version": "0.0.0-20201208152925-83fdc39ff7b5" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/mgechev/revive", - "Version": "1.0.3" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs", - "Version": "2.1.2" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/ramya-rao-a/go-outline", - "Version": "0.0.0-20200117021646-2a048b4510eb" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/go-delve/delve/cmd/dlv", - "Version": "1.6.0" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/golangci/golangci-lint", - "Version": "1.36.0" - } - } - }] + "go": { + "golang.org/x/tools/gopls": null, + "honnef.co/go/tools/...": null, + "golang.org/x/lint/golint": null, + "github.com/mgechev/revive": null, + "github.com/uudashr/gopkgs/v2/cmd/gopkgs": null, + "github.com/ramya-rao-a/go-outline": null, + "github.com/go-delve/delve/cmd/dlv": null, + "github.com/golangci/golangci-lint/cmd/golangci-lint": null + } } } diff --git a/containers/codespaces-linux/definition-manifest.json b/containers/codespaces-linux/definition-manifest.json index 9bb70d8e..74270120 100644 --- a/containers/codespaces-linux/definition-manifest.json +++ b/containers/codespaces-linux/definition-manifest.json @@ -15,7 +15,8 @@ "debian": [ "git-lfs", "moby-cli", - "moby-engine" + "moby-engine", + "powershell" ], "pipx": [ "pylint", @@ -100,70 +101,15 @@ "downloadUrl": "https://github.com/rvm/rvm" } }, - "manual": [{ - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/tools/gopls", - "Version": "0.6.4" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "honnef.co/go/tools", - "Version": "0.1.1" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/lint/golint", - "Version": "0.0.0-20201208152925-83fdc39ff7b5" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/mgechev/revive", - "Version": "1.0.3" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs", - "Version": "2.1.2" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/ramya-rao-a/go-outline", - "Version": "0.0.0-20200117021646-2a048b4510eb" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/go-delve/delve/cmd/dlv", - "Version": "1.6.0" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/golangci/golangci-lint", - "Version": "1.36.0" - } - } - }] + "go": { + "golang.org/x/tools/gopls": null, + "honnef.co/go/tools/...": null, + "golang.org/x/lint/golint": null, + "github.com/mgechev/revive": null, + "github.com/uudashr/gopkgs/v2/cmd/gopkgs": null, + "github.com/ramya-rao-a/go-outline": null, + "github.com/go-delve/delve/cmd/dlv": null, + "github.com/golangci/golangci-lint/cmd/golangci-lint": null + } } } diff --git a/containers/go/definition-manifest.json b/containers/go/definition-manifest.json index 2dd262f4..e57581bb 100644 --- a/containers/go/definition-manifest.json +++ b/containers/go/definition-manifest.json @@ -19,70 +19,15 @@ "Oh My Bash!": "/root/.oh-my-bash", "nvm": "/usr/local/share/nvm" }, - "manual": [{ - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/tools/gopls", - "Version": "0.6.4" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "honnef.co/go/tools", - "Version": "0.1.1" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "golang.org/x/lint/golint", - "Version": "0.0.0-20201208152925-83fdc39ff7b5" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/mgechev/revive", - "Version": "1.0.3" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs", - "Version": "2.1.2" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/ramya-rao-a/go-outline", - "Version": "0.0.0-20200117021646-2a048b4510eb" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/go-delve/delve/cmd/dlv", - "Version": "1.6.0" - } - } - }, { - "Component": { - "Type": "go", - "Go": { - "Name": "github.com/golangci/golangci-lint", - "Version": "1.36.0" - } - } - }] + "go": { + "golang.org/x/tools/gopls": null, + "honnef.co/go/tools/...": null, + "golang.org/x/lint/golint": null, + "github.com/mgechev/revive": null, + "github.com/uudashr/gopkgs/v2/cmd/gopkgs": null, + "github.com/ramya-rao-a/go-outline": null, + "github.com/go-delve/delve/cmd/dlv": null, + "github.com/golangci/golangci-lint/cmd/golangci-lint": null + } } }