microsoft
/
vscode-dev-containers
зеркало из https://github.com/microsoft/vscode-dev-containers.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Improve CG processing of go, linux packages (CI Ignore)

This commit is contained in:
Chuck Lantz 2021-02-16 00:15:55 +00:00 коммит произвёл GitHub
Родитель e360c12b7e
Коммит 8a9c8e9a72
6 изменённых файлов: 151 добавлений и 253 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

12
build/config.json
Просмотреть файл

@ -21,6 +21,18 @@
"codespaces-linux-stretch"
],
"poolKeys": {
"https://dl.yarnpkg.com/debian": "https://dl.yarnpkg.com/debian/pubkey.gpg",
"https://packages.microsoft.com/repos/microsoft-debian-stretch-prod": "https://packages.microsoft.com/keys/microsoft.asc",
"https://packages.microsoft.com/repos/microsoft-debian-buster-prod": "https://packages.microsoft.com/keys/microsoft.asc",
"https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod": "https://packages.microsoft.com/keys/microsoft.asc",
"https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod": "https://packages.microsoft.com/keys/microsoft.asc",
"http://archive.ubuntu.com/ubuntu": "http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg",
"http://security.ubuntu.com/ubuntu": "http://security.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg",
"https://packagecloud.io/github/git-lfs/ubuntu": "https://packagecloud.io/github/git-lfs/gpgkey",
"https://packagecloud.io/github/git-lfs/debian": "https://packagecloud.io/github/git-lfs/gpgkey"
},
"githubRepoName": "microsoft/vscode-dev-containers",
"containersPathInRepo": "containers",
"repoContainersToBuildPath": "repository-containers/images",

152
build/src/cgmanifest.js
Просмотреть файл

@ -3,6 +3,8 @@ const push = require('./push').push;
const asyncUtils = require('./utils/async');
const configUtils = require('./utils/config');
//TODO: Generate markdown of versions per image that can then be stored with the definitions on release as a different output format.
// Example manifest: https://dev.azure.com/mseng/AzureDevOps/_git/Governance.Specs?path=%2Fcgmanifest.json&version=GBusers%2Fcajone%2Fcgmanifest.json
// Docker images and native OS libraries need to be registered as "other" while others are scenario dependant
const dependencyLookupConfig = {
@ -135,11 +137,25 @@ async function getDefinitionManifest(repo, release, registry, registryPath, defi
await generatePipComponentList(dependencies.pip, imageTag, alreadyRegistered),
await generatePipComponentList(dependencies.pipx, imageTag, alreadyRegistered, true),
await generateGemComponentList(dependencies.gem, imageTag, alreadyRegistered),
await generateGoComponentList(dependencies.go, imageTag, alreadyRegistered),
await generateCargoComponentList(dependencies.cargo, imageTag, alreadyRegistered),
await generateOtherComponentList(dependencies.other, imageTag, alreadyRegistered),
filteredManualComponentRegistrations(dependencies.manual, alreadyRegistered));
}
/* Generate "Linux" entry for linux packages. E.g.
{
"Component": {
"Type": "linux",
"Linux": {
"Name": "yarn",
"Version": "1.22.5-1",
"Distribution": "Debian",
"Pool-URL": "https://dl.yarnpkg.com/debian",
"Key-URL": "https://dl.yarnpkg.com/debian/pubkey.gpg"
}
}
*/
async function generatePackageComponentList(config, packageList, imageTag, alreadyRegistered) {
if (!packageList) {
return [];
@ -165,74 +181,59 @@ async function generatePackageComponentList(config, packageList, imageTag, alrea
const packageVersionList = packageVersionListOutput.split('\n');
packageVersionList.forEach((packageVersion) => {
packageVersion = packageVersion.trim();
if (packageVersion !== '') {
const versionCaptureGroup = new RegExp(config.lineRegEx).exec(packageVersion);
if (!versionCaptureGroup) {
console.log(`(!) Warning: Unable to parse output "${packageVersion}". Likely not from command. Skipping.`);
} else {
const package = versionCaptureGroup[1];
const version = versionCaptureGroup[2];
const entry = createEntryForLinuxPackage(packageUriCommandOutput, config.namePrefix, package, version, alreadyRegistered, config.uriMatchRegex, config.uriSuffix)
if (entry) {
componentList.push(entry);
}
return;
}
const package = versionCaptureGroup[1];
const version = versionCaptureGroup[2];
const uniquePackageName = `${package}-${config.distro}`;
if (alreadyRegistered[uniquePackageName] && alreadyRegistered[uniquePackageName].indexOf(version) > -1) {
return;
}
alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || [];
alreadyRegistered[uniquePackageName].push(version);
const poolUrl = getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version)
componentList.push({
"Component": {
"Type": "linux",
"Linux": {
"Name": package,
"Version": version,
"Distribution": config.distro,
"Pool-URL": poolUrl,
"Pool-Key": configUtils.getPoolKeyForPoolUrl(poolUrl)
}
}
});
}
});
return componentList;
}
function getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version) {
// Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu
const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.');
const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a');
const uriCaptureGroup = new RegExp(
config.uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm')
.exec(packageUriCommandOutput);
/* Generate "Other" entry for linux packages. E.g.
{
"Component": {
"Type": "other",
"Other": {
"Name": "Debian Package: apt-transport-https",
"Version": "1.8.2.1",
"DownloadUrl": "http://deb.debian.org/debian/pool/main/a/apt/apt-transport-https_1.8.2.1_all.deb"
}
}
}
*/
function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, package, version, alreadyRegistered, uriMatchRegex, uriSuffix) {
const uniquePackageName = `${entryNamePrefix} ${package}`;
if (typeof alreadyRegistered[uniquePackageName] === 'undefined'
|| alreadyRegistered[uniquePackageName].indexOf(version) < 0) {
let downloadUrl = 'https://no-url-available'
// Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu
const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.');
const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a');
const uriCaptureGroup = new RegExp(
uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm')
.exec(packageUriCommandOutput);
if (!uriCaptureGroup) {
console.log(`(!) No URI found for ${package} ${version}`);
} else {
const uriString = uriCaptureGroup ? uriCaptureGroup[1] : '';
alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || [];
alreadyRegistered[uniquePackageName].push(version);
downloadUrl = `${uriString}${uriSuffix ? uriSuffix.replace('${PACKAGE}', package).replace('${VERSION}', version) : ''}`
}
return {
"Component": {
"Type": "other",
"Other": {
"Name": uniquePackageName,
"Version": version,
"DownloadUrl": downloadUrl
}
}
}
if (!uriCaptureGroup) {
console.log(`(!) No URI found for ${package} ${version}`);
throw new Error('No download URI found for package');
}
return null;
// Extract URIs
const uriString = uriCaptureGroup ? uriCaptureGroup[1] : '';
const poolUrlCaptureGroup = /(.+)\/pool\//.exec(uriString);
if (!poolUrlCaptureGroup) {
console.log(`(!) No pool found for ${package} ${version}`);
throw new Error('No pool URL found for package');
}
return poolUrlCaptureGroup[1];
}
/* Generate "Npm" entries. E.g.
@ -246,6 +247,7 @@ function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, pa
}
}
*/
//TODO: Get version info from inside container - this works for dev images as registered now, but not looking at older ones
async function generateNpmComponentList(packageList, alreadyRegistered) {
if (!packageList) {
return [];
@ -510,7 +512,47 @@ async function generateCargoComponentList(crates, imageTag, alreadyRegistered) {
return componentList;
}
/* Generate "Go" entries. E.g.
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/tools/gopls",
"Version": "0.6.4"
}
}
*/
async function generateGoComponentList(packages, imageTag, alreadyRegistered) {
if (!packages) {
return [];
}
const componentList = [];
console.log(`(*) Generating "Go" registrations...`);
const packageInstallOutput = await getDockerRunCommandOutput(imageTag, "cat /usr/local/etc/vscode-dev-containers/go.log");
for(let package in packages) {
console.log(`(*) Getting version for ${package}...`);
const versionCommand = packages[package];
let version;
if(versionCommand) {
version = await getDockerRunCommandOutput(imageTag, versionCommand);
} else {
const versionCaptureGroup = new RegExp(`${package}\\s.*v([0-9]+\\.[0-9]+\\.[0-9]+.*)\\n`,'m').exec(packageInstallOutput);
version = versionCaptureGroup ? versionCaptureGroup[1] : 'latest';
}
addIfUnique(`${package}-go`, version, componentList, alreadyRegistered, {
"Component": {
"Type": "Go",
"Go": {
"Name": package,
"Version": version,
}
}
});
}
return componentList;
}
function addIfUnique(uniqueName, uniqueVersion, componentList, alreadyRegistered, componentJson) {
if (typeof alreadyRegistered[uniqueName] === 'undefined' || alreadyRegistered[uniqueName].indexOf(uniqueVersion) < 0) {

7
build/src/utils/config.js
Просмотреть файл

@ -506,6 +506,12 @@ function getAllDependencies() {
return config.definitionDependencies;
}
function getPoolKeyForPoolUrl(poolUrl) {
const poolKey = config.poolKeys[poolUrl];
console.log (`(*) Key for ${poolUrl} is ${poolKey}`);
return poolKey;
}
async function getStagingFolder(release) {
if (!stagingFolders[release]) {
const stagingFolder = path.join(os.tmpdir(), 'vscode-dev-containers', release);
@ -544,6 +550,7 @@ module.exports = {
getLinuxDistroForDefinition: getLinuxDistroForDefinition,
getVersionFromRelease: getVersionFromRelease,
getTagsForVersion: getTagsForVersion,
getPoolKeyForPoolUrl: getPoolKeyForPoolUrl,
getConfig: getConfig,
shouldFlattenDefinitionBaseImage: shouldFlattenDefinitionBaseImage
};

78
containers/codespaces-linux-stretch/definition-manifest.json
Просмотреть файл

@ -15,7 +15,8 @@
"imageLink": "https://github.com/microsoft/oryx",
"debian": [
"git-lfs",
"moby-cli"
"moby-cli",
"powershell"
],
"pipx": [
"pylint",
@ -104,70 +105,15 @@
"downloadUrl": "https://www.ruby-lang.org/en/downloads/"
}
},
"manual": [{
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/tools/gopls",
"Version": "0.6.4"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "honnef.co/go/tools",
"Version": "0.1.1"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/lint/golint",
"Version": "0.0.0-20201208152925-83fdc39ff7b5"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/mgechev/revive",
"Version": "1.0.3"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
"Version": "2.1.2"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/ramya-rao-a/go-outline",
"Version": "0.0.0-20200117021646-2a048b4510eb"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/go-delve/delve/cmd/dlv",
"Version": "1.6.0"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/golangci/golangci-lint",
"Version": "1.36.0"
}
}
}]
"go": {
"golang.org/x/tools/gopls": null,
"honnef.co/go/tools/...": null,
"golang.org/x/lint/golint": null,
"github.com/mgechev/revive": null,
"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
"github.com/ramya-rao-a/go-outline": null,
"github.com/go-delve/delve/cmd/dlv": null,
"github.com/golangci/golangci-lint/cmd/golangci-lint": null
}
}
}

78
containers/codespaces-linux/definition-manifest.json
Просмотреть файл

@ -15,7 +15,8 @@
"debian": [
"git-lfs",
"moby-cli",
"moby-engine"
"moby-engine",
"powershell"
],
"pipx": [
"pylint",
@ -100,70 +101,15 @@
"downloadUrl": "https://github.com/rvm/rvm"
}
},
"manual": [{
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/tools/gopls",
"Version": "0.6.4"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "honnef.co/go/tools",
"Version": "0.1.1"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/lint/golint",
"Version": "0.0.0-20201208152925-83fdc39ff7b5"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/mgechev/revive",
"Version": "1.0.3"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
"Version": "2.1.2"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/ramya-rao-a/go-outline",
"Version": "0.0.0-20200117021646-2a048b4510eb"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/go-delve/delve/cmd/dlv",
"Version": "1.6.0"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/golangci/golangci-lint",
"Version": "1.36.0"
}
}
}]
"go": {
"golang.org/x/tools/gopls": null,
"honnef.co/go/tools/...": null,
"golang.org/x/lint/golint": null,
"github.com/mgechev/revive": null,
"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
"github.com/ramya-rao-a/go-outline": null,
"github.com/go-delve/delve/cmd/dlv": null,
"github.com/golangci/golangci-lint/cmd/golangci-lint": null
}
}
}

75
containers/go/definition-manifest.json
Просмотреть файл

@ -19,70 +19,15 @@
"Oh My Bash!": "/root/.oh-my-bash",
"nvm": "/usr/local/share/nvm"
},
"manual": [{
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/tools/gopls",
"Version": "0.6.4"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "honnef.co/go/tools",
"Version": "0.1.1"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "golang.org/x/lint/golint",
"Version": "0.0.0-20201208152925-83fdc39ff7b5"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/mgechev/revive",
"Version": "1.0.3"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
"Version": "2.1.2"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/ramya-rao-a/go-outline",
"Version": "0.0.0-20200117021646-2a048b4510eb"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/go-delve/delve/cmd/dlv",
"Version": "1.6.0"
}
}
}, {
"Component": {
"Type": "go",
"Go": {
"Name": "github.com/golangci/golangci-lint",
"Version": "1.36.0"
}
}
}]
"go": {
"golang.org/x/tools/gopls": null,
"honnef.co/go/tools/...": null,
"golang.org/x/lint/golint": null,
"github.com/mgechev/revive": null,
"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
"github.com/ramya-rao-a/go-outline": null,
"github.com/go-delve/delve/cmd/dlv": null,
"github.com/golangci/golangci-lint/cmd/golangci-lint": null
}
}
}