Improve CG processing of go, linux packages (CI Ignore)

build/config.json

@@ -11,7 +11,7 @@
 		"yarn.lock",
 		".npmignore"
 	],
-	
+
 	"needsDedicatedPage": [ 
 		"codespaces-linux",
 		"codespaces-linux-stretch" 
@@ -21,6 +21,18 @@
 		"codespaces-linux-stretch"
 	],
 
+	"poolKeys": {
+		"https://dl.yarnpkg.com/debian": "https://dl.yarnpkg.com/debian/pubkey.gpg",
+		"https://packages.microsoft.com/repos/microsoft-debian-stretch-prod": "https://packages.microsoft.com/keys/microsoft.asc",
+		"https://packages.microsoft.com/repos/microsoft-debian-buster-prod": "https://packages.microsoft.com/keys/microsoft.asc",
+		"https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod": "https://packages.microsoft.com/keys/microsoft.asc",
+		"https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod": "https://packages.microsoft.com/keys/microsoft.asc",
+		"http://archive.ubuntu.com/ubuntu": "http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg",
+		"http://security.ubuntu.com/ubuntu": "http://security.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg",
+		"https://packagecloud.io/github/git-lfs/ubuntu": "https://packagecloud.io/github/git-lfs/gpgkey",
+		"https://packagecloud.io/github/git-lfs/debian": "https://packagecloud.io/github/git-lfs/gpgkey"
+	},
+
 	"githubRepoName": "microsoft/vscode-dev-containers",
 	"containersPathInRepo": "containers",
 	"repoContainersToBuildPath": "repository-containers/images",

build/src/cgmanifest.js

@@ -3,6 +3,8 @@ const push = require('./push').push;
 const asyncUtils = require('./utils/async');
 const configUtils = require('./utils/config');
 
+//TODO: Generate markdown of versions per image that can then be stored with the definitions on release as a different output format.
+
 // Example manifest: https://dev.azure.com/mseng/AzureDevOps/_git/Governance.Specs?path=%2Fcgmanifest.json&version=GBusers%2Fcajone%2Fcgmanifest.json
 // Docker images and native OS libraries need to be registered as "other" while others are scenario dependant
 const dependencyLookupConfig = {
@@ -135,11 +137,25 @@ async function getDefinitionManifest(repo, release, registry, registryPath, defi
         await generatePipComponentList(dependencies.pip, imageTag, alreadyRegistered),
         await generatePipComponentList(dependencies.pipx, imageTag, alreadyRegistered, true),
         await generateGemComponentList(dependencies.gem, imageTag, alreadyRegistered),
+        await generateGoComponentList(dependencies.go, imageTag, alreadyRegistered),
         await generateCargoComponentList(dependencies.cargo, imageTag, alreadyRegistered),
         await generateOtherComponentList(dependencies.other, imageTag, alreadyRegistered),
         filteredManualComponentRegistrations(dependencies.manual, alreadyRegistered));
 }
 
+/* Generate "Linux" entry for linux packages. E.g.
+{
+    "Component": {
+    "Type": "linux",
+    "Linux": {
+        "Name": "yarn",
+        "Version": "1.22.5-1",
+        "Distribution": "Debian",
+        "Pool-URL": "https://dl.yarnpkg.com/debian",
+        "Key-URL": "https://dl.yarnpkg.com/debian/pubkey.gpg"
+    }
+}
+ */   
 async function generatePackageComponentList(config, packageList, imageTag, alreadyRegistered) {
     if (!packageList) {
         return [];
@@ -165,74 +181,59 @@ async function generatePackageComponentList(config, packageList, imageTag, alrea
     const packageVersionList = packageVersionListOutput.split('\n');
     packageVersionList.forEach((packageVersion) => {
         packageVersion = packageVersion.trim();
-
         if (packageVersion !== '') {
             const versionCaptureGroup = new RegExp(config.lineRegEx).exec(packageVersion);
             if (!versionCaptureGroup) {
                 console.log(`(!) Warning: Unable to parse output "${packageVersion}". Likely not from command. Skipping.`);
-            } else {
+                return;
-                const package = versionCaptureGroup[1];
-                const version = versionCaptureGroup[2];    
-                const entry = createEntryForLinuxPackage(packageUriCommandOutput, config.namePrefix, package, version, alreadyRegistered, config.uriMatchRegex, config.uriSuffix)
-                if (entry) {
-                    componentList.push(entry);
-                }    
             }
+            const package = versionCaptureGroup[1];
+            const version = versionCaptureGroup[2];
+            const uniquePackageName = `${package}-${config.distro}`;
+            if (alreadyRegistered[uniquePackageName] && alreadyRegistered[uniquePackageName].indexOf(version) > -1) {
+                return;
+            }
+            alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || [];
+            alreadyRegistered[uniquePackageName].push(version);
+            const poolUrl = getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version)
+            componentList.push({
+                "Component": {
+                    "Type": "linux",
+                    "Linux": {
+                        "Name": package,
+                        "Version": version,
+                        "Distribution": config.distro,
+                        "Pool-URL": poolUrl,
+                        "Pool-Key": configUtils.getPoolKeyForPoolUrl(poolUrl)
+                    }
+                }
+            });
         }
     });
 
     return componentList;
 }
+function getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version) {
+    // Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu 
+    const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.');
+    const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a');
+    const uriCaptureGroup = new RegExp(
+        config.uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm')
+        .exec(packageUriCommandOutput);
 
-/* Generate "Other" entry for linux packages. E.g.
+    if (!uriCaptureGroup) {
-{
+        console.log(`(!) No URI found for ${package} ${version}`);
-    "Component": {
+        throw new Error('No download URI found for package');
-        "Type": "other",
-        "Other": {
-            "Name": "Debian Package: apt-transport-https",
-            "Version": "1.8.2.1",
-            "DownloadUrl": "http://deb.debian.org/debian/pool/main/a/apt/apt-transport-https_1.8.2.1_all.deb"
-        }
-    }
-}
-*/
-function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, package, version, alreadyRegistered, uriMatchRegex, uriSuffix) {
-    const uniquePackageName = `${entryNamePrefix} ${package}`;
-    if (typeof alreadyRegistered[uniquePackageName] === 'undefined'
-        || alreadyRegistered[uniquePackageName].indexOf(version) < 0) {
-
-        let downloadUrl = 'https://no-url-available'
-
-        // Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu 
-        const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.');
-        const sanitizedVersion = version.replace(/\+/g, '\\+').replace(/\./g, '\\.').replace(/:/g, '%3a');
-        const uriCaptureGroup = new RegExp(
-            uriMatchRegex.replace('${PACKAGE}', sanitizedPackage).replace('${VERSION}', sanitizedVersion), 'm')
-            .exec(packageUriCommandOutput);
-
-        if (!uriCaptureGroup) {
-            console.log(`(!) No URI found for ${package} ${version}`);
-        } else {
-            const uriString = uriCaptureGroup ? uriCaptureGroup[1] : '';
-            alreadyRegistered[uniquePackageName] = alreadyRegistered[uniquePackageName] || [];
-            alreadyRegistered[uniquePackageName].push(version);
-            downloadUrl = `${uriString}${uriSuffix ? uriSuffix.replace('${PACKAGE}', package).replace('${VERSION}', version) : ''}`    
-        }
-
-
-        return {
-            "Component": {
-                "Type": "other",
-                "Other": {
-                    "Name": uniquePackageName,
-                    "Version": version,
-                    "DownloadUrl": downloadUrl
-                }
-            }
-        }
     }
 
-    return null;
+    // Extract URIs
+    const uriString = uriCaptureGroup ? uriCaptureGroup[1] : '';
+    const poolUrlCaptureGroup = /(.+)\/pool\//.exec(uriString);
+    if (!poolUrlCaptureGroup) {
+        console.log(`(!) No pool found for ${package} ${version}`);
+        throw new Error('No pool URL found for package');
+    }
+    return poolUrlCaptureGroup[1];
 }
 
 /* Generate "Npm" entries. E.g.
@@ -246,6 +247,7 @@ function createEntryForLinuxPackage(packageUriCommandOutput, entryNamePrefix, pa
     }
 }
 */
+//TODO: Get version info from inside container - this works for dev images as registered now, but not looking at older ones
 async function generateNpmComponentList(packageList, alreadyRegistered) {
     if (!packageList) {
         return [];
@@ -510,7 +512,47 @@ async function generateCargoComponentList(crates, imageTag, alreadyRegistered) {
     return componentList;
 }
 
+/* Generate "Go" entries. E.g.
+"Component": {
+    "Type": "go",
+    "Go": {
+        "Name": "golang.org/x/tools/gopls",
+        "Version": "0.6.4"
+    }
+}
+*/
+async function generateGoComponentList(packages, imageTag, alreadyRegistered) {
+    if (!packages) {
+        return [];
+    }
 
+    const componentList = [];
+    console.log(`(*) Generating "Go" registrations...`);
+
+    const packageInstallOutput = await getDockerRunCommandOutput(imageTag, "cat /usr/local/etc/vscode-dev-containers/go.log");
+    for(let package in packages) {
+        console.log(`(*) Getting version for ${package}...`);
+        const versionCommand = packages[package];
+        let version;
+        if(versionCommand) {
+            version = await getDockerRunCommandOutput(imageTag, versionCommand);
+        } else {
+            const versionCaptureGroup = new RegExp(`${package}\\s.*v([0-9]+\\.[0-9]+\\.[0-9]+.*)\\n`,'m').exec(packageInstallOutput);
+            version = versionCaptureGroup ? versionCaptureGroup[1] : 'latest';
+        }
+        addIfUnique(`${package}-go`, version, componentList, alreadyRegistered, {
+            "Component": {
+                "Type": "Go",
+                "Go": {
+                    "Name": package,
+                    "Version": version,
+                }
+            }
+        });
+    }
+
+    return componentList;
+}
 
 function addIfUnique(uniqueName, uniqueVersion, componentList, alreadyRegistered, componentJson) {
     if (typeof alreadyRegistered[uniqueName] === 'undefined' || alreadyRegistered[uniqueName].indexOf(uniqueVersion) < 0) {

build/src/utils/config.js

@@ -506,6 +506,12 @@ function getAllDependencies() {
     return config.definitionDependencies;
 }
 
+function getPoolKeyForPoolUrl(poolUrl) {
+    const poolKey = config.poolKeys[poolUrl];
+    console.log (`(*) Key for ${poolUrl} is ${poolKey}`);
+    return poolKey;
+}
+
 async function getStagingFolder(release) {
     if (!stagingFolders[release]) {
         const stagingFolder = path.join(os.tmpdir(), 'vscode-dev-containers', release);
@@ -544,6 +550,7 @@ module.exports = {
     getLinuxDistroForDefinition: getLinuxDistroForDefinition,
     getVersionFromRelease: getVersionFromRelease,
     getTagsForVersion: getTagsForVersion,
+    getPoolKeyForPoolUrl: getPoolKeyForPoolUrl,
     getConfig: getConfig,
     shouldFlattenDefinitionBaseImage: shouldFlattenDefinitionBaseImage
 };

containers/codespaces-linux-stretch/definition-manifest.json

@@ -15,7 +15,8 @@
 		"imageLink": "https://github.com/microsoft/oryx",
 		"debian": [
 			"git-lfs",
-			"moby-cli"
+			"moby-cli",
+			"powershell"
 		],
 		"pipx": [
 			"pylint",
@@ -104,70 +105,15 @@
 				"downloadUrl": "https://www.ruby-lang.org/en/downloads/"
 			}
 		},
-		"manual": [{
+		"go": {
-			"Component": {
+			"golang.org/x/tools/gopls": null,
-			   "Type": "go",
+			"honnef.co/go/tools/...": null,
-			   "Go": {
+			"golang.org/x/lint/golint": null,
-				 "Name": "golang.org/x/tools/gopls",
+			"github.com/mgechev/revive": null,
-				 "Version": "0.6.4"
+			"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
-			   }
+			"github.com/ramya-rao-a/go-outline": null,
-			 }
+			"github.com/go-delve/delve/cmd/dlv": null,
-		}, {
+			"github.com/golangci/golangci-lint/cmd/golangci-lint": null
-			"Component": {
+		}
-			   "Type": "go",
-			   "Go": {
-				 "Name": "honnef.co/go/tools",
-				 "Version": "0.1.1"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "golang.org/x/lint/golint",
-				 "Version": "0.0.0-20201208152925-83fdc39ff7b5"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/mgechev/revive",
-				 "Version": "1.0.3"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
-				 "Version": "2.1.2"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/ramya-rao-a/go-outline",
-				 "Version": "0.0.0-20200117021646-2a048b4510eb"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/go-delve/delve/cmd/dlv",
-				 "Version": "1.6.0"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/golangci/golangci-lint",
-				 "Version": "1.36.0"
-			   }
-			 }
-		}]
 	}
 }

containers/codespaces-linux/definition-manifest.json

@@ -15,7 +15,8 @@
 		"debian": [
 			"git-lfs",
 			"moby-cli",
-			"moby-engine"
+			"moby-engine",
+			"powershell"
 		],
 		"pipx": [
 			"pylint",
@@ -100,70 +101,15 @@
 				"downloadUrl": "https://github.com/rvm/rvm"
 			}
 		},
-		"manual": [{
+		"go": {
-			"Component": {
+			"golang.org/x/tools/gopls": null,
-			   "Type": "go",
+			"honnef.co/go/tools/...": null,
-			   "Go": {
+			"golang.org/x/lint/golint": null,
-				 "Name": "golang.org/x/tools/gopls",
+			"github.com/mgechev/revive": null,
-				 "Version": "0.6.4"
+			"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
-			   }
+			"github.com/ramya-rao-a/go-outline": null,
-			 }
+			"github.com/go-delve/delve/cmd/dlv": null,
-		}, {
+			"github.com/golangci/golangci-lint/cmd/golangci-lint": null
-			"Component": {
+		}
-			   "Type": "go",
-			   "Go": {
-				 "Name": "honnef.co/go/tools",
-				 "Version": "0.1.1"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "golang.org/x/lint/golint",
-				 "Version": "0.0.0-20201208152925-83fdc39ff7b5"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/mgechev/revive",
-				 "Version": "1.0.3"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
-				 "Version": "2.1.2"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/ramya-rao-a/go-outline",
-				 "Version": "0.0.0-20200117021646-2a048b4510eb"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/go-delve/delve/cmd/dlv",
-				 "Version": "1.6.0"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/golangci/golangci-lint",
-				 "Version": "1.36.0"
-			   }
-			 }
-		}]
 	}
 }

containers/go/definition-manifest.json

@@ -19,70 +19,15 @@
 			"Oh My Bash!": "/root/.oh-my-bash",
 			"nvm": "/usr/local/share/nvm"
 		},
-		"manual": [{
+		"go": {
-			"Component": {
+			"golang.org/x/tools/gopls": null,
-			   "Type": "go",
+			"honnef.co/go/tools/...": null,
-			   "Go": {
+			"golang.org/x/lint/golint": null,
-				 "Name": "golang.org/x/tools/gopls",
+			"github.com/mgechev/revive": null,
-				 "Version": "0.6.4"
+			"github.com/uudashr/gopkgs/v2/cmd/gopkgs": null,
-			   }
+			"github.com/ramya-rao-a/go-outline": null,
-			 }
+			"github.com/go-delve/delve/cmd/dlv": null,
-		}, {
+			"github.com/golangci/golangci-lint/cmd/golangci-lint": null
-			"Component": {
+		}
-			   "Type": "go",
-			   "Go": {
-				 "Name": "honnef.co/go/tools",
-				 "Version": "0.1.1"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "golang.org/x/lint/golint",
-				 "Version": "0.0.0-20201208152925-83fdc39ff7b5"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/mgechev/revive",
-				 "Version": "1.0.3"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/uudashr/gopkgs/v2/cmd/gopkgs",
-				 "Version": "2.1.2"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/ramya-rao-a/go-outline",
-				 "Version": "0.0.0-20200117021646-2a048b4510eb"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/go-delve/delve/cmd/dlv",
-				 "Version": "1.6.0"
-			   }
-			 }
-		}, {
-			"Component": {
-			   "Type": "go",
-			   "Go": {
-				 "Name": "github.com/golangci/golangci-lint",
-				 "Version": "1.36.0"
-			   }
-			 }
-		}]
 	}
 }