Upgrade infrastructure to support Node 10 (#201)

* allow tasks to upgrade their TS version

* make build directory consistent with azure-pipelines-tasks repo

https://github.com/microsoft/azure-pipelines-tasks/pull/7040/files#diff-53c70f37847db86d5ef8f8ed103140aebe630332facb6af5cd99e4845c180ef8L129

* update build infrastructure to support Node 10

* update dependencies

* change all tasks folder names to use CamelCase

* fix task names in make-options.json

* update downloadArchive

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 _temp/
 _results/
 _build/
+_download/
 
 gulp-tsc-tmp-*
 .gulp-tsc-tmp-*

make-options.json

@@ -1,8 +1,8 @@
 {
     "tasks": [
-        "app-store-promote",
-        "app-store-release",
-        "ipa-resign"
+        "AppStorePromote",
+        "AppStoreRelease",
+        "IpaResign"
     ],
     "taskResources": [
         "*.ps1",

make-util.js

@@ -19,6 +19,8 @@ var makeOptions = require('./make-options.json');
 // list of .NET culture names
 var cultureNames = [ 'cs', 'de', 'es', 'fr', 'it', 'ja', 'ko', 'pl', 'pt-BR', 'ru', 'tr', 'zh-Hans', 'zh-Hant' ];
 
+var allowedTypescriptVersions = ['2.3.4', '4.0.2'];
+
 //------------------------------------------------------------------------------
 // shell functions
 //------------------------------------------------------------------------------
@@ -139,10 +141,37 @@ exports.lintNodeTask = lintNodeTask;
 var buildNodeTask = function (taskPath, outDir) {
     var originalDir = pwd();
     cd(taskPath);
-    if (test('-f', rp('package.json'))) {
+    var packageJsonPath = rp('package.json');
+    var overrideTscPath;
+    if (test('-f', packageJsonPath)) {
+        // verify no dev dependencies
+        // we allow a TS dev-dependency to indicate a task should use a different TS version
+        var packageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString());
+        var devDeps = packageJson.devDependencies ? Object.keys(packageJson.devDependencies).length : 0;
+        if (devDeps == 1 && packageJson.devDependencies["typescript"]) {
+            var version = packageJson.devDependencies["typescript"];
+            if (!allowedTypescriptVersions.includes(version)) {
+                fail(`The package.json specifies a different TS version (${version}) that the allowed versions: ${allowedTypescriptVersions}. Offending package.json: ${packageJsonPath}`);
+            }
+            overrideTscPath = path.join(taskPath, "node_modules", "typescript");
+            console.log(`Detected Typescript version: ${version}`);
+        } else if (devDeps >= 1) {
+            fail('The package.json should not contain dev dependencies other than typescript. Move the dev dependencies into a package.json file under the Tests sub-folder. Offending package.json: ' + packageJsonPath);
+        }
+
         run('npm install');
     }
-    run('tsc --outDir ' + outDir + ' --rootDir ' + taskPath);
+
+    // Use the tsc version supplied by the task if it is available, otherwise use the global default.
+    if (overrideTscPath) {
+        var tscExec = path.join(overrideTscPath, "bin", "tsc");
+        run("node " + tscExec + ' --outDir "' + outDir + '" --rootDir "' + taskPath + '"');
+        // Don't include typescript in node_modules
+        rm("-rf", overrideTscPath);
+    } else {
+        run('tsc --outDir "' + outDir + '" --rootDir "' + taskPath + '"');
+    }
+
     cd(originalDir);
 }
 exports.buildNodeTask = buildNodeTask;
@@ -338,8 +367,21 @@ var downloadArchive = function (url, omitExtensionCheck) {
         throw new Error('Parameter "url" must be set.');
     }
 
-    if (!omitExtensionCheck && !url.match(/\.zip$/)) {
-        throw new Error('Expected .zip');
+    var isZip;
+    var isTargz;
+    if (omitExtensionCheck) {
+        isZip = true;
+    }
+    else {
+        if (url.match(/\.zip$/)) {
+            isZip = true;
+        }
+        else if (url.match(/\.tar\.gz$/) && (process.platform == 'darwin' || process.platform == 'linux')) {
+            isTargz = true;
+        }
+        else {
+            throw new Error('Unexpected archive extension');
+        }
     }
 
     // skip if already downloaded and extracted
@@ -358,8 +400,27 @@ var downloadArchive = function (url, omitExtensionCheck) {
 
         // extract
         mkdir('-p', targetPath);
-        var zip = new admZip(archivePath);
-        zip.extractAllTo(targetPath);
+        if (isZip) {
+            if (process.platform == 'win32') {
+                let escapedFile = archivePath.replace(/'/g, "''").replace(/"|\n|\r/g, ''); // double-up single quotes, remove double quotes and newlines
+                let escapedDest = targetPath.replace(/'/g, "''").replace(/"|\n|\r/g, '');
+
+                let command = `$ErrorActionPreference = 'Stop' ; try { Add-Type -AssemblyName System.IO.Compression.FileSystem } catch { } ; [System.IO.Compression.ZipFile]::ExtractToDirectory('${escapedFile}', '${escapedDest}')`;
+                run(`powershell -Command "${command}"`);
+            } else {
+                run(`unzip ${archivePath} -d ${targetPath}`);
+            }
+        }
+        else if (isTargz) {
+            var originalCwd = process.cwd();
+            cd(targetPath);
+            try {
+                run(`tar -xzf "${archivePath}"`);
+            }
+            finally {
+                cd(originalCwd);
+            }
+        }
 
         // write the completed marker
         fs.writeFileSync(marker, '');
@@ -981,4 +1042,90 @@ var storeNonAggregatedZip = function (zipPath, release, commit) {
     fs.writeFileSync(destMarker, '');
 }
 exports.storeNonAggregatedZip = storeNonAggregatedZip;
+
+var installNode = function (nodeVersion) {
+    switch (nodeVersion || '') {
+        case '14':
+            nodeVersion = 'v14.10.1';
+            break;
+        case '10':
+            nodeVersion = 'v10.21.0';
+            break;
+        case '6':
+        case '':
+            nodeVersion = 'v6.10.3';
+            break;
+        case '5':
+            nodeVersion = 'v5.10.1';
+            break;
+        default:
+            fail(`Unexpected node version '${nodeVersion}'. Expected 5 or 6.`);
+    }
+
+    if (nodeVersion === run('node -v')) {
+        console.log('skipping node install for tests since correct version is running');
+        return;
+    }
+
+    // determine the platform
+    var platform = os.platform();
+    if (platform != 'darwin' && platform != 'linux' && platform != 'win32') {
+        throw new Error('Unexpected platform: ' + platform);
+    }
+
+    var nodeUrl = 'https://nodejs.org/dist';
+    switch (platform) {
+        case 'darwin':
+            var nodeArchivePath = downloadArchive(nodeUrl + '/' + nodeVersion + '/node-' + nodeVersion + '-darwin-x64.tar.gz');
+            addPath(path.join(nodeArchivePath, 'node-' + nodeVersion + '-darwin-x64', 'bin'));
+            break;
+        case 'linux':
+            var nodeArchivePath = downloadArchive(nodeUrl + '/' + nodeVersion + '/node-' + nodeVersion + '-linux-x64.tar.gz');
+            addPath(path.join(nodeArchivePath, 'node-' + nodeVersion + '-linux-x64', 'bin'));
+            break;
+        case 'win32':
+            var nodeDirectory = path.join(downloadPath, `node-${nodeVersion}`);
+            var marker = nodeDirectory + '.completed';
+            if (!test('-f', marker)) {
+                var nodeExePath = downloadFile(nodeUrl + '/' + nodeVersion + '/win-x64/node.exe');
+                var nodeLibPath = downloadFile(nodeUrl + '/' + nodeVersion + '/win-x64/node.lib');
+                rm('-Rf', nodeDirectory);
+                mkdir('-p', nodeDirectory);
+                cp(nodeExePath, path.join(nodeDirectory, 'node.exe'));
+                cp(nodeLibPath, path.join(nodeDirectory, 'node.lib'));
+                fs.writeFileSync(marker, '');
+            }
+
+            addPath(nodeDirectory);
+            break;
+    }
+}
+exports.installNode = installNode;
+
+var getTaskNodeVersion = function(buildPath, taskName) {
+    var taskJsonPath = path.join(buildPath, taskName, "task.json");
+    if (!fs.existsSync(taskJsonPath)) {
+        console.warn('Unable to find task.json, defaulting to use Node 14');
+        return 14;
+    }
+    var taskJsonContents = fs.readFileSync(taskJsonPath, { encoding: 'utf-8' });
+    var taskJson = JSON.parse(taskJsonContents);
+    var execution = taskJson['execution'] || taskJson['prejobexecution'];
+    for (var key of Object.keys(execution)) {
+        if (key.toLowerCase() == 'node14') {
+            // Prefer node 14 and return immediately.
+            return 14;
+        } else if (key.toLowerCase() == 'node10') {
+            // Prefer node 10 and return immediately.
+            return 10;
+        } else if (key.toLowerCase() == 'node') {
+            return 6;
+        }
+    }
+
+    console.warn('Unable to determine execution type from task.json, defaulting to use Node 10');
+    return 10;
+}
+exports.getTaskNodeVersion = getTaskNodeVersion;
+
 //------------------------------------------------------------------------------

make.js

@@ -51,6 +51,7 @@ var getExternals = util.getExternals;
 var createResjson = util.createResjson;
 var createTaskLocJson = util.createTaskLocJson;
 var validateTask = util.validateTask;
+var getTaskNodeVersion = util.getTaskNodeVersion;
 
 // global paths
 var buildPath = path.join(__dirname, '_build', 'Tasks');
@@ -66,6 +67,9 @@ if (semver.lt(process.versions.node, minNodeVer)) {
     fail('requires node >= ' + minNodeVer + '.  installed: ' + process.versions.node);
 }
 
+// Node 14 is supported by the build system, but not currently by the agent. Block it for now
+var supportedNodeTargets = ["Node", "Node10"/*, "Node14"*/];
+
 // add node modules .bin to the path so we can dictate version of tsc etc...
 var binPath = path.join(__dirname, 'node_modules', '.bin');
 if (!test('-d', binPath)) {
@@ -125,14 +129,14 @@ target.build = function() {
             validateTask(taskDef);
 
             // fixup the outDir (required for relative pathing in legacy L0 tests)
-            outDir = path.join(buildPath, taskDef.name);
+            outDir = path.join(buildPath, taskName);
 
             // create loc files
             createTaskLocJson(taskPath);
             createResjson(taskDef, taskPath);
 
             // determine the type of task
-            shouldBuildNode = shouldBuildNode || taskDef.execution.hasOwnProperty('Node');
+            shouldBuildNode = shouldBuildNode || supportedNodeTargets.some(node => taskDef.execution.hasOwnProperty(node));
             shouldBuildPs3 = taskDef.execution.hasOwnProperty('PowerShell3');
         }
         else {
@@ -248,26 +252,56 @@ target.test = function() {
 
     // run the tests
     var suiteType = options.suite || 'L0';
-    var taskType = options.task || '*';
-    var pattern1 = buildPath + '/' + taskType + '/Tests/' + suiteType + '.js';
-    var pattern2 = buildPath + '/Common/' + taskType + '/Tests/' + suiteType + '.js';
-    var testsSpec = matchFind(pattern1, buildPath)
-        .concat(matchFind(pattern2, buildPath));
-    if (!testsSpec.length) {
-        fail(`Unable to find tests using the following patterns: ${JSON.stringify([pattern1, pattern2])}`);
+    function runTaskTests(taskName) {
+        banner('Testing: ' + taskName);
+        // find the tests
+        var nodeVersion = options.node || getTaskNodeVersion(buildPath, taskName) + "";
+        var pattern1 = path.join(buildPath, taskName, 'Tests', suiteType + '.js');
+        var pattern2 = path.join(buildPath, 'Common', taskName, 'Tests', suiteType + '.js');
+
+        var testsSpec = [];
+
+        if (fs.existsSync(pattern1)) {
+            testsSpec.push(pattern1);
+        }
+        if (fs.existsSync(pattern2)) {
+            testsSpec.push(pattern2);
         }
 
-    // set up any test reporting
-    var testResultsArgs = '';
-    if (options.testResults) {
-        if (options.testReporter) {
-            testResultsArgs += ' -R ' + options.testReporter;
+        if (testsSpec.length == 0) {
+            console.warn(`Unable to find tests using the following patterns: ${JSON.stringify([pattern1, pattern2])}`);
+            return;
         }
-        if (options.testReportLocation) {
-            testResultsArgs += ' -O mochaFile=' + path.join(__dirname, options.testReportLocation);
-        }
-    }
-    console.log('testResultsArgs=' + testResultsArgs);
 
-    run('mocha ' + testsSpec.join(' ') + testResultsArgs, /*inheritStreams:*/true);
+        // setup the version of node to run the tests
+        util.installNode(nodeVersion);
+
+        run('mocha ' + testsSpec.join(' ') /*+ ' --reporter mocha-junit-reporter --reporter-options mochaFile=../testresults/test-results.xml'*/, /*inheritStreams:*/true);
+    }
+
+    if (options.task) {
+        runTaskTests(options.task);
+    } else {
+        // Run tests for each task that exists
+        taskList.forEach(function(taskName) {
+            var taskPath = path.join(buildPath, taskName);
+            if (fs.existsSync(taskPath)) {
+                runTaskTests(taskName);
+            }
+        });
+
+        banner('Running common library tests');
+        var commonLibPattern = path.join(buildPath, 'Common', '*', 'Tests', suiteType + '.js');
+        var specs = [];
+        if (matchFind(commonLibPattern, buildPath).length > 0) {
+            specs.push(commonLibPattern);
+        }
+        if (specs.length > 0) {
+            // setup the version of node to run the tests
+            util.installNode(options.node);
+            run('mocha ' + specs.join(' ') /*+ ' --reporter mocha-junit-reporter --reporter-options mochaFile=../testresults/test-results.xml'*/, /*inheritStreams:*/true);
+        } else {
+            console.warn("No common library tests found");
+        }
+    }
 }

package-lock.json

@@ -5,15 +5,15 @@
   "requires": true,
   "dependencies": {
     "@types/mocha": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.0.tgz",
-      "integrity": "sha512-YeDiSEzznwZwwp766SJ6QlrTyBYUGPSIwmREHVTmktUYiT/WADdWtpt9iH0KuUSf8lZLdI4lP0X6PBzPo5//JQ==",
+      "version": "5.2.7",
+      "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz",
+      "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==",
       "dev": true
     },
     "@types/node": {
-      "version": "6.14.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-6.14.2.tgz",
-      "integrity": "sha512-JWB3xaVfsfnFY8Ofc9rTB/op0fqqTSqy4vBcVk1LuRJvta7KTX+D//fCkiTMeLGhdr2EbFZzQjC97gvmPilk9Q==",
+      "version": "10.17.50",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.50.tgz",
+      "integrity": "sha512-vwX+/ija9xKc/z9VqMCdbf4WYcMTGsI0I/L/6shIF3qXURxZOhPQlPRHtjTpiNhAwn0paMJzlOQqw6mAGEQnTA==",
       "dev": true
     },
     "@types/q": {
@@ -861,6 +861,12 @@
       "integrity": "sha512-mmGt/1pZqYRjMxB1axhTo16/snVZ5krrKkcmMeVKxzECMMXoCgnvTPp10QgHfcbQZw8Dq2jMNG6je4JlWU0gWg==",
       "dev": true
     },
+    "command-exists": {
+      "version": "1.2.9",
+      "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz",
+      "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==",
+      "dev": true
+    },
     "commander": {
       "version": "2.15.1",
       "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
@@ -2336,6 +2342,12 @@
       "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
       "dev": true
     },
+    "get-port": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz",
+      "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=",
+      "dev": true
+    },
     "get-stdin": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
@@ -4208,9 +4220,9 @@
       "dev": true
     },
     "qs": {
-      "version": "6.6.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.6.0.tgz",
-      "integrity": "sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA==",
+      "version": "6.9.4",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz",
+      "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==",
       "dev": true
     },
     "read": {
@@ -4884,14 +4896,16 @@
       }
     },
     "sync-request": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-3.0.1.tgz",
-      "integrity": "sha1-yqEjWq+Im6UBB2oYNMQ2gwqC+3M=",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-4.1.0.tgz",
+      "integrity": "sha512-iFbOBWYaznBNbheIKaMkj+3EabpEsXbuwcTVuYkRjoav+Om5L8VXXLIXms0cHxkouXMRCQaSfhfau9/HyIbM2Q==",
       "dev": true,
       "requires": {
-        "concat-stream": "^1.4.7",
-        "http-response-object": "^1.0.1",
-        "then-request": "^2.0.1"
+        "command-exists": "^1.2.2",
+        "concat-stream": "^1.6.0",
+        "get-port": "^3.1.0",
+        "http-response-object": "^1.1.0",
+        "then-request": "^2.2.0"
       }
     },
     "tar-stream": {

package.json

@@ -13,9 +13,9 @@
   },
   "homepage": "https://github.com/Microsoft/app-store-vsts-extension",
   "devDependencies": {
-    "@types/node": "^6.0.101",
+    "@types/node": "^10.17.0",
     "@types/q": "^1.0.7",
-    "@types/mocha": "5.2.0",
+    "@types/mocha": "^5.2.7",
     "adm-zip": "^0.4.13",
     "del": "^3.0.0",
     "gulp": "^4.0.0",
@@ -28,7 +28,7 @@
     "q": "1.4.1",
     "semver": "4.3.3",
     "shelljs": "^0.3.0",
-    "sync-request": "3.0.1",
+    "sync-request": "4.1.0",
     "tfx-cli": "^0.6.4",
     "tslint": "^3.15.1",
     "typescript": "3.2.2",