Limit gulp parallelism (#1933)

* Limit number of AutoRest processes spawned based on number of logical CPUs

* Simplify gulpfile help section and add --debug argument

* Parameterize concurrency level

* Fix formatting in gulp help

* Rename --concurrency to --parallel

* Add AutoRest to local packages for simplicity/consistency

* Fix output message about parallelism

* Fix subscriptions api-spec

api-specs.json

@@ -305,7 +305,8 @@
   "subscriptions": {
     "dir": "azure-mgmt-resources",
     "source": "specification/resources/resource-manager/readme.md",
-    "package": "com.microsoft.azure.management.resources --tag=package-subscriptions-2016-06"
+    "package": "com.microsoft.azure.management.resources",
+    "args": "--tag=package-subscriptions-2016-06"
   },
   "trafficmanager": {
     "dir": "azure-mgmt-trafficmanager",

gulpfile.js

@@ -2,7 +2,9 @@ var path = require('path');
 var gulp = require('gulp');
 var args = require('yargs').argv;
 var colors = require('colors');
-var spawn = require('child_process').spawn;
+var execa = require('execa');
+var pAll = require('p-all');
+var os = require('os');
 var fs = require('fs');
 
 const mappings = require('./api-specs.json');
@@ -15,6 +17,7 @@ gulp.task('default', function() {
         "[--autorest <autorest info>] " +
         "[--autorest-java <autorest.java info>] " +
         "[--debug] " +
+        "[--parallel <number>] " +
         "[--autorest-args <AutoRest arguments>]\n");
 
     console.log("--spec-root");
@@ -28,19 +31,24 @@ gulp.task('default', function() {
 
     console.log("--autorest");
     console.log("\tThe version of AutoRest. E.g. 2.0.9, or the location of AutoRest repo, e.g. E:\\repo\\autorest");
-    
+
     console.log("--autorest-java");
     console.log("\tPath to an autorest.java generator to pass as a --use argument to AutoRest.");
     console.log("\tUsually you'll only need to provide this and not a --autorest argument in order to work on Java code generation.");
     console.log("\tSee https://github.com/Azure/autorest/blob/master/docs/developer/autorest-extension.md");
-    
+
     console.log("--debug");
     console.log("\tFlag that allows you to attach a debugger to the autorest.java generator.");
 
+    console.log("--parallel");
+    console.log("\tSpecifies the maximum number of projects to generate in parallel.");
+    console.log("\tDefaults to the number of logical CPUs on the system. (On this system, " + os.cpus().length + ")");
+
     console.log("--autorest-args");
     console.log("\tPasses additional argument to AutoRest generator");
 });
 
+const maxParallelism = parseInt(args['parallel'], 10) || os.cpus().length;
 var specRoot = args['spec-root'] || defaultSpecRoot;
 var projects = args['projects'];
 var autoRestVersion = 'latest'; // default
@@ -63,24 +71,28 @@ gulp.task('codegen', function(cb) {
 });
 
 var handleInput = function(projects, cb) {
+    console.info(`Generating up to ${maxParallelism} projects in parallel..`);
     if (projects === undefined) {
-        Object.keys(mappings).forEach(function(proj) {
-            codegen(proj, cb);
+        const actions = Object.keys(mappings).map(proj => {
+            return () => codegen(proj, cb);
         });
+        pAll(actions, { concurrency: maxParallelism });
     } else {
-        projects.split(",").forEach(function(proj) {
-            proj = proj.replace(/\ /g, '');
-            if (mappings[proj] === undefined) {
-                console.error('Invalid project name "' + proj + '"!');
-                process.exit(1);
+        const actions = projects.split(",").map(proj => {
+            return () => {
+                proj = proj.replace(/\ /g, '');
+                if (mappings[proj] === undefined) {
+                    console.error('Invalid project name "' + proj + '"!');
+                    process.exit(1);
+                }
+                return codegen(proj, cb);
             }
-            codegen(proj, cb);
         });
+        pAll(actions, { maxParallelism });
     }
 }
 
 var codegen = function(project, cb) {
-
     if (!args['preserve']) {
         const sourcesToDelete = path.join(
             mappings[project].dir,
@@ -124,7 +136,7 @@ var codegen = function(project, cb) {
     }
 
     console.log('Command: ' + cmd);
-    spawn(cmd, [], { shell: true, stdio: "inherit" });
+    return execa(cmd, [], { shell: true, stdio: "inherit" });
 };
 
 var deleteFolderRecursive = function(path) {

package.json

@@ -16,9 +16,12 @@
   },
   "homepage": "https://github.com/Azure/azure-sdk-for-java#readme",
   "devDependencies": {
+    "autorest": "^2.0.4147",
     "colors": "1.1.2",
+    "execa": "^0.8.0",
     "gulp": "^3.9.0",
     "gulp-exec": "2.1.2",
+    "p-all": "^1.0.0",
     "yargs": "3.29.0"
   }
 }