azure-sdk-for-node/gulpfile.js

/*
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See License.txt in the project root for
 * license information.
 */

const gulp = require('gulp');
const args = require('yargs').argv;
const colors = require('colors');
const fs = require('fs');
const util = require('util');
const path = require('path');
const glob = require('glob');
const execSync = require('child_process').execSync;
const jsonStableStringify = require('json-stable-stringify');

var mappings = require('./codegen_mappings.json');

const defaultAutoRestVersion = '1.2.2';
var usingAutoRestVersion;
const specRoot = args['spec-root'] || 'https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/specification';
const project = args['project'];
const use = args['use'];
var modeler = 'Swagger';
const regexForExcludedServices = /\/(intune|documentdbManagement|insightsManagement|insights|search)\//i;

function getAutorestVersion(version) {
  if (!version) version = 'latest';
  let getVersion, execHelp;
  let result = true;
  try {
    let getVersionCmd = `autorest `;
    let execHelpCmd = `autorest --help`;
    console.log(getVersionCmd);
    getVersion = execSync(getVersionCmd, { encoding: 'utf8' });
    //console.debug(getVersion);
    console.log(execHelpCmd);
    execHelp = execSync(execHelpCmd, { encoding: 'utf8' });
    //console.debug(execHelp);
  } catch (err) {
    result = false;
    console.log(`An error occurred while getting the "${version}" of autorest and executing "autorest --help":\n ${util.inspect(err, { depth: null })}.`);
  }
  return result;
}

function deleteFolderRecursive(path) {
  if (fs.existsSync(path)) {
    fs.readdirSync(path).forEach(function (file, index) {
      var curPath = path + '/' + file;
      if (fs.lstatSync(curPath).isDirectory()) { // recurse
        deleteFolderRecursive(curPath);
      } else { // delete file
        fs.unlinkSync(curPath);
      }
    });
    fs.rmdirSync(path);
  }
}

function clearProjectBeforeGenerating(projectDir) {
  let modelsDir = `${projectDir}/models`;
  let operationsDir = `${projectDir}/operations`;
  let clientTypedefFile = path.basename(glob.sync(`${projectDir}/*.d.ts`)[0] || '');
  let clientJSFile = `${clientTypedefFile.split('.')[0]}.js`;
  let directoriesToBeDeleted = [modelsDir, operationsDir];
  let filesToBeDeleted = [clientTypedefFile, clientJSFile];
  directoriesToBeDeleted.forEach((dir) => {
    if (fs.existsSync(dir)) {
      deleteFolderRecursive(dir);
    }
  });
  filesToBeDeleted.forEach((file) => {
    if (fs.existsSync(file)) {
      fs.unlinkSync(file);
    }
  });
  return;
}

function generateProject(projectObj, specRoot, autoRestVersion) {
  let specPath = specRoot + '/' + projectObj.source;
  let isInputJson = projectObj.source.endsWith('json');
  let result;
  const azureTemplate = 'Azure.NodeJs';
  let language = azureTemplate;
  //servicefabric wants to generate using generic NodeJS.
  if (projectObj.language && projectObj.language.match(/^NodeJS$/ig) !== null) {
    language = projectObj.language;
  }
  let packageName = projectObj.packageName;
  console.log(`\n>>>>>>>>>>>>>>>>>>>Start: "${packageName}" >>>>>>>>>>>>>>>>>>>>>>>>>`);
  let outputDir = `${__dirname}/lib/services/${projectObj.dir}`;
  let cmd = 'autorest';
  if (projectObj.batchGeneration) {
    cmd += ` --nodejs-sdks-folder=${outputDir}`;
  } else {
    cmd += ` --nodejs.output-folder=${outputDir}`;
  }
  cmd += ` --package-name=${packageName}`;
  
  let packageVersion = projectObj.packageVersion;
  if (packageVersion) {
    cmd += ` --package-version=${packageVersion}`;
  }

  cmd += ` --nodejs --license-header=MICROSOFT_MIT_NO_VERSION`;

  // if using azure template, pass in azure-arm argument. otherwise, get the generic template by not passing in anything.
  if (language === azureTemplate) cmd += '  --azure-arm ';
  if (isInputJson) {
    cmd += ` --input-file=${specPath}`;
  }
  else {
    cmd += ` ${specPath}`;
  }

  if (use) {
    cmd += ` --use=${use}`;
  }

  if (projectObj.generatePackageJson) {
    cmd += ` --nodejs.generate-package-json=true`;
  }

  if (projectObj.generateReadmeMd) {
    cmd += ` --nodejs.generate-readme-md=true`;
  }

  if (projectObj.generateLicenseTxt) {
    cmd += ` --nodejs.generate-license-txt=true`;
  }

  if (projectObj.ft !== null && projectObj.ft !== undefined) cmd += ' --payload-flattening-threshold=' + projectObj.ft;
  if (projectObj.clientName !== null && projectObj.clientName !== undefined) cmd += ' --override-client-name=' + projectObj.clientName;
  if (projectObj.tag !== null && projectObj.tag !== undefined) cmd += `--tag=${projectObj.tag}`;
  if (projectObj.args !== undefined) {
    cmd += ` ${args}`;
  }

  try {
    //console.log(`Cleaning the output directory: "${outputDir}".`);
    //clearProjectBeforeGenerating(outputDir);
    console.log('Executing command:');
    console.log('------------------------------------------------------------');
    console.log(cmd);
    console.log('------------------------------------------------------------');
    result = execSync(cmd, { encoding: 'utf8' });
    console.log('Output:');
    console.log(result);
  } catch (err) {
    console.log('Error:');
    console.log(`An error occurred while generating client for package: "${packageName}":\n ${err.stderr}`);
  }
  console.log(`>>>>>>>>>>>>>>>>>>>>>End: "${packageName}" >>>>>>>>>>>>>>>>>>>>>>>>>\n`);
  return;
}

function installAutorest() {
  let installation;
  let isSuccessful = true;
  let autorestAlreadyInstalled = true;
  try {
    execSync(`autorest --help`);
  } catch (error) {
    autorestAlreadyInstalled = false;
  }
  try {
    if (!autorestAlreadyInstalled) {
      console.log('Looks like autorest is not installed on your machine. Installing autorest . . .');
      let installCmd = 'npm install -g autorest';
      console.log(installCmd);
      installation = execSync(installCmd, { encoding: 'utf8' });
      //console.debug('installation');
    }
    isSuccessful = getAutorestVersion();
  } catch (err) {
    isSuccessful = false;
    console.log(`An error occurred while installing autorest via npm:\n ${util.inspect(err, { depth: null })}.`);
  }
  return isSuccessful;
}

function codegen(projectObj, index) {
  let versionSuccessfullyFound = true;
  let usingAutoRestVersion = defaultAutoRestVersion;
  function checkAutorestVersion(actualProj) {
    if (actualProj.autoRestVersion) {
      usingAutoRestVersion = actualProj.autoRestVersion;
    }
    if (index === 0) {
      versionSuccessfullyFound = getAutorestVersion(usingAutoRestVersion);
      if (!versionSuccessfullyFound) {
        process.exit(1);
      }
    }
  }

  function iterateProject(proj, specRoot, usingAutoRestVersion) {
    for (const key in proj) {
      if (proj[key]['packageName']) {
        if (!versionSuccessfullyFound) {
          checkAutorestVersion(proj[key], index);
        }
        generateProject(proj[key], specRoot, usingAutoRestVersion);
      } else {
        iterateProject(proj[key], specRoot, usingAutoRestVersion);
      }
    }
  }

  return iterateProject(projectObj, specRoot, usingAutoRestVersion);
}

gulp.task('default', function () {
  console.log('Usage: gulp codegen [--spec-root <swagger specs root>] [--use <autorest.nodejs root> [--project <project name>]\n');
  console.log('--spec-root');
  console.log('\tRoot location of Swagger API specs, default value is \"https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/specification\"');
  console.log('--use');
  console.log('\tRoot location of autorest.nodejs repository. If this is not specified, then the latest install generator for NodeJS will be used.');
  console.log('--project\n\tProject to regenerate, default is all. List of available project names:');
  Object.keys(mappings).forEach(function (i) {
    console.log('\t' + i.magenta);
  });
});

//This task is used to generate libraries based on the mappings specified above.
gulp.task('codegen', function (cb) {
  if (project === undefined) {
    let arr = Object.keys(mappings);
    for (let i = 0; i < arr.length; i++) {
      codegen(mappings[arr[i]], i);
    }
  } else {
    if (mappings[project] === undefined) {
      console.error('Invalid project name "' + project + '"!');
      process.exit(1);
    }
    codegen(mappings[project], null);
  }
});

//This task validates that the entry in "main" and "types" in package.json points to a file that exists on the disk.
// for best results run on mac or linux. Windows is case insenstive for file paths. Hence it will not catch those issues.
//If not tested this will cause "module not found" errors for customers when they try to use the package.
gulp.task('validate-each-packagejson', (cb) => {
  let packagePaths = glob.sync(path.join(__dirname, '/lib/services', '/**/package.json'), { ignore:  '**/node_modules/**' });
  packagePaths.forEach((packagePath) => {
    const package = require(packagePath);
    //console.log(package);
    if (!package.name.startsWith('azure-asm-')) {
      console.log(`Validating package: ${package.name}`);
      if (package.main) {
        let mainPath = path.resolve(path.dirname(packagePath), package.main);
        if (!fs.existsSync(mainPath)) console.log(`\t>${mainPath} does not exist.`);
      } else {
        console.log(`\t>Could not find "main" entry in package.json for ${packagePath}.`);
      }
      if (package.types) {
        let typesPath = path.resolve(path.dirname(packagePath), package.types);
        if (!fs.existsSync(typesPath)) console.log(`\t>${typesPath} does not exist.`);
      } else {
        console.log(`\t>Could not find "types" entry in package.json for ${packagePath}.`);
      }
    }
  });
});

//This task updates the dependencies in package.json to the relative service libraries inside lib/services directory.
gulp.task('update-deps-rollup', (cb) => {

  let packagePaths = glob.sync(path.join(__dirname, './lib/services', '/**/package.json')).filter((packagePath) => {
    return packagePath.match(regexForExcludedServices) === null;
  });
  let rollupPackage = require('./package.json');
  let rollupDependencies = rollupPackage.dependencies;
  rollupDependencies['ms-rest'] = './runtime/ms-rest';
  rollupDependencies['ms-rest-azure'] = './runtime/ms-rest-azure';
  packagePaths.forEach((packagePath) => {
    const package = require(packagePath);
    //console.log(package);
    let packageName = package.name;
    const packageDir = path.dirname(packagePath);
    if (rollupDependencies[packageName]) {
      rollupDependencies[packageName] = packageDir;
    } else {
      console.log(`Could not find ${packageName} as a dependecy in rollup package.json file..`);
    }
  });
  fs.writeFileSync('./package.json', JSON.stringify(rollupPackage, null, 2), { 'encoding': 'utf8' });
});

//This task ensures that all the exposed createSomeClient() methods, can correctly instantiate clients. By doing this we test,
//that the "main" entry in package.json points to a file at the correct location. We test the signature of the client constructor 
//is as expected. As of now HD Isnight is expected to fail as it is still using the Hyak generator. Once it moves to Autorest, it should
//not fail. Before executing this task, execute `gulp update-deps-rollup`, `rm -rf node_modules` and `npm install` so that the changes inside the sdks in lib/services
//are installed inside the node_modules folder.
gulp.task('test-create-rollup', (cb) => {
  const azure = require('./lib/azure');
  const keys = Object.keys(azure).filter((key) => { return key.startsWith('create') && !key.startsWith('createASM') && key.endsWith('Client') && key !== 'createSchedulerClient'; });
  //console.dir(keys);
  //console.log(keys.length);
  const creds = { signRequest: {} };
  const subId = '1234556';

  keys.forEach((key) => {
    console.log(key);
    const Client = azure[key];
    var c;
    try {
      if (key === 'createKeyVaultClient' || key === 'createSubscriptionManagementClient' ||
        key === 'createDataLakeAnalyticsJobManagementClient' || key === 'createDataLakeStoreFileSystemManagementClient' ||
        key === 'createDataLakeAnalyticsCatalogManagementClient') {
        c = new Client(creds);
      } else if (key === 'createServiceFabricClient') {
        c = new Client();
      } else {
        c = new Client(creds, subId);
      }
      //console.dir(Object.keys(c));
    } catch (err) {
      console.dir(err);
    }
  });
});

// This task updates the codegen_mappings.json file in sync with the azure-rest-api-specs public repo.
gulp.task('sync-mappings-with-repo', (cb) => {
  if (!specRoot) {
    return cb(new Error('Please provide --spec-root <Absolute path to the specification folder in your local clone of the azure-rest-api-specs repository.>'));
  }
  const dirs = fs.readdirSync(specRoot).filter(f => fs.statSync(`${specRoot}/${f}`).isDirectory());
  let newlyAdded = [];
  let originalProjectCount = Object.keys(mappings).length;
  const resourceProvidersToIgnore = ['azsadmin', 'common-types', 'databricks', 'intune', 'timeseriesinsights'];
  const resourceProviderDataPlanesToIgnore = ['applicationinsights', 'operationalinsights'];

  function createDescriptor() {
    return {
      'packageVersion': '1.0.0-preview',
      'generatePackageJson': true,
      'generateReadmeMd': true,
      'generateLicenseTxt': true
    };
  }

  function createManagementDescriptor(resourceProviderName) {
    const descriptor = createDescriptor();
    descriptor.packageName = `azure-arm-${resourceProviderName.toLowerCase()}`;
    descriptor.dir = `${resourceProviderName}Management/lib`;
    descriptor.source = `${resourceProviderName}/resource-manager/readme.md`;
    return descriptor;
  }

  function createDataplaneDescriptor(resourceProviderName) {
    const descriptor = createDescriptor();
    descriptor.packageName = `azure-${resourceProviderName.toLowerCase()}`;
    descriptor.dir = `${resourceProviderName}/lib`;
    descriptor.source = `${resourceProviderName}/data-plane/readme.md`;
    return descriptor;
  }

  for (let rp of dirs) {
    if (resourceProvidersToIgnore.indexOf(rp.toLowerCase()) === -1) {
      let rm = `${specRoot}/${rp}/resource-manager`;
      let dp = `${specRoot}/${rp}/data-plane`;
      if (!mappings[rp]) {
        mappings[rp] = {};
        if (fs.existsSync(rm)) {
          mappings[rp]['resource-manager'] = createManagementDescriptor(rp);
          newlyAdded.push(`${rp}['resource-manager']`);
          console.log(`Updating RP: ${rp}, "resource-manager".`);
          console.dir(mappings[rp]['resource-manager'], { depth: null, colors: true });
        }
        if (resourceProviderDataPlanesToIgnore.indexOf(rp.toLowerCase()) === -1) {
          if (fs.existsSync(dp)) {
            mappings[rp]['data-plane'] = createDataplaneDescriptor(rp);
            newlyAdded.push(`${rp}['data-plane']`);
            console.log(`Updating RP: ${rp}, "data-plane".`);
            console.dir(mappings[rp]['data-plane'], { depth: null, colors: true });
          }
        }
      } else {
        if (fs.existsSync(rm) && !mappings[rp]['resource-manager']) {
          mappings[rp]['resource-manager'] = createManagementDescriptor(rp);
          newlyAdded.push(`${rp}['resource-manager']`);
          console.log(`Updating RP: ${rp}, "resource-manager".`);
          console.dir(mappings[rp]['resource-manager'], { depth: null, colors: true });
        }
        if (resourceProviderDataPlanesToIgnore.indexOf(rp.toLowerCase()) === -1) {
          if (fs.existsSync(dp) && !mappings[rp]['data-plane']) {
            mappings[rp]['data-plane'] = createDataplaneDescriptor(rp);
            newlyAdded.push(`${rp}['data-plane']`);
            console.log(`Updating RP: ${rp}, "data-plane".`);
            console.dir(mappings[rp]['data-plane'], { depth: null, colors: true });
          }
        }
      }
    }
  }
  if (!newlyAdded.length) {
    console.log('\n\n> Mappings in ./codegen_mappings.json are already in sync...');
  } else {
    console.log(`\n\n> Basic properties like "packageName", "dir" and "source" have been added to ` +
      `the newly added projects "${newlyAdded.join()}" in the mappings.\n\n> Please ensure that other properties ` +
      `like: "ft", "clientName", etc. are correctly added as deemed necessary.\n\n> If the specs repo had multiple ` +
      `specs in data-plane or resource-manager (for example: "datalake-analytics.data-plane" has "catalog" ` +
      `and "job" in it), then please update the project mappings yourself.`);
  }
  console.log(`\n\n>>>>>  Total projects in the mappings before sync: ${originalProjectCount}`);
  console.log(`\n>>>>>  Total projects in the mappings after  sync: ${Object.keys(mappings).length}`);
  fs.writeFileSync('./codegen_mappings.json', JSON.stringify(mappings, null, 2));
});

// This task synchronizes the dependencies in package.json to the versions of relative service libraries inside lib/services directory.
// This should be done in the end to ensure that all the package dependencies have the correct version.
gulp.task('sync-deps-rollup', (cb) => {
  let packagePaths = glob.sync(path.join(__dirname, './lib/services', '/**/package.json')).filter((packagePath) => {
    return packagePath.match(regexForExcludedServices) === null;
  });
  //console.log(packagePaths);
  console.log(`Total packages found under lib/services: ${packagePaths.length}`);
  let rollupPackage = require('./package.json');
  let rollupDependencies = rollupPackage.dependencies;
  rollupDependencies['ms-rest'] = '^2.2.2';
  rollupDependencies['ms-rest-azure'] = '^2.3.4';
  packagePaths.forEach((packagePath) => {
    const package = require(packagePath);
    //console.log(package);
    let packageName = package.name;
    let packageVersion = package.version;
    rollupDependencies[packageName] = packageVersion;
  });
  rollupPackage.dependencies = Object.keys(rollupDependencies).sort().reduce((r, k) => (r[k] = rollupDependencies[k], r), {});
  console.log(`Total number of dependencies in the rollup package: ${Object.keys(rollupPackage.dependencies).length}`);
  fs.writeFileSync('./package.json', JSON.stringify(rollupPackage, null, 2), { 'encoding': 'utf8' });
});

gulp.task('sync-package-service-mapping', (cb) => {
  let packageMapping = require('./package_service_mapping');
  for (const serviceName in mappings) {
    if (serviceName) {
      const serviceObj = mappings[serviceName];
      const resourceMgr = serviceObj['resource-manager'];
      const Dataplane = serviceObj['data-plane'];
      if (resourceMgr) {
        if (resourceMgr.packageName) {
          if (!packageMapping[resourceMgr.packageName]) {
            packageMapping[resourceMgr.packageName] = {
              category: 'Management',
              'service_name': resourceMgr.dir.split('/')[0]
            };
          }
        } else {
          for (let service in resourceMgr) {
            if (resourceMgr[service].packageName) {
              if (!packageMapping[resourceMgr[service].packageName]) {
                packageMapping[resourceMgr[service].packageName] = {
                  'category': 'Management',
                  'service_name': resourceMgr[service].dir.split('/')[0]
                };
              }
            }
          }
        }
      }
      if (Dataplane) {
        if (Dataplane.packageName) {
          if (!packageMapping[Dataplane.packageName]) {
            packageMapping[Dataplane.packageName] = {
              category: 'Client',
              'service_name': Dataplane.dir.split('/')[0]
            };
          }
        } else {
          for (let service in Dataplane) {
            if (Dataplane[service].packageName) {
              if (!packageMapping[Dataplane[service].packageName]) {
                packageMapping[Dataplane[service].packageName] = {
                  category: 'Client',
                  'service_name': Dataplane[service].dir.split('/')[0]
                };
              }
            }
          }
        }
      }
    }
  }
  packageMapping = Object.keys(packageMapping).sort().reduce((r, k) => (r[k] = packageMapping[k], r), {});
  fs.writeFileSync('./package_service_mapping.json', JSON.stringify(packageMapping, null, 2), { 'encoding': 'utf8' });
});

gulp.task('sort-codegen-mappings', (cb) =>
{
  const codegenMappings = require('./codegen_mappings.json');
  fs.writeFileSync('./codegen_mappings.json', jsonStableStringify(codegenMappings, { space: '  ' }));
});

function findDirProperties(codegenMappingObject, packageFolderPaths) {
  if (codegenMappingObject && typeof codegenMappingObject === 'object') {
    for (const propertyName in codegenMappingObject) {
      if (propertyName) {
        const propertyValue = codegenMappingObject[propertyName];
        if (propertyValue) {
          if (propertyName == 'dir' && typeof propertyValue === 'string') {
            if (!packageFolderPaths.includes(propertyValue)) {
              packageFolderPaths.push(propertyValue);
            }
          }
          else if (propertyValue) {
            findDirProperties(propertyValue, packageFolderPaths);
          }
        }
      }
    }
  }
}

gulp.task('publish-packages', (cb) => {
  const mappings = require('./codegen_mappings.json');
  
  const packageFolderPaths = [];
  findDirProperties(mappings, packageFolderPaths);

  let errorPackages = 0;
  let upToDatePackages = 0;
  let publishedPackages = 0;
  for (const index in packageFolderPaths) {
    if (true) {
      const packageFolderPath = `./lib/services/${packageFolderPaths[index]}`;
      const packageJsonFilePath = `${packageFolderPath}/package.json`;
      if (!fs.existsSync(packageJsonFilePath)) {
        console.log(`ERROR: Package folder ${packageFolderPath} is missing a package.json file.`);
        errorPackages++;
      }
      else {
        const packageJson = require(packageJsonFilePath);
        const packageName = packageJson.name;
        const localPackageVersion = packageJson.version;
        if (!localPackageVersion) {
          console.log(`ERROR: "${packageJsonFilePath}" doesn't have a non-empty version property.`);
          errorPackages++;
        }
        else {
          let npmPackageVersion;
          try {
            const npmViewResult = JSON.parse(execSync(`npm view ${packageName} --json`, { stdio: ['pipe', 'pipe', 'ignore'] }));
            npmPackageVersion = npmViewResult['dist-tags']['latest'];
          }
          catch (error) {
            // This happens if the package doesn't exist in NPM.
          }

          //console.log(`Found package "${packageName}" with local version "${localPackageVersion}" and NPM version "${npmPackageVersion}".`);

          if (localPackageVersion === npmPackageVersion) {
            //console.log(`The NPM package "${packageName}" is up to date.`);
            upToDatePackages++;
          }
          else {
            console.log(`Publishing package "${packageName}" with version "${localPackageVersion}"...`);
            publishedPackages++;
          }
        }
      }
    }
  }

  console.log(`Error packages:      ${errorPackages}`);
  console.log(`Up to date packages: ${upToDatePackages}`);
  console.log(`Published packages:  ${publishedPackages}`);
});